From 795ef49856f860a69fc5ad801c0e2fada9f45cb8 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 18 Sep 2025 17:00:44 +0100 Subject: [PATCH 001/190] Basic trace recording --- Include/internal/pycore_interp_structs.h | 3 + Include/internal/pycore_uop.h | 7 +- Python/bytecodes.c | 29 +- Python/ceval.c | 23 +- Python/ceval_macros.h | 21 + Python/generated_cases.c.h | 12307 +++++++++++++++++++- Python/opcode_targets.h | 747 +- Python/pystate.c | 6 + Tools/cases_generator/target_generator.py | 25 +- Tools/cases_generator/tier1_generator.py | 19 +- Tools/cases_generator/tracer_generator.py | 66 + 11 files changed, 13209 insertions(+), 44 deletions(-) create mode 100644 Tools/cases_generator/tracer_generator.py diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 0e039de8ae05b3..a0a0e8665398a1 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -943,6 +943,9 @@ struct _is { struct types_state types; struct callable_cache callable_cache; PyObject *common_consts[NUM_COMMON_CONSTANTS]; + int jit_tracer_code_curr_size; + _Py_CODEUNIT *jit_tracer_code_buffer; + _Py_CODEUNIT *jit_tracer_initial_instr; bool jit; bool compiling; struct _PyUOpInstruction *jit_uop_buffer; diff --git a/Include/internal/pycore_uop.h b/Include/internal/pycore_uop.h index 4abefd3b95d21a..f364c01ba92083 100644 --- a/Include/internal/pycore_uop.h +++ b/Include/internal/pycore_uop.h @@ -35,10 +35,15 @@ typedef struct _PyUOpInstruction{ #endif } _PyUOpInstruction; -// This is the length of the trace we project initially. +// This is the length of the trace we translate initially. #define UOP_MAX_TRACE_LENGTH 1200 #define UOP_BUFFER_SIZE (UOP_MAX_TRACE_LENGTH * sizeof(_PyUOpInstruction)) +// This is the length of the trace we record. +// This includes the inline caches. +#define TRACE_MAX_TRACE_LENGTH 1000 +#define TRACER_BUFFER_SIZE ((int)(TRACE_MAX_TRACE_LENGTH * sizeof(_Py_CODEUNIT))) + #ifdef __cplusplus } #endif diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 9b993188fb73c7..99b3bc849b6040 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2960,19 +2960,26 @@ dummy_func( oparg >>= 8; start--; } - _PyExecutorObject *executor; - int optimized = _PyOptimizer_Optimize(frame, start, &executor, 0); - if (optimized <= 0) { - this_instr[1].counter = restart_backoff_counter(counter); - ERROR_IF(optimized < 0); + if (tstate->interp->jit_tracer_code_buffer == NULL) { + tstate->interp->jit_tracer_code_buffer = (_Py_CODEUNIT *)_PyObject_VirtualAlloc(TRACER_BUFFER_SIZE); + tstate->interp->jit_tracer_code_curr_size = 0; + if (tstate->interp->jit_tracer_code_buffer == NULL) { + // Don't error, just go to next instruction. + DISPATCH(); + } } - else { - this_instr[1].counter = initial_jump_backoff_counter(); - assert(tstate->current_executor == NULL); - assert(executor != tstate->interp->cold_executor); - tstate->jit_exit = NULL; - TIER1_TO_TIER2(executor); + if (this_instr == tstate->interp->jit_tracer_initial_instr) { + // Looped back to initial instr. End tracing. + LEAVE_TRACING(); + DISPATCH(); + } + ENTER_TRACING(); + // Nothing in the buffer, begin tracing! + if (tstate->interp->jit_tracer_code_curr_size == 0) { + tstate->interp->jit_tracer_initial_instr = this_instr; } + // Not tracing dispatch, normal dispatch because we don't record the current instruction. + DISPATCH(); } else { ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); diff --git a/Python/ceval.c b/Python/ceval.c index 7abbc9e9fd12b6..453ec103484b5a 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -973,6 +973,21 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch) } } +// 1 for trace full, 0 for successful write. +static inline int +add_to_code_trace(PyThreadState *tstate, _Py_CODEUNIT *this_instr) +{ + assert(tstate->interp->jit_tracer_code_curr_size < TRACE_MAX_TRACE_LENGTH); + int curr_size = tstate->interp->jit_tracer_code_curr_size; + int nsize = _PyOpcode_Caches[this_instr->op.code] + 1; + if (curr_size + nsize > TRACE_MAX_TRACE_LENGTH) { + return 1; + } + for (int i = 0; i < nsize; i++) { + tstate->interp->jit_tracer_code_buffer[curr_size + i] = *(this_instr + i); + } + return 0; +} /* _PyEval_EvalFrameDefault is too large to optimize for speed with PGO on MSVC. */ @@ -1102,9 +1117,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int stack_pointer = _PyFrame_GetStackPointer(frame); #if _Py_TAIL_CALL_INTERP # if Py_STATS - return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, instruction_funcptr_table, 0, lastopcode); + return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, instruction_funcptr_handler_table, 0, lastopcode); # else - return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, instruction_funcptr_table, 0); + return _TAIL_CALL_error(frame, stack_pointer, tstate, next_instr, instruction_funcptr_handler_table, 0); # endif #else goto error; @@ -1113,9 +1128,9 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #if _Py_TAIL_CALL_INTERP # if Py_STATS - return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, instruction_funcptr_table, 0, lastopcode); + return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, instruction_funcptr_handler_table, 0, lastopcode); # else - return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, instruction_funcptr_table, 0); + return _TAIL_CALL_start_frame(frame, NULL, tstate, NULL, instruction_funcptr_handler_table, 0); # endif #else goto start_frame; diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 4ed03b7fb01bdf..1afe5d4003792b 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -78,6 +78,13 @@ # define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, instruction_funcptr_table, oparg #endif +# define TRACING_DISPATCH_GOTO() do { \ + if (add_to_code_trace(tstate, this_instr)) { \ + LEAVE_TRACING(); \ + } \ + DISPATCH_GOTO(); \ + } while (0); + #if _Py_TAIL_CALL_INTERP // Note: [[clang::musttail]] works for GCC 15, but not __attribute__((musttail)) at the moment. # define Py_MUSTTAIL [[clang::musttail]] @@ -85,6 +92,9 @@ Py_PRESERVE_NONE_CC typedef PyObject* (*py_tail_call_funcptr)(TAIL_CALL_PARAMS); # define TARGET(op) Py_PRESERVE_NONE_CC PyObject *_TAIL_CALL_##op(TAIL_CALL_PARAMS) +# define TRACING_TARGET(op) Py_PRESERVE_NONE_CC PyObject *_TAIL_CALL_TRACING_##op(TAIL_CALL_PARAMS) +# define LEAVE_TRACING() instruction_funcptr_table = instruction_funcptr_handler_table; +# define ENTER_TRACING() instruction_funcptr_table = instruction_funcptr_tracing_table # define DISPATCH_GOTO() \ do { \ Py_MUSTTAIL return (((py_tail_call_funcptr *)instruction_funcptr_table)[opcode])(TAIL_CALL_ARGS); \ @@ -107,7 +117,10 @@ # define LABEL(name) TARGET(name) #elif USE_COMPUTED_GOTOS # define TARGET(op) TARGET_##op: +# define TRACING_TARGET(op) TARGET_TRACING_##op: # define DISPATCH_GOTO() goto *opcode_targets[opcode] +# define LEAVE_TRACING() opcode_targets = opcode_targets_table; +# define ENTER_TRACING() opcode_targets = opcode_tracing_targets_table; # define JUMP_TO_LABEL(name) goto name; # define JUMP_TO_PREDICTED(name) goto PREDICTED_##name; # define LABEL(name) name: @@ -172,6 +185,14 @@ do { \ JUMP_TO_LABEL(start_frame); \ } while (0) +#define TRACING_DISPATCH() \ + { \ + assert(frame->stackpointer == NULL); \ + NEXTOPARG(); \ + PRE_DISPATCH_GOTO(); \ + TRACING_DISPATCH_GOTO(); \ + } + /* Tuple access macros */ #ifndef Py_DEBUG diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index e33d15f2e51e16..884f2e928733d9 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -7650,25 +7650,24 @@ oparg >>= 8; start--; } - _PyExecutorObject *executor; - _PyFrame_SetStackPointer(frame, stack_pointer); - int optimized = _PyOptimizer_Optimize(frame, start, &executor, 0); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (optimized <= 0) { - this_instr[1].counter = restart_backoff_counter(counter); - if (optimized < 0) { - JUMP_TO_LABEL(error); - } - } - else { + if (tstate->interp->jit_tracer_code_buffer == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); - this_instr[1].counter = initial_jump_backoff_counter(); + tstate->interp->jit_tracer_code_buffer = (_Py_CODEUNIT *)_PyObject_VirtualAlloc(TRACER_BUFFER_SIZE); stack_pointer = _PyFrame_GetStackPointer(frame); - assert(tstate->current_executor == NULL); - assert(executor != tstate->interp->cold_executor); - tstate->jit_exit = NULL; - TIER1_TO_TIER2(executor); + tstate->interp->jit_tracer_code_curr_size = 0; + if (tstate->interp->jit_tracer_code_buffer == NULL) { + DISPATCH(); + } + } + if (this_instr == tstate->interp->jit_tracer_initial_instr) { + LEAVE_TRACING(); + DISPATCH(); + } + ENTER_TRACING(); + if (tstate->interp->jit_tracer_code_curr_size == 0) { + tstate->interp->jit_tracer_initial_instr = this_instr; } + DISPATCH(); } else { ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); @@ -12097,6 +12096,12282 @@ assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } + #ifdef _Py_TIER2 /* BEGIN TRACING INSTRUCTIONS */ + + + TRACING_TARGET(BINARY_OP) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP); + PREDICTED_TRACING_BINARY_OP:; + _Py_CODEUNIT* const this_instr = next_instr - 6; + (void)this_instr; + _PyStackRef lhs; + _PyStackRef rhs; + _PyStackRef res; + // _SPECIALIZE_BINARY_OP + { + rhs = stack_pointer[-1]; + lhs = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(BINARY_OP); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + assert(NB_ADD <= oparg); + assert(oparg <= NB_OPARG_LAST); + } + /* Skip 4 cache entries */ + // _BINARY_OP + { + PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); + PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); + assert(_PyEval_BinaryOps[oparg]); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = lhs; + lhs = res; + stack_pointer[-2] = lhs; + PyStackRef_CLOSE(tmp); + tmp = rhs; + rhs = PyStackRef_NULL; + stack_pointer[-1] = rhs; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_ADD_FLOAT) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_ADD_FLOAT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_TOS_FLOAT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + // _GUARD_NOS_FLOAT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_ADD_FLOAT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval + + ((PyFloatObject *)right_o)->ob_fval; + res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (PyStackRef_IsNull(res)) { + JUMP_TO_LABEL(pop_2_error); + } + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_ADD_INT) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_ADD_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_ADD_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + // _GUARD_NOS_INT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!_PyLong_CheckExactAndCompact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_ADD_INT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + STAT_INC(BINARY_OP, hit); + res = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_ADD_UNICODE) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_ADD_UNICODE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_TOS_UNICODE + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + // _GUARD_NOS_UNICODE + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_ADD_UNICODE + { + right = value; + left = nos; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyUnicode_CheckExact(left_o)); + assert(PyUnicode_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyUnicode_Concat(left_o, right_o); + PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + if (res_o == NULL) { + JUMP_TO_LABEL(pop_2_error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_EXTEND) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_EXTEND; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_EXTEND); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + /* Skip 1 cache entry */ + // _GUARD_BINARY_OP_EXTEND + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *descr = read_obj(&this_instr[2].cache); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; + assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); + assert(d && d->guard); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = d->guard(left_o, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!res) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip -4 cache entry */ + // _BINARY_OP_EXTEND + { + PyObject *descr = read_obj(&this_instr[2].cache); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); + _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; + STAT_INC(BINARY_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = d->action(left_o, right_o); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_INPLACE_ADD_UNICODE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef left; + _PyStackRef right; + // _GUARD_TOS_UNICODE + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + // _GUARD_NOS_UNICODE + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_INPLACE_ADD_UNICODE + { + right = value; + left = nos; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + assert(PyUnicode_CheckExact(left_o)); + assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(right))); + int next_oparg; + #if TIER_ONE + assert(next_instr->op.code == STORE_FAST); + next_oparg = next_instr->op.arg; + #else + next_oparg = (int)CURRENT_OPERAND0(); + #endif + _PyStackRef *target_local = &GETLOCAL(next_oparg); + assert(PyUnicode_CheckExact(left_o)); + if (PyStackRef_AsPyObjectBorrow(*target_local) != left_o) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + STAT_INC(BINARY_OP, hit); + assert(Py_REFCNT(left_o) >= 2 || !PyStackRef_IsHeapSafe(left)); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); + PyObject *right_o = PyStackRef_AsPyObjectSteal(right); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyUnicode_Append(&temp, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + *target_local = PyStackRef_FromPyObjectSteal(temp); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsNull(*target_local)) { + JUMP_TO_LABEL(error); + } + #if TIER_ONE + + assert(next_instr->op.code == STORE_FAST); + SKIP_OVER(1); + #endif + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_MULTIPLY_FLOAT) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_MULTIPLY_FLOAT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_TOS_FLOAT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + // _GUARD_NOS_FLOAT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_MULTIPLY_FLOAT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval * + ((PyFloatObject *)right_o)->ob_fval; + res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (PyStackRef_IsNull(res)) { + JUMP_TO_LABEL(pop_2_error); + } + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_MULTIPLY_INT) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_MULTIPLY_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + // _GUARD_NOS_INT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!_PyLong_CheckExactAndCompact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_MULTIPLY_INT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + STAT_INC(BINARY_OP, hit); + res = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_SUBSCR_DICT) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBSCR_DICT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_DICT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef nos; + _PyStackRef dict_st; + _PyStackRef sub_st; + _PyStackRef res; + // _GUARD_NOS_DICT + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyDict_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBSCR_DICT + { + sub_st = stack_pointer[-1]; + dict_st = nos; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + assert(PyDict_CheckExact(dict)); + STAT_INC(BINARY_OP, hit); + PyObject *res_o; + _PyFrame_SetStackPointer(frame, stack_pointer); + int rc = PyDict_GetItemRef(dict, sub, &res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (rc == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetKeyError(sub); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = sub_st; + sub_st = PyStackRef_NULL; + stack_pointer[-1] = sub_st; + PyStackRef_CLOSE(tmp); + tmp = dict_st; + dict_st = PyStackRef_NULL; + stack_pointer[-2] = dict_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (rc <= 0) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_SUBSCR_GETITEM) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBSCR_GETITEM; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_GETITEM); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef container; + _PyStackRef getitem; + _PyStackRef sub; + _PyStackRef new_frame; + /* Skip 5 cache entries */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + // _BINARY_OP_SUBSCR_CHECK_FUNC + { + container = stack_pointer[-2]; + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); + if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; + PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); + if (getitem_o == NULL) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + assert(PyFunction_Check(getitem_o)); + uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); + if (((PyFunctionObject *)getitem_o)->func_version != cached_version) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); + assert(code->co_argcount == 2); + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + getitem = PyStackRef_FromPyObjectNew(getitem_o); + STAT_INC(BINARY_OP, hit); + } + // _BINARY_OP_SUBSCR_INIT_CALL + { + sub = stack_pointer[-1]; + _PyInterpreterFrame* pushed_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); + pushed_frame->localsplus[0] = container; + pushed_frame->localsplus[1] = sub; + frame->return_offset = 6u ; + new_frame = PyStackRef_Wrap(pushed_frame); + } + // _PUSH_FRAME + { + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_SUBSCR_LIST_INT) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBSCR_LIST_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef list_st; + _PyStackRef sub_st; + _PyStackRef res; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + // _GUARD_NOS_LIST + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyList_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBSCR_LIST_INT + { + sub_st = value; + list_st = nos; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + assert(PyLong_CheckExact(sub)); + assert(PyList_CheckExact(list)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + #ifdef Py_GIL_DISABLED + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + STAT_INC(BINARY_OP, hit); + res = PyStackRef_FromPyObjectSteal(res_o); + #else + if (index >= PyList_GET_SIZE(list)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyList_GET_ITEM(list, index); + assert(res_o != NULL); + res = PyStackRef_FromPyObjectNew(res_o); + #endif + STAT_INC(BINARY_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = list_st; + list_st = res; + stack_pointer[-2] = list_st; + PyStackRef_CLOSE(tmp); + tmp = sub_st; + sub_st = PyStackRef_NULL; + stack_pointer[-1] = sub_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_SUBSCR_LIST_SLICE) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBSCR_LIST_SLICE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_SLICE); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef nos; + _PyStackRef list_st; + _PyStackRef sub_st; + _PyStackRef res; + // _GUARD_TOS_SLICE + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PySlice_Check(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + // _GUARD_NOS_LIST + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyList_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBSCR_LIST_SLICE + { + sub_st = tos; + list_st = nos; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + assert(PySlice_Check(sub)); + assert(PyList_CheckExact(list)); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyList_SliceSubscript(list, sub); + stack_pointer = _PyFrame_GetStackPointer(frame); + STAT_INC(BINARY_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = sub_st; + sub_st = PyStackRef_NULL; + stack_pointer[-1] = sub_st; + PyStackRef_CLOSE(tmp); + tmp = list_st; + list_st = PyStackRef_NULL; + stack_pointer[-2] = list_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_SUBSCR_STR_INT) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBSCR_STR_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_STR_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef str_st; + _PyStackRef sub_st; + _PyStackRef res; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + // _GUARD_NOS_UNICODE + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBSCR_STR_INT + { + sub_st = value; + str_st = nos; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); + assert(PyLong_CheckExact(sub)); + assert(PyUnicode_CheckExact(str)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (PyUnicode_GET_LENGTH(str) <= index) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + Py_UCS4 c = PyUnicode_READ_CHAR(str, index); + if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + STAT_INC(BINARY_OP, hit); + PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(str_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_FromPyObjectBorrow(res_o); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_SUBSCR_TUPLE_INT) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBSCR_TUPLE_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_TUPLE_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef tuple_st; + _PyStackRef sub_st; + _PyStackRef res; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + // _GUARD_NOS_TUPLE + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyTuple_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBSCR_TUPLE_INT + { + sub_st = value; + tuple_st = nos; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); + assert(PyLong_CheckExact(sub)); + assert(PyTuple_CheckExact(tuple)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (index >= PyTuple_GET_SIZE(tuple)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyTuple_GET_ITEM(tuple, index); + assert(res_o != NULL); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + res = PyStackRef_FromPyObjectNew(res_o); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = tuple_st; + tuple_st = res; + stack_pointer[-1] = tuple_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_SUBTRACT_FLOAT) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBTRACT_FLOAT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_TOS_FLOAT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + // _GUARD_NOS_FLOAT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBTRACT_FLOAT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval - + ((PyFloatObject *)right_o)->ob_fval; + res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (PyStackRef_IsNull(res)) { + JUMP_TO_LABEL(pop_2_error); + } + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_SUBTRACT_INT) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBTRACT_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + // _GUARD_NOS_INT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!_PyLong_CheckExactAndCompact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBTRACT_INT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + STAT_INC(BINARY_OP, hit); + res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(BINARY_OP); + } + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_SLICE) { + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_SLICE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BINARY_SLICE); + _PyStackRef container; + _PyStackRef start; + _PyStackRef stop; + _PyStackRef res; + // _SPECIALIZE_BINARY_SLICE + { + #if ENABLE_SPECIALIZATION + OPCODE_DEFERRED_INC(BINARY_SLICE); + #endif /* ENABLE_SPECIALIZATION */ + } + // _BINARY_SLICE + { + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), + PyStackRef_AsPyObjectSteal(stop)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyObject *res_o; + if (slice == NULL) { + res_o = NULL; + } + else { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); + Py_DECREF(slice); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += 2; + } + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(container); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BUILD_INTERPOLATION) { + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_INTERPOLATION; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_INTERPOLATION); + _PyStackRef value; + _PyStackRef str; + _PyStackRef *format; + _PyStackRef interpolation; + format = &stack_pointer[-(oparg & 1)]; + str = stack_pointer[-1 - (oparg & 1)]; + value = stack_pointer[-2 - (oparg & 1)]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + PyObject *str_o = PyStackRef_AsPyObjectBorrow(str); + int conversion = oparg >> 2; + PyObject *format_o; + if (oparg & 1) { + format_o = PyStackRef_AsPyObjectBorrow(format[0]); + } + else { + format_o = &_Py_STR(empty); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *interpolation_o = _PyInterpolation_Build(value_o, str_o, conversion, format_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (oparg & 1) { + stack_pointer += -(oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(format[0]); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + stack_pointer += -(oparg & 1); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(str); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (interpolation_o == NULL) { + JUMP_TO_LABEL(error); + } + interpolation = PyStackRef_FromPyObjectSteal(interpolation_o); + stack_pointer[0] = interpolation; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BUILD_LIST) { + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_LIST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_LIST); + _PyStackRef *values; + _PyStackRef list; + values = &stack_pointer[-oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *list_o = _PyList_FromStackRefStealOnSuccess(values, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (list_o == NULL) { + JUMP_TO_LABEL(error); + } + list = PyStackRef_FromPyObjectStealMortal(list_o); + stack_pointer[-oparg] = list; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BUILD_MAP) { + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_MAP; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_MAP); + _PyStackRef *values; + _PyStackRef map; + values = &stack_pointer[-oparg*2]; + STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); + if (CONVERSION_FAILED(values_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg*2; --_i >= 0;) { + tmp = values[_i]; + values[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg*2; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *map_o = _PyDict_FromItems( + values_o, 2, + values_o+1, 2, + oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg*2; --_i >= 0;) { + tmp = values[_i]; + values[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg*2; + assert(WITHIN_STACK_BOUNDS()); + if (map_o == NULL) { + JUMP_TO_LABEL(error); + } + map = PyStackRef_FromPyObjectStealMortal(map_o); + stack_pointer[0] = map; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BUILD_SET) { + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_SET; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_SET); + _PyStackRef *values; + _PyStackRef set; + values = &stack_pointer[-oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *set_o = PySet_New(NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (set_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = values[_i]; + values[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } + int err = 0; + for (Py_ssize_t i = 0; i < oparg; i++) { + _PyStackRef value = values[i]; + values[i] = PyStackRef_NULL; + if (err == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = _PySet_AddTakeRef((PySetObject *)set_o, PyStackRef_AsPyObjectSteal(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + if (err) { + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(set_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + set = PyStackRef_FromPyObjectStealMortal(set_o); + stack_pointer[-oparg] = set; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BUILD_SLICE) { + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_SLICE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_SLICE); + _PyStackRef *args; + _PyStackRef slice; + args = &stack_pointer[-oparg]; + PyObject *start_o = PyStackRef_AsPyObjectBorrow(args[0]); + PyObject *stop_o = PyStackRef_AsPyObjectBorrow(args[1]); + PyObject *step_o = oparg == 3 ? PyStackRef_AsPyObjectBorrow(args[2]) : NULL; + PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + if (slice_o == NULL) { + JUMP_TO_LABEL(error); + } + slice = PyStackRef_FromPyObjectStealMortal(slice_o); + stack_pointer[0] = slice; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BUILD_STRING) { + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_STRING; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_STRING); + _PyStackRef *pieces; + _PyStackRef str; + pieces = &stack_pointer[-oparg]; + STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); + if (CONVERSION_FAILED(pieces_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = pieces[_i]; + pieces[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } + PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = pieces[_i]; + pieces[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + if (str_o == NULL) { + JUMP_TO_LABEL(error); + } + str = PyStackRef_FromPyObjectSteal(str_o); + stack_pointer[0] = str; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BUILD_TEMPLATE) { + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_TEMPLATE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_TEMPLATE); + _PyStackRef strings; + _PyStackRef interpolations; + _PyStackRef template; + interpolations = stack_pointer[-1]; + strings = stack_pointer[-2]; + PyObject *strings_o = PyStackRef_AsPyObjectBorrow(strings); + PyObject *interpolations_o = PyStackRef_AsPyObjectBorrow(interpolations); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *template_o = _PyTemplate_Build(strings_o, interpolations_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(interpolations); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(strings); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (template_o == NULL) { + JUMP_TO_LABEL(error); + } + template = PyStackRef_FromPyObjectSteal(template_o); + stack_pointer[0] = template; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BUILD_TUPLE) { + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_TUPLE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_TUPLE); + _PyStackRef *values; + _PyStackRef tup; + values = &stack_pointer[-oparg]; + PyObject *tup_o = _PyTuple_FromStackRefStealOnSuccess(values, oparg); + if (tup_o == NULL) { + JUMP_TO_LABEL(error); + } + tup = PyStackRef_FromPyObjectStealMortal(tup_o); + stack_pointer[-oparg] = tup; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CACHE) { + #if _Py_TAIL_CALL_INTERP + int opcode = CACHE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CACHE); + assert(0 && "Executing a cache."); + Py_FatalError("Executing a cache."); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL); + PREDICTED_TRACING_CALL:; + _Py_CODEUNIT* const this_instr = next_instr - 4; + (void)this_instr; + opcode = CALL; + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + // _SPECIALIZE_CALL + { + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_Call(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(CALL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 2 cache entries */ + // _MAYBE_EXPAND_METHOD + { + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + self_or_null = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(method); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + // _DO_CALL + { + args = &stack_pointer[-oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, total_args, NULL, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (new_frame == NULL) { + JUMP_TO_LABEL(error); + } + frame->return_offset = 4u ; + DISPATCH_INLINED(new_frame); + } + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); + if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + } + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_ALLOC_AND_ENTER_INIT) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_ALLOC_AND_ENTER_INIT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef init; + _PyStackRef self; + _PyStackRef *args; + _PyStackRef init_frame; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_AND_ALLOCATE_OBJECT + { + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyStackRef_IsNull(self_or_null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (!PyType_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyTypeObject *tp = (PyTypeObject *)callable_o; + if (FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + assert(tp->tp_new == PyBaseObject_Type.tp_new); + assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); + assert(tp->tp_alloc == PyType_GenericAlloc); + PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; + PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); + PyCodeObject *code = (PyCodeObject *)init_func->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *self_o = PyType_GenericAlloc(tp, 0); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (self_o == NULL) { + JUMP_TO_LABEL(error); + } + self_or_null = PyStackRef_FromPyObjectSteal(self_o); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(init_func); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _CREATE_INIT_FRAME + { + args = &stack_pointer[-oparg]; + self = self_or_null; + init = callable; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( + tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); + assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); + shim->localsplus[0] = PyStackRef_DUP(self); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, init, NULL, args-1, oparg+1, NULL, shim); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FrameClearAndPop(tstate, shim); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; + tstate->py_recursion_remaining--; + init_frame = PyStackRef_Wrap(temp); + } + // _PUSH_FRAME + { + new_frame = init_frame; + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_BOUND_METHOD_EXACT_ARGS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS + { + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _INIT_CALL_BOUND_METHOD_EXACT_ARGS + { + self_or_null = null; + assert(PyStackRef_IsNull(self_or_null)); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + STAT_INC(CALL, hit); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // flush + // _CHECK_FUNCTION_VERSION + { + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_FUNCTION_EXACT_ARGS + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_STACK_SPACE + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _INIT_CALL_PY_EXACT_ARGS + { + args = &stack_pointer[-oparg]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_BOUND_METHOD_GENERAL) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_BOUND_METHOD_GENERAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_METHOD_VERSION + { + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + if (!PyFunction_Check(func)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _EXPAND_METHOD + { + self_or_null = null; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyStackRef_IsNull(self_or_null)); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(callable)); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // flush + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _PY_FRAME_GENERAL + { + args = &stack_pointer[-oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + args, total_args, NULL, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_BUILTIN_CLASS) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_BUILTIN_CLASS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_CLASS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_CLASS + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyType_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyTypeObject *tp = (PyTypeObject *)callable_o; + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + if (tp->tp_vectorcall == NULL) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + STAT_INC(CALL, hit); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_BUILTIN_FAST) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_BUILTIN_FAST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_FAST + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + if (!PyCFunction_CheckExact(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunctionFast_CAST(cfunc)( + PyCFunction_GET_SELF(callable_o), + args_o, + total_args); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_BUILTIN_FAST_WITH_KEYWORDS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_FAST_WITH_KEYWORDS + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + if (!PyCFunction_CheckExact(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCFunctionFastWithKeywords cfunc = + _PyCFunctionFastWithKeywords_CAST(PyCFunction_GET_FUNCTION(callable_o)); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_BUILTIN_O) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_BUILTIN_O; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_O); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_O + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + if (total_args != 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (!PyCFunction_CheckExact(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (_Py_ReachedRecursionLimit(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); + _PyStackRef arg = args[0]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg)); + stack_pointer = _PyFrame_GetStackPointer(frame); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_FUNCTION_EX) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_FUNCTION_EX; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_FUNCTION_EX); + opcode = CALL_FUNCTION_EX; + _PyStackRef func; + _PyStackRef callargs; + _PyStackRef func_st; + _PyStackRef null; + _PyStackRef callargs_st; + _PyStackRef kwargs_st; + _PyStackRef result; + // _MAKE_CALLARGS_A_TUPLE + { + callargs = stack_pointer[-2]; + func = stack_pointer[-4]; + PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); + if (!PyTuple_CheckExact(callargs_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *tuple_o = PySequence_Tuple(callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (tuple_o == NULL) { + JUMP_TO_LABEL(error); + } + _PyStackRef temp = callargs; + callargs = PyStackRef_FromPyObjectSteal(tuple_o); + stack_pointer[-2] = callargs; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + // _DO_CALL_FUNCTION_EX + { + kwargs_st = stack_pointer[-1]; + callargs_st = callargs; + null = stack_pointer[-3]; + func_st = func; + (void)null; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); + PyObject *result_o; + assert(!_PyErr_Occurred(tstate)); + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + assert(PyTuple_CheckExact(callargs)); + PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? + PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; + stack_pointer[-2] = callargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + result_o = PyObject_Call(func, callargs, kwargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!PyFunction_Check(func) && !PyMethod_Check(func)) { + if (result_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(result_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + } + } + else { + if (Py_TYPE(func) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( + tstate, func_st, locals, + nargs, callargs, kwargs, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (new_frame == NULL) { + JUMP_TO_LABEL(error); + } + assert( 1u == 1); + frame->return_offset = 1; + DISPATCH_INLINED(new_frame); + } + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + stack_pointer[-2] = callargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + result_o = PyObject_Call(func, callargs, kwargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(kwargs_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callargs_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(func_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (result_o == NULL) { + JUMP_TO_LABEL(error); + } + result = PyStackRef_FromPyObjectSteal(result_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = result; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_INTRINSIC_1) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_INTRINSIC_1; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_INTRINSIC_1); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + assert(oparg <= MAX_INTRINSIC_1); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_INTRINSIC_2) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_INTRINSIC_2; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_INTRINSIC_2); + _PyStackRef value2_st; + _PyStackRef value1_st; + _PyStackRef res; + value1_st = stack_pointer[-1]; + value2_st = stack_pointer[-2]; + assert(oparg <= MAX_INTRINSIC_2); + PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); + PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); + _PyStackRef tmp = value1_st; + value1_st = PyStackRef_NULL; + stack_pointer[-1] = value1_st; + PyStackRef_CLOSE(tmp); + tmp = value2_st; + value2_st = PyStackRef_NULL; + stack_pointer[-2] = value2_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_ISINSTANCE) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_ISINSTANCE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_ISINSTANCE); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef null; + _PyStackRef callable; + _PyStackRef instance; + _PyStackRef cls; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_THIRD_NULL + { + null = stack_pointer[-3]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _GUARD_CALLABLE_ISINSTANCE + { + callable = stack_pointer[-4]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyInterpreterState *interp = tstate->interp; + if (callable_o != interp->callable_cache.isinstance) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CALL_ISINSTANCE + { + cls = stack_pointer[-1]; + instance = stack_pointer[-2]; + STAT_INC(CALL, hit); + PyObject *inst_o = PyStackRef_AsPyObjectBorrow(instance); + PyObject *cls_o = PyStackRef_AsPyObjectBorrow(cls); + _PyFrame_SetStackPointer(frame, stack_pointer); + int retval = PyObject_IsInstance(inst_o, cls_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (retval < 0) { + JUMP_TO_LABEL(error); + } + (void)null; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(cls); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(instance); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = retval ? PyStackRef_True : PyStackRef_False; + assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_KW) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_KW; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW); + PREDICTED_TRACING_CALL_KW:; + _Py_CODEUNIT* const this_instr = next_instr - 4; + (void)this_instr; + opcode = CALL_KW; + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef res; + // _SPECIALIZE_CALL_KW + { + self_or_null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(CALL_KW); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 2 cache entries */ + // _MAYBE_EXPAND_METHOD_KW + { + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + self_or_null = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(method); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + // _DO_CALL_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(kwnames); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (new_frame == NULL) { + JUMP_TO_LABEL(error); + } + assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); + frame->return_offset = 4u ; + DISPATCH_INLINED(new_frame); + } + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = kwnames; + kwnames = PyStackRef_NULL; + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + stack_pointer[-1] = kwnames; + PyStackRef_CLOSE(tmp); + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-2 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-3 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL_KW) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); + if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = kwnames; + kwnames = PyStackRef_NULL; + stack_pointer[-1] = kwnames; + PyStackRef_CLOSE(tmp); + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-2 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-3 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_KW_BOUND_METHOD) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_KW_BOUND_METHOD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _CHECK_METHOD_VERSION_KW + { + null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + if (!PyFunction_Check(func)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _EXPAND_METHOD_KW + { + self_or_null = null; + assert(PyStackRef_IsNull(self_or_null)); + _PyStackRef callable_s = callable; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(callable)); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable_s); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // flush + // _PY_FRAME_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(kwnames); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_KW_NON_PY) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_KW_NON_PY; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_NON_PY); + opcode = CALL_KW_NON_PY; + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CHECK_IS_NOT_PY_CALLABLE_KW + { + callable = stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + if (Py_TYPE(callable_o) == &PyMethod_Type) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _CALL_KW_NON_PY + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + #if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); + #endif + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = kwnames; + kwnames = PyStackRef_NULL; + stack_pointer[-1] = kwnames; + PyStackRef_CLOSE(tmp); + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-2 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-3 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(kwnames); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_KW_PY) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_KW_PY; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_PY); + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _CHECK_FUNCTION_VERSION_KW + { + callable = stack_pointer[-3 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } + // _PY_FRAME_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(kwnames); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_LEN) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_LEN; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_LEN); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef null; + _PyStackRef callable; + _PyStackRef arg; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_NOS_NULL + { + null = stack_pointer[-2]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _GUARD_CALLABLE_LEN + { + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyInterpreterState *interp = tstate->interp; + if (callable_o != interp->callable_cache.len) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CALL_LEN + { + arg = stack_pointer[-1]; + (void)null; + STAT_INC(CALL, hit); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_ssize_t len_i = PyObject_Length(arg_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (len_i < 0) { + JUMP_TO_LABEL(error); + } + PyObject *res_o = PyLong_FromSsize_t(len_i); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_LIST_APPEND) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_LIST_APPEND; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_LIST_APPEND); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef nos; + _PyStackRef self; + _PyStackRef arg; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_CALLABLE_LIST_APPEND + { + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyInterpreterState *interp = tstate->interp; + if (callable_o != interp->callable_cache.list_append) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _GUARD_NOS_NOT_NULL + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (o == NULL) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _GUARD_NOS_LIST + { + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyList_CheckExact(o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CALL_LIST_APPEND + { + arg = stack_pointer[-1]; + self = nos; + assert(oparg == 1); + PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); + if (!PyList_CheckExact(self_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (!LOCK_OBJECT(self_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + STAT_INC(CALL, hit); + int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); + UNLOCK_OBJECT(self_o); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(self); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } + #if TIER_ONE + + assert(next_instr->op.code == POP_TOP); + SKIP_OVER(1); + #endif + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_METHOD_DESCRIPTOR_FAST) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_METHOD_DESCRIPTOR_FAST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_FAST + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + if (total_args == 0) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyMethodDef *meth = method->d_method; + if (meth->ml_flags != METH_FASTCALL) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); + assert(self != NULL); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + STAT_INC(CALL, hit); + int nargs = total_args - 1; + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCFunctionFast cfunc = _PyCFunctionFast_CAST(meth->ml_meth); + PyObject *res_o = cfunc(self, (args_o + 1), nargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + if (total_args == 0) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyMethodDef *meth = method->d_method; + if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyTypeObject *d_type = method->d_common.d_type; + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); + assert(self != NULL); + if (!Py_IS_TYPE(self, d_type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + STAT_INC(CALL, hit); + int nargs = total_args - 1; + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCFunctionFastWithKeywords cfunc = + _PyCFunctionFastWithKeywords_CAST(meth->ml_meth); + PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_METHOD_DESCRIPTOR_NOARGS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_NOARGS + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + assert(oparg == 0 || oparg == 1); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + if (total_args != 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyMethodDef *meth = method->d_method; + _PyStackRef self_stackref = args[0]; + PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (meth->ml_flags != METH_NOARGS) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (_Py_ReachedRecursionLimit(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(self_stackref); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_METHOD_DESCRIPTOR_O) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_METHOD_DESCRIPTOR_O; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_O + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (total_args != 2) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyMethodDef *meth = method->d_method; + if (meth->ml_flags != METH_O) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (_Py_ReachedRecursionLimit(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + _PyStackRef arg_stackref = arguments[1]; + _PyStackRef self_stackref = arguments[0]; + if (!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), + method->d_common.d_type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, + PyStackRef_AsPyObjectBorrow(self_stackref), + PyStackRef_AsPyObjectBorrow(arg_stackref)); + stack_pointer = _PyFrame_GetStackPointer(frame); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_NON_PY_GENERAL) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_NON_PY_GENERAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_NON_PY_GENERAL); + opcode = CALL_NON_PY_GENERAL; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CHECK_IS_NOT_PY_CALLABLE + { + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + if (Py_TYPE(callable_o) == &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CALL_NON_PY_GENERAL + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + #if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); + #endif + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_PY_EXACT_ARGS) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_PY_EXACT_ARGS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_FUNCTION_VERSION + { + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_FUNCTION_EXACT_ARGS + { + self_or_null = stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_STACK_SPACE + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _INIT_CALL_PY_EXACT_ARGS + { + args = &stack_pointer[-oparg]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_PY_GENERAL) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_PY_GENERAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_FUNCTION_VERSION + { + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _PY_FRAME_GENERAL + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + args, total_args, NULL, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_STR_1) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_STR_1; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_STR_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef null; + _PyStackRef callable; + _PyStackRef arg; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_NOS_NULL + { + null = stack_pointer[-2]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _GUARD_CALLABLE_STR_1 + { + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (callable_o != (PyObject *)&PyUnicode_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CALL_STR_1 + { + arg = stack_pointer[-1]; + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Str(arg_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + (void)callable; + (void)null; + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_TUPLE_1) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_TUPLE_1; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_TUPLE_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef null; + _PyStackRef callable; + _PyStackRef arg; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_NOS_NULL + { + null = stack_pointer[-2]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _GUARD_CALLABLE_TUPLE_1 + { + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (callable_o != (PyObject *)&PyTuple_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CALL_TUPLE_1 + { + arg = stack_pointer[-1]; + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PySequence_Tuple(arg_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + (void)callable; + (void)null; + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_TYPE_1) { + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_TYPE_1; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_TYPE_1); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef null; + _PyStackRef callable; + _PyStackRef arg; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_NOS_NULL + { + null = stack_pointer[-2]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _GUARD_CALLABLE_TYPE_1 + { + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (callable_o != (PyObject *)&PyType_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(CALL); + } + } + // _CALL_TYPE_1 + { + arg = stack_pointer[-1]; + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); + (void)callable; + (void)null; + STAT_INC(CALL, hit); + res = PyStackRef_FromPyObjectNew(Py_TYPE(arg_o)); + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CHECK_EG_MATCH) { + #if _Py_TAIL_CALL_INTERP + int opcode = CHECK_EG_MATCH; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CHECK_EG_MATCH); + _PyStackRef exc_value_st; + _PyStackRef match_type_st; + _PyStackRef rest; + _PyStackRef match; + match_type_st = stack_pointer[-1]; + exc_value_st = stack_pointer[-2]; + PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); + PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = match_type_st; + match_type_st = PyStackRef_NULL; + stack_pointer[-1] = match_type_st; + PyStackRef_CLOSE(tmp); + tmp = exc_value_st; + exc_value_st = PyStackRef_NULL; + stack_pointer[-2] = exc_value_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } + PyObject *match_o = NULL; + PyObject *rest_o = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PyEval_ExceptionGroupMatch(frame, exc_value, match_type, + &match_o, &rest_o); + _PyStackRef tmp = match_type_st; + match_type_st = PyStackRef_NULL; + stack_pointer[-1] = match_type_st; + PyStackRef_CLOSE(tmp); + tmp = exc_value_st; + exc_value_st = PyStackRef_NULL; + stack_pointer[-2] = exc_value_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res < 0) { + JUMP_TO_LABEL(error); + } + assert((match_o == NULL) == (rest_o == NULL)); + if (match_o == NULL) { + JUMP_TO_LABEL(error); + } + if (!Py_IsNone(match_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyErr_SetHandledException(match_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + rest = PyStackRef_FromPyObjectSteal(rest_o); + match = PyStackRef_FromPyObjectSteal(match_o); + stack_pointer[0] = rest; + stack_pointer[1] = match; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CHECK_EXC_MATCH) { + #if _Py_TAIL_CALL_INTERP + int opcode = CHECK_EXC_MATCH; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CHECK_EXC_MATCH); + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyExceptionInstance_Check(left_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyEval_CheckExceptTypeValid(tstate, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = PyErr_GivenExceptionMatches(left_o, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(right); + stack_pointer = _PyFrame_GetStackPointer(frame); + b = res ? PyStackRef_True : PyStackRef_False; + stack_pointer[0] = b; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CLEANUP_THROW) { + #if _Py_TAIL_CALL_INTERP + int opcode = CLEANUP_THROW; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CLEANUP_THROW); + _PyStackRef sub_iter; + _PyStackRef last_sent_val; + _PyStackRef exc_value_st; + _PyStackRef none; + _PyStackRef value; + exc_value_st = stack_pointer[-1]; + last_sent_val = stack_pointer[-2]; + sub_iter = stack_pointer[-3]; + PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); + #if !_Py_TAIL_CALL_INTERP + assert(throwflag); + #endif + assert(exc_value && PyExceptionInstance_Check(exc_value)); + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = sub_iter; + sub_iter = value; + stack_pointer[-3] = sub_iter; + PyStackRef_CLOSE(tmp); + tmp = exc_value_st; + exc_value_st = PyStackRef_NULL; + stack_pointer[-1] = exc_value_st; + PyStackRef_CLOSE(tmp); + tmp = last_sent_val; + last_sent_val = PyStackRef_NULL; + stack_pointer[-2] = last_sent_val; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + none = PyStackRef_None; + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); + monitor_reraise(tstate, frame, this_instr); + JUMP_TO_LABEL(exception_unwind); + } + stack_pointer[0] = none; + stack_pointer[1] = value; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(COMPARE_OP) { + #if _Py_TAIL_CALL_INTERP + int opcode = COMPARE_OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP); + PREDICTED_TRACING_COMPARE_OP:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _SPECIALIZE_COMPARE_OP + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_CompareOp(left, right, next_instr, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(COMPARE_OP); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _COMPARE_OP + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert((oparg >> 5) <= Py_GE); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + if (oparg & 16) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int res_bool = PyObject_IsTrue(res_o); + Py_DECREF(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_bool < 0) { + JUMP_TO_LABEL(error); + } + res = res_bool ? PyStackRef_True : PyStackRef_False; + } + else { + res = PyStackRef_FromPyObjectSteal(res_o); + } + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(COMPARE_OP_FLOAT) { + #if _Py_TAIL_CALL_INTERP + int opcode = COMPARE_OP_FLOAT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_FLOAT); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_TOS_FLOAT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(COMPARE_OP); + } + } + // _GUARD_NOS_FLOAT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(COMPARE_OP); + } + } + /* Skip 1 cache entry */ + // _COMPARE_OP_FLOAT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(COMPARE_OP, hit); + double dleft = PyFloat_AS_DOUBLE(left_o); + double dright = PyFloat_AS_DOUBLE(right_o); + int sign_ish = COMPARISON_BIT(dleft, dright); + PyStackRef_CLOSE_SPECIALIZED(left, _PyFloat_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyFloat_ExactDealloc); + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(COMPARE_OP_INT) { + #if _Py_TAIL_CALL_INTERP + int opcode = COMPARE_OP_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_INT); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(COMPARE_OP); + } + } + // _GUARD_NOS_INT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!_PyLong_CheckExactAndCompact(left_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(COMPARE_OP); + } + } + /* Skip 1 cache entry */ + // _COMPARE_OP_INT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(_PyLong_IsCompact((PyLongObject *)left_o)); + assert(_PyLong_IsCompact((PyLongObject *)right_o)); + STAT_INC(COMPARE_OP, hit); + assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && + _PyLong_DigitCount((PyLongObject *)right_o) <= 1); + Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o); + Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); + int sign_ish = COMPARISON_BIT(ileft, iright); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(COMPARE_OP_STR) { + #if _Py_TAIL_CALL_INTERP + int opcode = COMPARE_OP_STR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_STR); + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_TOS_UNICODE + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(COMPARE_OP); + } + } + // _GUARD_NOS_UNICODE + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(COMPARE_OP); + } + } + /* Skip 1 cache entry */ + // _COMPARE_OP_STR + { + right = value; + left = nos; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(COMPARE_OP, hit); + int eq = _PyUnicode_Equal(left_o, right_o); + assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + assert(eq == 0 || eq == 1); + assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); + assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); + res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CONTAINS_OP) { + #if _Py_TAIL_CALL_INTERP + int opcode = CONTAINS_OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP); + PREDICTED_TRACING_CONTAINS_OP:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; + // _SPECIALIZE_CONTAINS_OP + { + right = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_ContainsOp(right, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(CONTAINS_OP); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _CONTAINS_OP + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = PySequence_Contains(right_o, left_o); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res < 0) { + JUMP_TO_LABEL(error); + } + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[0] = b; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CONTAINS_OP_DICT) { + #if _Py_TAIL_CALL_INTERP + int opcode = CONTAINS_OP_DICT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP_DICT); + static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + // _GUARD_TOS_DICT + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyDict_CheckExact(o)) { + UPDATE_MISS_STATS(CONTAINS_OP); + assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); + JUMP_TO_PREDICTED(CONTAINS_OP); + } + } + /* Skip 1 cache entry */ + // _CONTAINS_OP_DICT + { + right = tos; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyDict_CheckExact(right_o)); + STAT_INC(CONTAINS_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = PyDict_Contains(right_o, left_o); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res < 0) { + JUMP_TO_LABEL(error); + } + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[0] = b; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CONTAINS_OP_SET) { + #if _Py_TAIL_CALL_INTERP + int opcode = CONTAINS_OP_SET; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP_SET); + static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + // _GUARD_TOS_ANY_SET + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyAnySet_CheckExact(o)) { + UPDATE_MISS_STATS(CONTAINS_OP); + assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); + JUMP_TO_PREDICTED(CONTAINS_OP); + } + } + /* Skip 1 cache entry */ + // _CONTAINS_OP_SET + { + right = tos; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyAnySet_CheckExact(right_o)); + STAT_INC(CONTAINS_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PySet_Contains((PySetObject *)right_o, left_o); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res < 0) { + JUMP_TO_LABEL(error); + } + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[0] = b; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CONVERT_VALUE) { + #if _Py_TAIL_CALL_INTERP + int opcode = CONVERT_VALUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CONVERT_VALUE); + _PyStackRef value; + _PyStackRef result; + value = stack_pointer[-1]; + conversion_func conv_fn; + assert(oparg >= FVC_STR && oparg <= FVC_ASCII); + conv_fn = _PyEval_ConversionFuncs[oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (result_o == NULL) { + JUMP_TO_LABEL(error); + } + result = PyStackRef_FromPyObjectSteal(result_o); + stack_pointer[0] = result; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(COPY) { + #if _Py_TAIL_CALL_INTERP + int opcode = COPY; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(COPY); + _PyStackRef bottom; + _PyStackRef top; + bottom = stack_pointer[-1 - (oparg-1)]; + top = PyStackRef_DUP(bottom); + stack_pointer[0] = top; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(COPY_FREE_VARS) { + #if _Py_TAIL_CALL_INTERP + int opcode = COPY_FREE_VARS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(COPY_FREE_VARS); + PyCodeObject *co = _PyFrame_GetCode(frame); + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + PyObject *closure = func->func_closure; + assert(oparg == co->co_nfreevars); + int offset = co->co_nlocalsplus - oparg; + for (int i = 0; i < oparg; ++i) { + PyObject *o = PyTuple_GET_ITEM(closure, i); + frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(DELETE_ATTR) { + #if _Py_TAIL_CALL_INTERP + int opcode = DELETE_ATTR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_ATTR); + _PyStackRef owner; + owner = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(DELETE_DEREF) { + #if _Py_TAIL_CALL_INTERP + int opcode = DELETE_DEREF; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_DEREF); + PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); + if (oldobj == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(oldobj); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(DELETE_FAST) { + #if _Py_TAIL_CALL_INTERP + int opcode = DELETE_FAST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_FAST); + _PyStackRef v = GETLOCAL(oparg); + if (PyStackRef_IsNull(v)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = PyStackRef_NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(DELETE_GLOBAL) { + #if _Py_TAIL_CALL_INTERP + int opcode = DELETE_GLOBAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_GLOBAL); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyDict_Pop(GLOBALS(), name, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + if (err == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(DELETE_NAME) { + #if _Py_TAIL_CALL_INTERP + int opcode = DELETE_NAME; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_NAME); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *ns = LOCALS(); + int err; + if (ns == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_SystemError, + "no locals when deleting %R", name); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_DelItem(ns, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, + name); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(DELETE_SUBSCR) { + #if _Py_TAIL_CALL_INTERP + int opcode = DELETE_SUBSCR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_SUBSCR); + _PyStackRef container; + _PyStackRef sub; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), + PyStackRef_AsPyObjectBorrow(sub)); + _PyStackRef tmp = sub; + sub = PyStackRef_NULL; + stack_pointer[-1] = sub; + PyStackRef_CLOSE(tmp); + tmp = container; + container = PyStackRef_NULL; + stack_pointer[-2] = container; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (err) { + JUMP_TO_LABEL(error); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(DICT_MERGE) { + #if _Py_TAIL_CALL_INTERP + int opcode = DICT_MERGE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DICT_MERGE); + _PyStackRef callable; + _PyStackRef dict; + _PyStackRef update; + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + callable = stack_pointer[-5 - (oparg - 1)]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyDict_MergeEx(dict_o, update_o, 2); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatKwargsError(tstate, callable_o, update_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(update); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(update); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(DICT_UPDATE) { + #if _Py_TAIL_CALL_INTERP + int opcode = DICT_UPDATE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DICT_UPDATE); + _PyStackRef dict; + _PyStackRef update; + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyDict_Update(dict_o, update_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object is not a mapping", + Py_TYPE(update_o)->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(update); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(update); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(END_ASYNC_FOR) { + #if _Py_TAIL_CALL_INTERP + int opcode = END_ASYNC_FOR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(END_ASYNC_FOR); + _PyStackRef awaitable_st; + _PyStackRef exc_st; + exc_st = stack_pointer[-1]; + awaitable_st = stack_pointer[-2]; + JUMPBY(0); + (void)oparg; + PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); + assert(exc && PyExceptionInstance_Check(exc)); + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = exc_st; + exc_st = PyStackRef_NULL; + stack_pointer[-1] = exc_st; + PyStackRef_CLOSE(tmp); + tmp = awaitable_st; + awaitable_st = PyStackRef_NULL; + stack_pointer[-2] = awaitable_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + } + else { + Py_INCREF(exc); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetRaisedException(tstate, exc); + monitor_reraise(tstate, frame, this_instr); + JUMP_TO_LABEL(exception_unwind); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(END_FOR) { + #if _Py_TAIL_CALL_INTERP + int opcode = END_FOR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(END_FOR); + _PyStackRef value; + value = stack_pointer[-1]; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(END_SEND) { + #if _Py_TAIL_CALL_INTERP + int opcode = END_SEND; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(END_SEND); + _PyStackRef receiver; + _PyStackRef value; + _PyStackRef val; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + val = value; + stack_pointer[-2] = val; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(receiver); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(ENTER_EXECUTOR) { + #if _Py_TAIL_CALL_INTERP + int opcode = ENTER_EXECUTOR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(ENTER_EXECUTOR); + opcode = ENTER_EXECUTOR; + #ifdef _Py_TIER2 + PyCodeObject *code = _PyFrame_GetCode(frame); + _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; + assert(executor->vm_data.index == INSTR_OFFSET() - 1); + assert(executor->vm_data.code == code); + assert(executor->vm_data.valid); + assert(tstate->current_executor == NULL); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + opcode = executor->vm_data.opcode; + oparg = (oparg & ~255) | executor->vm_data.oparg; + next_instr = this_instr; + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); + } + DISPATCH_GOTO(); + } + assert(executor != tstate->interp->cold_executor); + tstate->jit_exit = NULL; + TIER1_TO_TIER2(executor); + #else + Py_FatalError("ENTER_EXECUTOR is not supported in this build"); + #endif /* _Py_TIER2 */ + TRACING_DISPATCH(); + } + + TRACING_TARGET(EXIT_INIT_CHECK) { + #if _Py_TAIL_CALL_INTERP + int opcode = EXIT_INIT_CHECK; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(EXIT_INIT_CHECK); + _PyStackRef should_be_none; + should_be_none = stack_pointer[-1]; + if (!PyStackRef_IsNone(should_be_none)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyErr_Format(PyExc_TypeError, + "__init__() should return None, not '%.200s'", + Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(EXTENDED_ARG) { + #if _Py_TAIL_CALL_INTERP + int opcode = EXTENDED_ARG; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(EXTENDED_ARG); + opcode = EXTENDED_ARG; + assert(oparg); + opcode = next_instr->op.code; + oparg = oparg << 8 | next_instr->op.arg; + PRE_DISPATCH_GOTO(); + DISPATCH_GOTO(); + } + + TRACING_TARGET(FORMAT_SIMPLE) { + #if _Py_TAIL_CALL_INTERP + int opcode = FORMAT_SIMPLE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(FORMAT_SIMPLE); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Format(value_o, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + else { + res = value; + stack_pointer += -1; + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(FORMAT_WITH_SPEC) { + #if _Py_TAIL_CALL_INTERP + int opcode = FORMAT_WITH_SPEC; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(FORMAT_WITH_SPEC); + _PyStackRef value; + _PyStackRef fmt_spec; + _PyStackRef res; + fmt_spec = stack_pointer[-1]; + value = stack_pointer[-2]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); + _PyStackRef tmp = fmt_spec; + fmt_spec = PyStackRef_NULL; + stack_pointer[-1] = fmt_spec; + PyStackRef_CLOSE(tmp); + tmp = value; + value = PyStackRef_NULL; + stack_pointer[-2] = value; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(FOR_ITER) { + #if _Py_TAIL_CALL_INTERP + int opcode = FOR_ITER; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER); + PREDICTED_TRACING_FOR_ITER:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef iter; + _PyStackRef null_or_index; + _PyStackRef next; + // _SPECIALIZE_FOR_ITER + { + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_ForIter(iter, null_or_index, next_instr, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(FOR_ITER); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _FOR_ITER + { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!PyStackRef_IsValid(item)) { + if (PyStackRef_IsError(item)) { + JUMP_TO_LABEL(error); + } + JUMPBY(oparg + 1); + stack_pointer[-1] = null_or_index; + TRACING_DISPATCH(); + } + next = item; + } + stack_pointer[-1] = null_or_index; + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(FOR_ITER_GEN) { + #if _Py_TAIL_CALL_INTERP + int opcode = FOR_ITER_GEN; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_GEN); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef gen_frame; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } + } + // _FOR_ITER_GEN_FRAME + { + iter = stack_pointer[-2]; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(gen) != &PyGen_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } + #ifdef Py_GIL_DISABLED + if (!_PyObject_IsUniquelyReferenced((PyObject *)gen)) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } + #endif + if (gen->gi_frame_state >= FRAME_EXECUTING) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } + STAT_INC(FOR_ITER, hit); + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + _PyFrame_StackPush(pushed_frame, PyStackRef_None); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + pushed_frame->previous = frame; + frame->return_offset = (uint16_t)( 2u + oparg); + gen_frame = PyStackRef_Wrap(pushed_frame); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(FOR_ITER_LIST) { + #if _Py_TAIL_CALL_INTERP + int opcode = FOR_ITER_LIST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_LIST); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef null_or_index; + _PyStackRef next; + /* Skip 1 cache entry */ + // _ITER_CHECK_LIST + { + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o) != &PyList_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } + assert(PyStackRef_IsTaggedInt(null_or_index)); + #ifdef Py_GIL_DISABLED + if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } + #endif + } + // _ITER_JUMP_LIST + { + #ifdef Py_GIL_DISABLED + + #else + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(list_o) == &PyList_Type); + STAT_INC(FOR_ITER, hit); + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { + null_or_index = PyStackRef_TagInt(-1); + JUMPBY(oparg + 1); + stack_pointer[-1] = null_or_index; + TRACING_DISPATCH(); + } + #endif + } + // _ITER_NEXT_LIST + { + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(PyList_CheckExact(list_o)); + #ifdef Py_GIL_DISABLED + assert(_Py_IsOwnedByCurrentThread(list_o) || + _PyObject_GC_IS_SHARED(list_o)); + STAT_INC(FOR_ITER, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (result < 0) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } + if (result == 0) { + null_or_index = PyStackRef_TagInt(-1); + JUMPBY(oparg + 1); + stack_pointer[-1] = null_or_index; + TRACING_DISPATCH(); + } + #else + next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index))); + #endif + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); + } + stack_pointer[-1] = null_or_index; + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(FOR_ITER_RANGE) { + #if _Py_TAIL_CALL_INTERP + int opcode = FOR_ITER_RANGE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_RANGE); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef next; + /* Skip 1 cache entry */ + // _ITER_CHECK_RANGE + { + iter = stack_pointer[-2]; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(r) != &PyRangeIter_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } + #ifdef Py_GIL_DISABLED + if (!_PyObject_IsUniquelyReferenced((PyObject *)r)) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } + #endif + } + // _ITER_JUMP_RANGE + { + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(r) == &PyRangeIter_Type); + #ifdef Py_GIL_DISABLED + assert(_PyObject_IsUniquelyReferenced((PyObject *)r)); + #endif + STAT_INC(FOR_ITER, hit); + if (r->len <= 0) { + JUMPBY(oparg + 1); + TRACING_DISPATCH(); + } + } + // _ITER_NEXT_RANGE + { + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(r) == &PyRangeIter_Type); + #ifdef Py_GIL_DISABLED + assert(_PyObject_IsUniquelyReferenced((PyObject *)r)); + #endif + assert(r->len > 0); + long value = r->start; + r->start = value + r->step; + r->len--; + PyObject *res = PyLong_FromLong(value); + if (res == NULL) { + JUMP_TO_LABEL(error); + } + next = PyStackRef_FromPyObjectSteal(res); + } + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(FOR_ITER_TUPLE) { + #if _Py_TAIL_CALL_INTERP + int opcode = FOR_ITER_TUPLE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef null_or_index; + _PyStackRef next; + /* Skip 1 cache entry */ + // _ITER_CHECK_TUPLE + { + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o) != &PyTuple_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(FOR_ITER); + } + assert(PyStackRef_IsTaggedInt(null_or_index)); + } + // _ITER_JUMP_TUPLE + { + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + (void)tuple_o; + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + STAT_INC(FOR_ITER, hit); + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { + null_or_index = PyStackRef_TagInt(-1); + JUMPBY(oparg + 1); + stack_pointer[-1] = null_or_index; + TRACING_DISPATCH(); + } + } + // _ITER_NEXT_TUPLE + { + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + uintptr_t i = PyStackRef_UntagInt(null_or_index); + assert((size_t)i < (size_t)PyTuple_GET_SIZE(tuple_o)); + next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(tuple_o, i)); + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); + } + stack_pointer[-1] = null_or_index; + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(GET_AITER) { + #if _Py_TAIL_CALL_INTERP + int opcode = GET_AITER; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_AITER); + _PyStackRef obj; + _PyStackRef iter; + obj = stack_pointer[-1]; + unaryfunc getter = NULL; + PyObject *obj_o = PyStackRef_AsPyObjectBorrow(obj); + PyObject *iter_o; + PyTypeObject *type = Py_TYPE(obj_o); + if (type->tp_as_async != NULL) { + getter = type->tp_as_async->am_aiter; + } + if (getter == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' requires an object with " + "__aiter__ method, got %.100s", + type->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(obj); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + iter_o = (*getter)(obj_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(obj); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (iter_o == NULL) { + JUMP_TO_LABEL(error); + } + if (Py_TYPE(iter_o)->tp_as_async == NULL || + Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' received an object from __aiter__ " + "that does not implement __anext__: %.100s", + Py_TYPE(iter_o)->tp_name); + Py_DECREF(iter_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + stack_pointer[0] = iter; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(GET_ANEXT) { + #if _Py_TAIL_CALL_INTERP + int opcode = GET_ANEXT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_ANEXT); + _PyStackRef aiter; + _PyStackRef awaitable; + aiter = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (awaitable_o == NULL) { + JUMP_TO_LABEL(error); + } + awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); + stack_pointer[0] = awaitable; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(GET_AWAITABLE) { + #if _Py_TAIL_CALL_INTERP + int opcode = GET_AWAITABLE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_AWAITABLE); + _PyStackRef iterable; + _PyStackRef iter; + iterable = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (iter_o == NULL) { + JUMP_TO_LABEL(error); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + stack_pointer[0] = iter; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(GET_ITER) { + #if _Py_TAIL_CALL_INTERP + int opcode = GET_ITER; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_ITER); + _PyStackRef iterable; + _PyStackRef iter; + _PyStackRef index_or_null; + iterable = stack_pointer[-1]; + #ifdef Py_STATS + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_GatherStats_GetIter(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + #endif + + PyTypeObject *tp = PyStackRef_TYPE(iterable); + if (tp == &PyTuple_Type || tp == &PyList_Type) { + iter = iterable; + index_or_null = PyStackRef_TagInt(0); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (iter_o == NULL) { + JUMP_TO_LABEL(error); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + index_or_null = PyStackRef_NULL; + stack_pointer += 1; + } + stack_pointer[-1] = iter; + stack_pointer[0] = index_or_null; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(GET_LEN) { + #if _Py_TAIL_CALL_INTERP + int opcode = GET_LEN; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_LEN); + _PyStackRef obj; + _PyStackRef len; + obj = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (len_i < 0) { + JUMP_TO_LABEL(error); + } + PyObject *len_o = PyLong_FromSsize_t(len_i); + if (len_o == NULL) { + JUMP_TO_LABEL(error); + } + len = PyStackRef_FromPyObjectSteal(len_o); + stack_pointer[0] = len; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(GET_YIELD_FROM_ITER) { + #if _Py_TAIL_CALL_INTERP + int opcode = GET_YIELD_FROM_ITER; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_YIELD_FROM_ITER); + _PyStackRef iterable; + _PyStackRef iter; + iterable = stack_pointer[-1]; + PyObject *iterable_o = PyStackRef_AsPyObjectBorrow(iterable); + if (PyCoro_CheckExact(iterable_o)) { + if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetString(tstate, PyExc_TypeError, + "cannot 'yield from' a coroutine object " + "in a non-coroutine generator"); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + iter = iterable; + } + else if (PyGen_CheckExact(iterable_o)) { + iter = iterable; + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = PyObject_GetIter(iterable_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (iter_o == NULL) { + JUMP_TO_LABEL(error); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = iterable; + iterable = iter; + stack_pointer[-1] = iterable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + stack_pointer[-1] = iter; + TRACING_DISPATCH(); + } + + TRACING_TARGET(IMPORT_FROM) { + #if _Py_TAIL_CALL_INTERP + int opcode = IMPORT_FROM; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IMPORT_FROM); + _PyStackRef from; + _PyStackRef res; + from = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(IMPORT_NAME) { + #if _Py_TAIL_CALL_INTERP + int opcode = IMPORT_NAME; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IMPORT_NAME); + _PyStackRef level; + _PyStackRef fromlist; + _PyStackRef res; + fromlist = stack_pointer[-1]; + level = stack_pointer[-2]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyEval_ImportName(tstate, frame, name, + PyStackRef_AsPyObjectBorrow(fromlist), + PyStackRef_AsPyObjectBorrow(level)); + _PyStackRef tmp = fromlist; + fromlist = PyStackRef_NULL; + stack_pointer[-1] = fromlist; + PyStackRef_CLOSE(tmp); + tmp = level; + level = PyStackRef_NULL; + stack_pointer[-2] = level; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_CALL) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_CALL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(INSTRUMENTED_CALL); + opcode = INSTRUMENTED_CALL; + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef func; + _PyStackRef maybe_self; + _PyStackRef *args; + _PyStackRef res; + /* Skip 3 cache entries */ + // _MAYBE_EXPAND_METHOD + { + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + self_or_null = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(method); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + // _MONITOR_CALL + { + args = &stack_pointer[-oparg]; + maybe_self = self_or_null; + func = callable; + int is_meth = !PyStackRef_IsNull(maybe_self); + PyObject *function = PyStackRef_AsPyObjectBorrow(func); + PyObject *arg0; + if (is_meth) { + arg0 = PyStackRef_AsPyObjectBorrow(maybe_self); + } + else if (oparg) { + arg0 = PyStackRef_AsPyObjectBorrow(args[0]); + } + else { + arg0 = &_PyInstrumentation_MISSING; + } + stack_pointer[-2 - oparg] = func; + stack_pointer[-1 - oparg] = maybe_self; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg0 + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } + } + // _DO_CALL + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, total_args, NULL, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (new_frame == NULL) { + JUMP_TO_LABEL(error); + } + frame->return_offset = 4u ; + DISPATCH_INLINED(new_frame); + } + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); + if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + } + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_CALL_FUNCTION_EX; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); + opcode = INSTRUMENTED_CALL_FUNCTION_EX; + _PyStackRef func; + _PyStackRef callargs; + _PyStackRef func_st; + _PyStackRef null; + _PyStackRef callargs_st; + _PyStackRef kwargs_st; + _PyStackRef result; + // _MAKE_CALLARGS_A_TUPLE + { + callargs = stack_pointer[-2]; + func = stack_pointer[-4]; + PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); + if (!PyTuple_CheckExact(callargs_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *tuple_o = PySequence_Tuple(callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (tuple_o == NULL) { + JUMP_TO_LABEL(error); + } + _PyStackRef temp = callargs; + callargs = PyStackRef_FromPyObjectSteal(tuple_o); + stack_pointer[-2] = callargs; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + // _DO_CALL_FUNCTION_EX + { + kwargs_st = stack_pointer[-1]; + callargs_st = callargs; + null = stack_pointer[-3]; + func_st = func; + (void)null; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); + PyObject *result_o; + assert(!_PyErr_Occurred(tstate)); + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + assert(PyTuple_CheckExact(callargs)); + PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? + PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; + stack_pointer[-2] = callargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + result_o = PyObject_Call(func, callargs, kwargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!PyFunction_Check(func) && !PyMethod_Check(func)) { + if (result_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(result_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + } + } + else { + if (Py_TYPE(func) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( + tstate, func_st, locals, + nargs, callargs, kwargs, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (new_frame == NULL) { + JUMP_TO_LABEL(error); + } + assert( 1u == 1); + frame->return_offset = 1; + DISPATCH_INLINED(new_frame); + } + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + stack_pointer[-2] = callargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + result_o = PyObject_Call(func, callargs, kwargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(kwargs_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callargs_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(func_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (result_o == NULL) { + JUMP_TO_LABEL(error); + } + result = PyStackRef_FromPyObjectSteal(result_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = result; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_CALL_KW) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_CALL_KW; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); + opcode = INSTRUMENTED_CALL_KW; + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _MAYBE_EXPAND_METHOD_KW + { + self_or_null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + self_or_null = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(method); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + // _MONITOR_CALL_KW + { + args = &stack_pointer[-1 - oparg]; + int is_meth = !PyStackRef_IsNull(self_or_null); + PyObject *arg; + if (is_meth) { + arg = PyStackRef_AsPyObjectBorrow(self_or_null); + } + else if (args) { + arg = PyStackRef_AsPyObjectBorrow(args[0]); + } + else { + arg = &_PyInstrumentation_MISSING; + } + PyObject *function = PyStackRef_AsPyObjectBorrow(callable); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } + } + // _DO_CALL_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(kwnames); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (new_frame == NULL) { + JUMP_TO_LABEL(error); + } + assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); + frame->return_offset = 4u ; + DISPATCH_INLINED(new_frame); + } + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = kwnames; + kwnames = PyStackRef_NULL; + stack_pointer[-1] = kwnames; + PyStackRef_CLOSE(tmp); + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-2 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-3 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL_KW) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); + if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = kwnames; + kwnames = PyStackRef_NULL; + stack_pointer[-1] = kwnames; + PyStackRef_CLOSE(tmp); + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-2 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-3 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_END_ASYNC_FOR) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_END_ASYNC_FOR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_ASYNC_FOR); + _PyStackRef awaitable_st; + _PyStackRef exc_st; + // _MONITOR_END_ASYNC_FOR + { + assert((next_instr-oparg)->op.code == END_SEND || (next_instr-oparg)->op.code >= MIN_INSTRUMENTED_OPCODE); + INSTRUMENTED_JUMP(next_instr-oparg, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT); + } + // _END_ASYNC_FOR + { + exc_st = stack_pointer[-1]; + awaitable_st = stack_pointer[-2]; + JUMPBY(0); + (void)oparg; + PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); + assert(exc && PyExceptionInstance_Check(exc)); + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = exc_st; + exc_st = PyStackRef_NULL; + stack_pointer[-1] = exc_st; + PyStackRef_CLOSE(tmp); + tmp = awaitable_st; + awaitable_st = PyStackRef_NULL; + stack_pointer[-2] = awaitable_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + } + else { + Py_INCREF(exc); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetRaisedException(tstate, exc); + monitor_reraise(tstate, frame, this_instr); + JUMP_TO_LABEL(exception_unwind); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_END_FOR) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_END_FOR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_FOR); + _PyStackRef receiver; + _PyStackRef value; + value = stack_pointer[-1]; + receiver = stack_pointer[-3]; + if (PyStackRef_GenCheck(receiver)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_END_SEND) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_END_SEND; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_SEND); + _PyStackRef receiver; + _PyStackRef value; + _PyStackRef val; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); + if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } + } + val = value; + stack_pointer[-2] = val; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(receiver); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_FOR_ITER) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_FOR_ITER; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); + _PyStackRef iter; + _PyStackRef null_or_index; + _PyStackRef next; + /* Skip 1 cache entry */ + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!PyStackRef_IsValid(item)) { + if (PyStackRef_IsError(item)) { + JUMP_TO_LABEL(error); + } + JUMPBY(oparg + 1); + stack_pointer[-1] = null_or_index; + TRACING_DISPATCH(); + } + next = item; + INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); + stack_pointer[-1] = null_or_index; + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_INSTRUCTION) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_INSTRUCTION; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); + opcode = INSTRUMENTED_INSTRUCTION; + _PyFrame_SetStackPointer(frame, stack_pointer); + int next_opcode = _Py_call_instrumentation_instruction( + tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (next_opcode < 0) { + JUMP_TO_LABEL(error); + } + next_instr = this_instr; + if (_PyOpcode_Caches[next_opcode]) { + PAUSE_ADAPTIVE_COUNTER(next_instr[1].counter); + } + assert(next_opcode > 0 && next_opcode < 256); + opcode = next_opcode; + DISPATCH_GOTO(); + } + + TRACING_TARGET(INSTRUMENTED_JUMP_BACKWARD) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_JUMP_BACKWARD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); + /* Skip 1 cache entry */ + // _CHECK_PERIODIC + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + // _MONITOR_JUMP_BACKWARD + { + INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_JUMP_FORWARD) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_JUMP_FORWARD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_LINE) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_LINE; + (void)(opcode); + #endif + _Py_CODEUNIT* const prev_instr = frame->instr_ptr; + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_LINE); + opcode = INSTRUMENTED_LINE; + int original_opcode = 0; + if (tstate->tracing) { + PyCodeObject *code = _PyFrame_GetCode(frame); + int index = (int)(this_instr - _PyFrame_GetBytecode(frame)); + original_opcode = code->_co_monitoring->lines->data[index*code->_co_monitoring->lines->bytes_per_entry]; + next_instr = this_instr; + } else { + _PyFrame_SetStackPointer(frame, stack_pointer); + original_opcode = _Py_call_instrumentation_line( + tstate, frame, this_instr, prev_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (original_opcode < 0) { + next_instr = this_instr+1; + JUMP_TO_LABEL(error); + } + next_instr = frame->instr_ptr; + if (next_instr != this_instr) { + TRACING_DISPATCH(); + } + } + if (_PyOpcode_Caches[original_opcode]) { + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); + PAUSE_ADAPTIVE_COUNTER(cache->counter); + } + opcode = original_opcode; + DISPATCH_GOTO(); + } + + TRACING_TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_LOAD_SUPER_ATTR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); + opcode = INSTRUMENTED_LOAD_SUPER_ATTR; + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _LOAD_SUPER_ATTR + { + self_st = stack_pointer[-1]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } + } + PyObject *stack[] = {class, self}; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + if (super == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(super); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + if (super == NULL) { + JUMP_TO_LABEL(error); + } + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = PyObject_GetAttr(super, name); + Py_DECREF(super); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) { + JUMP_TO_LABEL(error); + } + attr = PyStackRef_FromPyObjectSteal(attr_o); + } + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[1]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer[0] = attr; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_NOT_TAKEN) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_NOT_TAKEN; + (void)(opcode); + #endif + _Py_CODEUNIT* const prev_instr = frame->instr_ptr; + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_NOT_TAKEN); + (void)this_instr; + INSTRUMENTED_JUMP(prev_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_POP_ITER) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_POP_ITER; + (void)(opcode); + #endif + _Py_CODEUNIT* const prev_instr = frame->instr_ptr; + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_POP_ITER); + _PyStackRef iter; + _PyStackRef index_or_null; + index_or_null = stack_pointer[-1]; + iter = stack_pointer[-2]; + (void)index_or_null; + INSTRUMENTED_JUMP(prev_instr, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iter); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_POP_JUMP_IF_FALSE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); + _PyStackRef cond; + /* Skip 1 cache entry */ + cond = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(cond)); + int jump = PyStackRef_IsFalse(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_POP_JUMP_IF_NONE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); + _PyStackRef value; + /* Skip 1 cache entry */ + value = stack_pointer[-1]; + int jump = PyStackRef_IsNone(value); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } + else { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += 1; + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_POP_JUMP_IF_NOT_NONE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); + _PyStackRef value; + /* Skip 1 cache entry */ + value = stack_pointer[-1]; + int jump = !PyStackRef_IsNone(value); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } + else { + stack_pointer += -1; + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_POP_JUMP_IF_TRUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); + _PyStackRef cond; + /* Skip 1 cache entry */ + cond = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(cond)); + int jump = PyStackRef_IsTrue(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_RESUME) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_RESUME; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RESUME); + // _LOAD_BYTECODE + { + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_CODEUNIT *bytecode = + _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (bytecode == NULL) { + JUMP_TO_LABEL(error); + } + ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); + frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; + frame->instr_ptr = bytecode + off; + next_instr = frame->instr_ptr; + TRACING_DISPATCH(); + } + #endif + } + // _MAYBE_INSTRUMENT + { + #ifdef Py_GIL_DISABLED + + int check_instrumentation = 1; + #else + int check_instrumentation = (tstate->tracing == 0); + #endif + if (check_instrumentation) { + uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + if (code_version != global_version) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } + next_instr = this_instr; + TRACING_DISPATCH(); + } + } + } + // _CHECK_PERIODIC_IF_NOT_YIELD_FROM + { + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + } + // _MONITOR_RESUME + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation( + tstate, oparg > 0, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } + if (frame->instr_ptr != this_instr) { + next_instr = frame->instr_ptr; + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_RETURN_VALUE) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_RETURN_VALUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); + _PyStackRef val; + _PyStackRef retval; + _PyStackRef res; + // _RETURN_VALUE_EVENT + { + val = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } + } + // _RETURN_VALUE + { + retval = val; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + _PyStackRef temp = PyStackRef_MakeHeapSafe(retval); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(STACK_LEVEL() == 0); + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_IP(frame->return_offset); + res = temp; + LLTRACE_RESUME_FRAME(); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_YIELD_VALUE) { + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_YIELD_VALUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); + _PyStackRef val; + _PyStackRef retval; + _PyStackRef value; + // _YIELD_VALUE_EVENT + { + val = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_YIELD, + frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } + if (frame->instr_ptr != this_instr) { + next_instr = frame->instr_ptr; + TRACING_DISPATCH(); + } + } + // _YIELD_VALUE + { + retval = val; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + frame->instr_ptr++; + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + _PyStackRef temp = retval; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + #if TIER_ONE + assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || + frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #endif + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + value = PyStackRef_MakeHeapSafe(temp); + LLTRACE_RESUME_FRAME(); + } + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INTERPRETER_EXIT) { + #if _Py_TAIL_CALL_INTERP + int opcode = INTERPRETER_EXIT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INTERPRETER_EXIT); + _PyStackRef retval; + retval = stack_pointer[-1]; + assert(frame->owner == FRAME_OWNED_BY_INTERPRETER); + assert(_PyFrame_IsIncomplete(frame)); + tstate->current_frame = frame->previous; + assert(!_PyErr_Occurred(tstate)); + PyObject *result = PyStackRef_AsPyObjectSteal(retval); + #if !_Py_TAIL_CALL_INTERP + assert(frame == &entry.frame); + #endif + #ifdef _Py_TIER2 + _PyStackRef executor = frame->localsplus[0]; + assert(tstate->current_executor == NULL); + if (!PyStackRef_IsNull(executor)) { + tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(executor); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += 1; + } + #endif + LLTRACE_RESUME_FRAME(); + return result; + } + + TRACING_TARGET(IS_OP) { + #if _Py_TAIL_CALL_INTERP + int opcode = IS_OP; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IS_OP); + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + b = res ? PyStackRef_True : PyStackRef_False; + stack_pointer[0] = b; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(JUMP_BACKWARD) { + #if _Py_TAIL_CALL_INTERP + int opcode = JUMP_BACKWARD; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(JUMP_BACKWARD); + PREDICTED_TRACING_JUMP_BACKWARD:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + /* Skip 1 cache entry */ + // _SPECIALIZE_JUMP_BACKWARD + { + #if ENABLE_SPECIALIZATION + if (this_instr->op.code == JUMP_BACKWARD) { + this_instr->op.code = tstate->interp->jit ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; + next_instr = this_instr; + DISPATCH_SAME_OPARG(); + } + #endif + } + // _CHECK_PERIODIC + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + // _JUMP_BACKWARD_NO_INTERRUPT + { + assert(oparg <= INSTR_OFFSET()); + JUMPBY(-oparg); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(JUMP_BACKWARD_JIT) { + #if _Py_TAIL_CALL_INTERP + int opcode = JUMP_BACKWARD_JIT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(JUMP_BACKWARD_JIT); + static_assert(1 == 1, "incorrect cache size"); + /* Skip 1 cache entry */ + // _CHECK_PERIODIC + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + // _JUMP_BACKWARD_NO_INTERRUPT + { + assert(oparg <= INSTR_OFFSET()); + JUMPBY(-oparg); + } + // _JIT + { + #ifdef _Py_TIER2 + _Py_BackoffCounter counter = this_instr[1].counter; + if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD_JIT) { + _Py_CODEUNIT *start = this_instr; + while (oparg > 255) { + oparg >>= 8; + start--; + } + if (tstate->interp->jit_tracer_code_buffer == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + tstate->interp->jit_tracer_code_buffer = (_Py_CODEUNIT *)_PyObject_VirtualAlloc(TRACER_BUFFER_SIZE); + stack_pointer = _PyFrame_GetStackPointer(frame); + tstate->interp->jit_tracer_code_curr_size = 0; + if (tstate->interp->jit_tracer_code_buffer == NULL) { + TRACING_DISPATCH(); + } + } + if (this_instr == tstate->interp->jit_tracer_initial_instr) { + LEAVE_TRACING(); + TRACING_DISPATCH(); + } + ENTER_TRACING(); + if (tstate->interp->jit_tracer_code_curr_size == 0) { + tstate->interp->jit_tracer_initial_instr = this_instr; + } + TRACING_DISPATCH(); + } + else { + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + } + #endif + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(JUMP_BACKWARD_NO_INTERRUPT) { + #if _Py_TAIL_CALL_INTERP + int opcode = JUMP_BACKWARD_NO_INTERRUPT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); + assert(oparg <= INSTR_OFFSET()); + JUMPBY(-oparg); + TRACING_DISPATCH(); + } + + TRACING_TARGET(JUMP_BACKWARD_NO_JIT) { + #if _Py_TAIL_CALL_INTERP + int opcode = JUMP_BACKWARD_NO_JIT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(JUMP_BACKWARD_NO_JIT); + static_assert(1 == 1, "incorrect cache size"); + /* Skip 1 cache entry */ + // _CHECK_PERIODIC + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + // _JUMP_BACKWARD_NO_INTERRUPT + { + assert(oparg <= INSTR_OFFSET()); + JUMPBY(-oparg); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(JUMP_FORWARD) { + #if _Py_TAIL_CALL_INTERP + int opcode = JUMP_FORWARD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(JUMP_FORWARD); + JUMPBY(oparg); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LIST_APPEND) { + #if _Py_TAIL_CALL_INTERP + int opcode = LIST_APPEND; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LIST_APPEND); + _PyStackRef list; + _PyStackRef v; + v = stack_pointer[-1]; + list = stack_pointer[-2 - (oparg-1)]; + int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), + PyStackRef_AsPyObjectSteal(v)); + if (err < 0) { + JUMP_TO_LABEL(pop_1_error); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LIST_EXTEND) { + #if _Py_TAIL_CALL_INTERP + int opcode = LIST_EXTEND; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LIST_EXTEND); + _PyStackRef list_st; + _PyStackRef iterable_st; + iterable_st = stack_pointer[-1]; + list_st = stack_pointer[-2 - (oparg-1)]; + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (none_val == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches && + (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) + { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Clear(tstate); + _PyErr_Format(tstate, PyExc_TypeError, + "Value after * must be an iterable, not %.200s", + Py_TYPE(iterable)->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + assert(Py_IsNone(none_val)); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR); + PREDICTED_TRACING_LOAD_ATTR:; + _Py_CODEUNIT* const this_instr = next_instr - 10; + (void)this_instr; + _PyStackRef owner; + _PyStackRef *attr; + _PyStackRef *self_or_null; + // _SPECIALIZE_LOAD_ATTR + { + owner = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_LoadAttr(owner, next_instr, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(LOAD_ATTR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 8 cache entries */ + // _LOAD_ATTR + { + attr = &stack_pointer[-1]; + self_or_null = &stack_pointer[0]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + if (oparg & 1) { + *attr = PyStackRef_NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + int is_meth = _PyObject_GetMethodStackRef(tstate, PyStackRef_AsPyObjectBorrow(owner), name, attr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (is_meth) { + assert(!PyStackRef_IsNull(*attr)); + self_or_null[0] = owner; + } + else { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsNull(*attr)) { + JUMP_TO_LABEL(error); + } + self_or_null[0] = PyStackRef_NULL; + stack_pointer += 1; + } + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) { + JUMP_TO_LABEL(error); + } + *attr = PyStackRef_FromPyObjectSteal(attr_o); + stack_pointer += 1; + } + } + stack_pointer += (oparg&1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_CLASS) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_CLASS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_CLASS); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _CHECK_ATTR_CLASS + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (!PyType_Check(owner_o)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_CLASS + { + PyObject *descr = read_obj(&this_instr[6].cache); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = PyStackRef_FromPyObjectNew(descr); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = owner; + owner = attr; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[0]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_CLASS_WITH_METACLASS_CHECK; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _CHECK_ATTR_CLASS + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (!PyType_Check(owner_o)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + // _GUARD_TYPE_VERSION + { + uint32_t type_version = read_u32(&this_instr[4].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + // _LOAD_ATTR_CLASS + { + PyObject *descr = read_obj(&this_instr[6].cache); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = PyStackRef_FromPyObjectNew(descr); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = owner; + owner = attr; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[0]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + /* Skip 1 cache entry */ + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + uint32_t func_version = read_u32(&this_instr[4].cache); + PyObject *getattribute = read_obj(&this_instr[6].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert((oparg & 1) == 0); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + PyTypeObject *cls = Py_TYPE(owner_o); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); + PyFunctionObject *f = (PyFunctionObject *)getattribute; + assert(func_version != 0); + if (f->func_version != func_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + PyCodeObject *code = (PyCodeObject *)f->func_code; + assert(code->co_argcount == 2); + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + STAT_INC(LOAD_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( + tstate, PyStackRef_FromPyObjectNew(f), 2, frame); + new_frame->localsplus[0] = owner; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + new_frame->localsplus[1] = PyStackRef_FromPyObjectNew(name); + frame->return_offset = 10u ; + DISPATCH_INLINED(new_frame); + } + + TRACING_TARGET(LOAD_ATTR_INSTANCE_VALUE) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_INSTANCE_VALUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + // _CHECK_MANAGED_OBJECT_HAS_VALUES + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + // _LOAD_ATTR_INSTANCE_VALUE + { + uint16_t offset = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); + if (attr_o == NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr); + if (!increfed) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif + STAT_INC(LOAD_ATTR, hit); + stack_pointer[-1] = attr; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + /* Skip 5 cache entries */ + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[0]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_METHOD_LAZY_DICT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + // _CHECK_ATTR_METHOD_LAZY_DICT + { + uint16_t dictoffset = read_u16(&this_instr[4].cache); + char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; + PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); + if (dict != NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + /* Skip 1 cache entry */ + // _LOAD_ATTR_METHOD_LAZY_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_METHOD_NO_DICT) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_METHOD_NO_DICT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_METHOD_NO_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_METHOD_WITH_VALUES; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + PyDictValues *ivs = _PyObject_InlineValues(owner_o); + if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + // _GUARD_KEYS_VERSION + { + uint32_t keys_version = read_u32(&this_instr[4].cache); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + // _LOAD_ATTR_METHOD_WITH_VALUES + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_MODULE) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_MODULE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_MODULE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _LOAD_ATTR_MODULE + { + owner = stack_pointer[-1]; + uint32_t dict_version = read_u32(&this_instr[2].cache); + uint16_t index = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; + assert(dict != NULL); + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + assert(keys->dk_kind == DICT_KEYS_UNICODE); + assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(keys->dk_nentries)); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(keys) + index; + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); + if (attr_o == NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); + if (!increfed) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif + STAT_INC(LOAD_ATTR, hit); + stack_pointer[-1] = attr; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + /* Skip 5 cache entries */ + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[0]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_NONDESCRIPTOR_NO_DICT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + attr = PyStackRef_FromPyObjectNew(descr); + } + stack_pointer[0] = attr; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + PyDictValues *ivs = _PyObject_InlineValues(owner_o); + if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + // _GUARD_KEYS_VERSION + { + uint32_t keys_version = read_u32(&this_instr[4].cache); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + attr = PyStackRef_FromPyObjectNew(descr); + } + stack_pointer[0] = attr; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_PROPERTY) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_PROPERTY; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_PROPERTY_FRAME + { + PyObject *fget = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + assert(Py_IS_TYPE(fget, &PyFunction_Type)); + PyFunctionObject *f = (PyFunctionObject *)fget; + PyCodeObject *code = (PyCodeObject *)f->func_code; + if ((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + if (code->co_kwonlyargcount) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + if (code->co_argcount != 1) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + STAT_INC(LOAD_ATTR, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); + pushed_frame->localsplus[0] = owner; + new_frame = PyStackRef_Wrap(pushed_frame); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_SLOT) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_SLOT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_SLOT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + // _LOAD_ATTR_SLOT + { + uint16_t index = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject **addr = (PyObject **)((char *)owner_o + index); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); + if (attr_o == NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); + if (!increfed) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif + STAT_INC(LOAD_ATTR, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = owner; + owner = attr; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + /* Skip 5 cache entries */ + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[0]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_WITH_HINT) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_WITH_HINT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + // _LOAD_ATTR_WITH_HINT + { + uint16_t hint = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); + if (dict == NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + PyDictKeysObject *dk = FT_ATOMIC_LOAD_PTR(dict->ma_keys); + assert(PyDict_CheckExact((PyObject *)dict)); + #ifdef Py_GIL_DISABLED + if (!_Py_IsOwnedByCurrentThread((PyObject *)dict) && !_PyObject_GC_IS_SHARED(dict)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + #endif + PyObject *attr_o; + if (hint >= (size_t)FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_nentries)) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + if (dk->dk_kind != DICT_KEYS_UNICODE) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dk) + hint; + if (FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_key) != name) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + attr_o = FT_ATOMIC_LOAD_PTR(ep->me_value); + if (attr_o == NULL) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + STAT_INC(LOAD_ATTR, hit); + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); + if (!increfed) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(LOAD_ATTR); + } + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif + stack_pointer[-1] = attr; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + /* Skip 5 cache entries */ + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[0]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_BUILD_CLASS) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_BUILD_CLASS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_BUILD_CLASS); + _PyStackRef bc; + PyObject *bc_o; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + if (bc_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetString(tstate, PyExc_NameError, + "__build_class__ not found"); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + bc = PyStackRef_FromPyObjectSteal(bc_o); + stack_pointer[0] = bc; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_COMMON_CONSTANT) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_COMMON_CONSTANT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); + _PyStackRef value; + assert(oparg < NUM_COMMON_CONSTANTS); + value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_CONST) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_CONST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_CONST); + _PyStackRef value; + PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); + value = PyStackRef_FromPyObjectBorrow(obj); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_DEREF) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_DEREF; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_DEREF); + _PyStackRef value; + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + _PyFrame_SetStackPointer(frame, stack_pointer); + value = _PyCell_GetStackRef(cell); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsNull(value)) { + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_FAST) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FAST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST); + _PyStackRef value; + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_FAST_AND_CLEAR) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FAST_AND_CLEAR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); + _PyStackRef value; + value = GETLOCAL(oparg); + GETLOCAL(oparg) = PyStackRef_NULL; + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_FAST_BORROW) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FAST_BORROW; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_BORROW); + _PyStackRef value; + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_FAST_BORROW_LOAD_FAST_BORROW) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FAST_BORROW_LOAD_FAST_BORROW; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_BORROW_LOAD_FAST_BORROW); + _PyStackRef value1; + _PyStackRef value2; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + value1 = PyStackRef_Borrow(GETLOCAL(oparg1)); + value2 = PyStackRef_Borrow(GETLOCAL(oparg2)); + stack_pointer[0] = value1; + stack_pointer[1] = value2; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_FAST_CHECK) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FAST_CHECK; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_CHECK); + _PyStackRef value; + _PyStackRef value_s = GETLOCAL(oparg); + if (PyStackRef_IsNull(value_s)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + value = PyStackRef_DUP(value_s); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_FAST_LOAD_FAST) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FAST_LOAD_FAST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); + _PyStackRef value1; + _PyStackRef value2; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + value1 = PyStackRef_DUP(GETLOCAL(oparg1)); + value2 = PyStackRef_DUP(GETLOCAL(oparg2)); + stack_pointer[0] = value1; + stack_pointer[1] = value2; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_FROM_DICT_OR_DEREF) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FROM_DICT_OR_DEREF; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); + _PyStackRef class_dict_st; + _PyStackRef value; + class_dict_st = stack_pointer[-1]; + PyObject *value_o; + PyObject *name; + PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); + assert(class_dict); + assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); + name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(class_dict, name, &value_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + if (!value_o) { + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + value_o = PyCell_GetRef(cell); + if (value_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(class_dict_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + value = PyStackRef_FromPyObjectSteal(value_o); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_FROM_DICT_OR_GLOBALS) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FROM_DICT_OR_GLOBALS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); + _PyStackRef mod_or_class_dict; + _PyStackRef v; + mod_or_class_dict = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *v_o; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(mod_or_class_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + if (v_o == NULL) { + if (PyDict_CheckExact(GLOBALS()) + && PyDict_CheckExact(BUILTINS())) + { + _PyFrame_SetStackPointer(frame, stack_pointer); + v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + (PyDictObject *)BUILTINS(), + name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (v_o == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + JUMP_TO_LABEL(error); + } + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(GLOBALS(), name, &v_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + if (v_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(BUILTINS(), name, &v_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + if (v_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + } + } + } + v = PyStackRef_FromPyObjectSteal(v_o); + stack_pointer[0] = v; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_GLOBAL) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_GLOBAL; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL); + PREDICTED_TRACING_LOAD_GLOBAL:; + _Py_CODEUNIT* const this_instr = next_instr - 5; + (void)this_instr; + _PyStackRef *res; + _PyStackRef *null; + // _SPECIALIZE_LOAD_GLOBAL + { + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(LOAD_GLOBAL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 1 cache entry */ + /* Skip 1 cache entry */ + /* Skip 1 cache entry */ + // _LOAD_GLOBAL + { + res = &stack_pointer[0]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsNull(*res)) { + JUMP_TO_LABEL(error); + } + } + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[1]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_GLOBAL_BUILTIN) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_GLOBAL_BUILTIN; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); + static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); + _PyStackRef res; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _GUARD_GLOBALS_VERSION + { + uint16_t version = read_u16(&this_instr[2].cache); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + if (!PyDict_CheckExact(dict)) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } + assert(DK_IS_UNICODE(keys)); + } + // _LOAD_GLOBAL_BUILTINS + { + uint16_t version = read_u16(&this_instr[3].cache); + uint16_t index = read_u16(&this_instr[4].cache); + PyDictObject *dict = (PyDictObject *)BUILTINS(); + if (!PyDict_CheckExact(dict)) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } + assert(DK_IS_UNICODE(keys)); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); + PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); + if (res_o == NULL) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } + #if Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); + if (!increfed) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } + #else + res = PyStackRef_FromPyObjectNew(res_o); + #endif + STAT_INC(LOAD_GLOBAL, hit); + } + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[1]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer[0] = res; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_GLOBAL_MODULE) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_GLOBAL_MODULE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); + static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); + _PyStackRef res; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _NOP + { + } + // _LOAD_GLOBAL_MODULE + { + uint16_t version = read_u16(&this_instr[2].cache); + uint16_t index = read_u16(&this_instr[4].cache); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + if (!PyDict_CheckExact(dict)) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } + assert(DK_IS_UNICODE(keys)); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); + assert(index < DK_SIZE(keys)); + PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); + if (res_o == NULL) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } + #if Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); + if (!increfed) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(LOAD_GLOBAL); + } + #else + res = PyStackRef_FromPyObjectNew(res_o); + #endif + STAT_INC(LOAD_GLOBAL, hit); + } + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[1]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer[0] = res; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_LOCALS) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_LOCALS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_LOCALS); + _PyStackRef locals; + PyObject *l = LOCALS(); + if (l == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetString(tstate, PyExc_SystemError, + "no locals found"); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + locals = PyStackRef_FromPyObjectNew(l); + stack_pointer[0] = locals; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_NAME) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_NAME; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_NAME); + _PyStackRef v; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *v_o = _PyEval_LoadName(tstate, frame, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (v_o == NULL) { + JUMP_TO_LABEL(error); + } + v = PyStackRef_FromPyObjectSteal(v_o); + stack_pointer[0] = v; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_SMALL_INT) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_SMALL_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_SMALL_INT); + _PyStackRef value; + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_SPECIAL) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_SPECIAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_SPECIAL); + _PyStackRef self; + _PyStackRef *method_and_self; + // _INSERT_NULL + { + self = stack_pointer[-1]; + method_and_self = &stack_pointer[-1]; + method_and_self[1] = self; + method_and_self[0] = PyStackRef_NULL; + } + // _LOAD_SPECIAL + { + method_and_self = &stack_pointer[-1]; + PyObject *name = _Py_SpecialMethods[oparg].name; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyObject_LookupSpecialMethod(name, method_and_self); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err <= 0) { + if (err == 0) { + PyObject *owner = PyStackRef_AsPyObjectBorrow(method_and_self[1]); + _PyFrame_SetStackPointer(frame, stack_pointer); + const char *errfmt = _PyEval_SpecialMethodCanSuggest(owner, oparg) + ? _Py_SpecialMethods[oparg].error_suggestion + : _Py_SpecialMethods[oparg].error; + stack_pointer = _PyFrame_GetStackPointer(frame); + assert(!_PyErr_Occurred(tstate)); + assert(errfmt != NULL); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, errfmt, owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_SUPER_ATTR) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_SUPER_ATTR; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR); + PREDICTED_TRACING_LOAD_SUPER_ATTR:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + opcode = LOAD_SUPER_ATTR; + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr; + _PyStackRef *null; + // _SPECIALIZE_LOAD_SUPER_ATTR + { + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + int load_method = oparg & 1; + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _LOAD_SUPER_ATTR + { + self_st = stack_pointer[-1]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + JUMP_TO_LABEL(error); + } + } + PyObject *stack[] = {class, self}; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + if (super == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(super); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + if (super == NULL) { + JUMP_TO_LABEL(error); + } + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = PyObject_GetAttr(super, name); + Py_DECREF(super); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) { + JUMP_TO_LABEL(error); + } + attr = PyStackRef_FromPyObjectSteal(attr_o); + } + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[1]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer[0] = attr; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_SUPER_ATTR_ATTR) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_SUPER_ATTR_ATTR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); + static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr_st; + /* Skip 1 cache entry */ + self_st = stack_pointer[-1]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + assert(!(oparg & 1)); + if (global_super != (PyObject *)&PySuper_Type) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); + } + if (!PyType_Check(class)) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); + } + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + if (attr == NULL) { + JUMP_TO_LABEL(error); + } + attr_st = PyStackRef_FromPyObjectSteal(attr); + stack_pointer[0] = attr_st; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_SUPER_ATTR_METHOD) { + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_SUPER_ATTR_METHOD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); + static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr; + _PyStackRef self_or_null; + /* Skip 1 cache entry */ + self_st = stack_pointer[-1]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + assert(oparg & 1); + if (global_super != (PyObject *)&PySuper_Type) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); + } + if (!PyType_Check(class)) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); + } + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + PyTypeObject *cls = (PyTypeObject *)class; + int method_found = 0; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = _PySuper_Lookup(cls, self, name, + Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) { + JUMP_TO_LABEL(error); + } + if (method_found) { + self_or_null = self_st; + } else { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(self_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + self_or_null = PyStackRef_NULL; + stack_pointer += 1; + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = global_super_st; + global_super_st = self_or_null; + stack_pointer[-2] = global_super_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-1] = class_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + attr = PyStackRef_FromPyObjectSteal(attr_o); + stack_pointer[0] = attr; + stack_pointer[1] = self_or_null; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(MAKE_CELL) { + #if _Py_TAIL_CALL_INTERP + int opcode = MAKE_CELL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAKE_CELL); + PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyObject *cell = PyCell_New(initial); + if (cell == NULL) { + JUMP_TO_LABEL(error); + } + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = PyStackRef_FromPyObjectSteal(cell); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(MAKE_FUNCTION) { + #if _Py_TAIL_CALL_INTERP + int opcode = MAKE_FUNCTION; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAKE_FUNCTION); + _PyStackRef codeobj_st; + _PyStackRef func; + codeobj_st = stack_pointer[-1]; + PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyFunctionObject *func_obj = (PyFunctionObject *) + PyFunction_New(codeobj, GLOBALS()); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(codeobj_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (func_obj == NULL) { + JUMP_TO_LABEL(error); + } + _PyFunction_SetVersion( + func_obj, ((PyCodeObject *)codeobj)->co_version); + func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); + stack_pointer[0] = func; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(MAP_ADD) { + #if _Py_TAIL_CALL_INTERP + int opcode = MAP_ADD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAP_ADD); + _PyStackRef dict_st; + _PyStackRef key; + _PyStackRef value; + value = stack_pointer[-1]; + key = stack_pointer[-2]; + dict_st = stack_pointer[-3 - (oparg - 1)]; + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + assert(PyDict_CheckExact(dict)); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyDict_SetItem_Take2( + (PyDictObject *)dict, + PyStackRef_AsPyObjectSteal(key), + PyStackRef_AsPyObjectSteal(value) + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(pop_2_error); + } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(MATCH_CLASS) { + #if _Py_TAIL_CALL_INTERP + int opcode = MATCH_CLASS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_CLASS); + _PyStackRef subject; + _PyStackRef type; + _PyStackRef names; + _PyStackRef attrs; + names = stack_pointer[-1]; + type = stack_pointer[-2]; + subject = stack_pointer[-3]; + assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names))); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attrs_o = _PyEval_MatchClass(tstate, + PyStackRef_AsPyObjectBorrow(subject), + PyStackRef_AsPyObjectBorrow(type), oparg, + PyStackRef_AsPyObjectBorrow(names)); + _PyStackRef tmp = names; + names = PyStackRef_NULL; + stack_pointer[-1] = names; + PyStackRef_CLOSE(tmp); + tmp = type; + type = PyStackRef_NULL; + stack_pointer[-2] = type; + PyStackRef_CLOSE(tmp); + tmp = subject; + subject = PyStackRef_NULL; + stack_pointer[-3] = subject; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + if (attrs_o) { + assert(PyTuple_CheckExact(attrs_o)); + attrs = PyStackRef_FromPyObjectSteal(attrs_o); + } + else { + if (_PyErr_Occurred(tstate)) { + JUMP_TO_LABEL(error); + } + attrs = PyStackRef_None; + } + stack_pointer[0] = attrs; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(MATCH_KEYS) { + #if _Py_TAIL_CALL_INTERP + int opcode = MATCH_KEYS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_KEYS); + _PyStackRef subject; + _PyStackRef keys; + _PyStackRef values_or_none; + keys = stack_pointer[-1]; + subject = stack_pointer[-2]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, + PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (values_or_none_o == NULL) { + JUMP_TO_LABEL(error); + } + values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); + stack_pointer[0] = values_or_none; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(MATCH_MAPPING) { + #if _Py_TAIL_CALL_INTERP + int opcode = MATCH_MAPPING; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_MAPPING); + _PyStackRef subject; + _PyStackRef res; + subject = stack_pointer[-1]; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; + res = match ? PyStackRef_True : PyStackRef_False; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(MATCH_SEQUENCE) { + #if _Py_TAIL_CALL_INTERP + int opcode = MATCH_SEQUENCE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_SEQUENCE); + _PyStackRef subject; + _PyStackRef res; + subject = stack_pointer[-1]; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; + res = match ? PyStackRef_True : PyStackRef_False; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(NOP) { + #if _Py_TAIL_CALL_INTERP + int opcode = NOP; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(NOP); + TRACING_DISPATCH(); + } + + TRACING_TARGET(NOT_TAKEN) { + #if _Py_TAIL_CALL_INTERP + int opcode = NOT_TAKEN; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(NOT_TAKEN); + TRACING_DISPATCH(); + } + + TRACING_TARGET(POP_EXCEPT) { + #if _Py_TAIL_CALL_INTERP + int opcode = POP_EXCEPT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_EXCEPT); + _PyStackRef exc_value; + exc_value = stack_pointer[-1]; + _PyErr_StackItem *exc_info = tstate->exc_info; + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_XSETREF(exc_info->exc_value, + PyStackRef_IsNone(exc_value) + ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(POP_ITER) { + #if _Py_TAIL_CALL_INTERP + int opcode = POP_ITER; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_ITER); + _PyStackRef iter; + _PyStackRef index_or_null; + index_or_null = stack_pointer[-1]; + iter = stack_pointer[-2]; + (void)index_or_null; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iter); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(POP_JUMP_IF_FALSE) { + #if _Py_TAIL_CALL_INTERP + int opcode = POP_JUMP_IF_FALSE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_FALSE); + _PyStackRef cond; + /* Skip 1 cache entry */ + cond = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsFalse(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(POP_JUMP_IF_NONE) { + #if _Py_TAIL_CALL_INTERP + int opcode = POP_JUMP_IF_NONE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_NONE); + _PyStackRef value; + _PyStackRef b; + _PyStackRef cond; + /* Skip 1 cache entry */ + // _IS_NONE + { + value = stack_pointer[-1]; + if (PyStackRef_IsNone(value)) { + b = PyStackRef_True; + } + else { + b = PyStackRef_False; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = value; + value = b; + stack_pointer[-1] = value; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + // _POP_JUMP_IF_TRUE + { + cond = b; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsTrue(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(POP_JUMP_IF_NOT_NONE) { + #if _Py_TAIL_CALL_INTERP + int opcode = POP_JUMP_IF_NOT_NONE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); + _PyStackRef value; + _PyStackRef b; + _PyStackRef cond; + /* Skip 1 cache entry */ + // _IS_NONE + { + value = stack_pointer[-1]; + if (PyStackRef_IsNone(value)) { + b = PyStackRef_True; + } + else { + b = PyStackRef_False; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = value; + value = b; + stack_pointer[-1] = value; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + // _POP_JUMP_IF_FALSE + { + cond = b; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsFalse(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(POP_JUMP_IF_TRUE) { + #if _Py_TAIL_CALL_INTERP + int opcode = POP_JUMP_IF_TRUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_TRUE); + _PyStackRef cond; + /* Skip 1 cache entry */ + cond = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsTrue(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(POP_TOP) { + #if _Py_TAIL_CALL_INTERP + int opcode = POP_TOP; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_TOP); + _PyStackRef value; + value = stack_pointer[-1]; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(PUSH_EXC_INFO) { + #if _Py_TAIL_CALL_INTERP + int opcode = PUSH_EXC_INFO; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(PUSH_EXC_INFO); + _PyStackRef exc; + _PyStackRef prev_exc; + _PyStackRef new_exc; + exc = stack_pointer[-1]; + _PyErr_StackItem *exc_info = tstate->exc_info; + if (exc_info->exc_value != NULL) { + prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value); + } + else { + prev_exc = PyStackRef_None; + } + assert(PyStackRef_ExceptionInstanceCheck(exc)); + exc_info->exc_value = PyStackRef_AsPyObjectNew(exc); + new_exc = exc; + stack_pointer[-1] = prev_exc; + stack_pointer[0] = new_exc; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(PUSH_NULL) { + #if _Py_TAIL_CALL_INTERP + int opcode = PUSH_NULL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(PUSH_NULL); + _PyStackRef res; + res = PyStackRef_NULL; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(RAISE_VARARGS) { + #if _Py_TAIL_CALL_INTERP + int opcode = RAISE_VARARGS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RAISE_VARARGS); + _PyStackRef *args; + args = &stack_pointer[-oparg]; + assert(oparg < 3); + PyObject *cause = oparg == 2 ? PyStackRef_AsPyObjectSteal(args[1]) : NULL; + PyObject *exc = oparg > 0 ? PyStackRef_AsPyObjectSteal(args[0]) : NULL; + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = do_raise(tstate, exc, cause); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + assert(oparg == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_reraise(tstate, frame, this_instr); + JUMP_TO_LABEL(exception_unwind); + } + JUMP_TO_LABEL(error); + } + + TRACING_TARGET(RERAISE) { + #if _Py_TAIL_CALL_INTERP + int opcode = RERAISE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RERAISE); + _PyStackRef *values; + _PyStackRef exc_st; + exc_st = stack_pointer[-1]; + values = &stack_pointer[-1 - oparg]; + PyObject *exc = PyStackRef_AsPyObjectSteal(exc_st); + assert(oparg >= 0 && oparg <= 2); + if (oparg) { + frame->instr_ptr = _PyFrame_GetBytecode(frame) + PyStackRef_UntagInt(values[0]); + } + assert(exc && PyExceptionInstance_Check(exc)); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetRaisedException(tstate, exc); + monitor_reraise(tstate, frame, this_instr); + JUMP_TO_LABEL(exception_unwind); + } + + TRACING_TARGET(RESERVED) { + #if _Py_TAIL_CALL_INTERP + int opcode = RESERVED; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESERVED); + assert(0 && "Executing RESERVED instruction."); + Py_FatalError("Executing RESERVED instruction."); + TRACING_DISPATCH(); + } + + TRACING_TARGET(RESUME) { + #if _Py_TAIL_CALL_INTERP + int opcode = RESUME; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESUME); + PREDICTED_TRACING_RESUME:; + _Py_CODEUNIT* const this_instr = next_instr - 1; + (void)this_instr; + // _LOAD_BYTECODE + { + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_CODEUNIT *bytecode = + _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (bytecode == NULL) { + JUMP_TO_LABEL(error); + } + ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); + frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; + frame->instr_ptr = bytecode + off; + next_instr = frame->instr_ptr; + TRACING_DISPATCH(); + } + #endif + } + // _MAYBE_INSTRUMENT + { + #ifdef Py_GIL_DISABLED + + int check_instrumentation = 1; + #else + int check_instrumentation = (tstate->tracing == 0); + #endif + if (check_instrumentation) { + uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + if (code_version != global_version) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } + next_instr = this_instr; + TRACING_DISPATCH(); + } + } + } + // _QUICKEN_RESUME + { + #if ENABLE_SPECIALIZATION_FT + if (tstate->tracing == 0 && this_instr->op.code == RESUME) { + FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); + } + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _CHECK_PERIODIC_IF_NOT_YIELD_FROM + { + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + JUMP_TO_LABEL(error); + } + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(RESUME_CHECK) { + #if _Py_TAIL_CALL_INTERP + int opcode = RESUME_CHECK; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESUME_CHECK); + static_assert(0 == 0, "incorrect cache size"); + #if defined(__EMSCRIPTEN__) + if (_Py_emscripten_signal_clock == 0) { + UPDATE_MISS_STATS(RESUME); + assert(_PyOpcode_Deopt[opcode] == (RESUME)); + JUMP_TO_PREDICTED(RESUME); + } + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + assert((version & _PY_EVAL_EVENTS_MASK) == 0); + if (eval_breaker != version) { + UPDATE_MISS_STATS(RESUME); + assert(_PyOpcode_Deopt[opcode] == (RESUME)); + JUMP_TO_PREDICTED(RESUME); + } + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + UPDATE_MISS_STATS(RESUME); + assert(_PyOpcode_Deopt[opcode] == (RESUME)); + JUMP_TO_PREDICTED(RESUME); + } + #endif + TRACING_DISPATCH(); + } + + TRACING_TARGET(RETURN_GENERATOR) { + #if _Py_TAIL_CALL_INTERP + int opcode = RETURN_GENERATOR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_GENERATOR); + _PyStackRef res; + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (gen == NULL) { + JUMP_TO_LABEL(error); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + frame->instr_ptr++; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + LOAD_IP(frame->return_offset); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(RETURN_VALUE) { + #if _Py_TAIL_CALL_INTERP + int opcode = RETURN_VALUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_VALUE); + _PyStackRef retval; + _PyStackRef res; + retval = stack_pointer[-1]; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + _PyStackRef temp = PyStackRef_MakeHeapSafe(retval); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(STACK_LEVEL() == 0); + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_IP(frame->return_offset); + res = temp; + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(SEND) { + #if _Py_TAIL_CALL_INTERP + int opcode = SEND; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND); + PREDICTED_TRACING_SEND:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef receiver; + _PyStackRef v; + _PyStackRef retval; + // _SPECIALIZE_SEND + { + receiver = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_Send(receiver, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(SEND); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _SEND + { + v = stack_pointer[-1]; + PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); + PyObject *retval_o; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + if ((tstate->interp->eval_frame == NULL) && + (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && + ((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING) + { + PyGenObject *gen = (PyGenObject *)receiver_o; + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + _PyFrame_StackPush(gen_frame, PyStackRef_MakeHeapSafe(v)); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert( 2u + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)( 2u + oparg); + assert(gen_frame->previous == NULL); + gen_frame->previous = frame; + DISPATCH_INLINED(gen_frame); + } + if (PyStackRef_IsNone(v) && PyIter_Check(receiver_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + retval_o = PyObject_CallMethodOneArg(receiver_o, + &_Py_ID(send), + PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + if (retval_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_MonitorRaise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyGen_FetchStopIterationValue(&retval_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err == 0) { + assert(retval_o != NULL); + JUMPBY(oparg); + } + else { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); + retval = PyStackRef_FromPyObjectSteal(retval_o); + } + stack_pointer[0] = retval; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(SEND_GEN) { + #if _Py_TAIL_CALL_INTERP + int opcode = SEND_GEN; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND_GEN); + static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); + _PyStackRef receiver; + _PyStackRef v; + _PyStackRef gen_frame; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + JUMP_TO_PREDICTED(SEND); + } + } + // _SEND_GEN_FRAME + { + v = stack_pointer[-1]; + receiver = stack_pointer[-2]; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); + if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + JUMP_TO_PREDICTED(SEND); + } + if (gen->gi_frame_state >= FRAME_EXECUTING) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + JUMP_TO_PREDICTED(SEND); + } + STAT_INC(SEND, hit); + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert( 2u + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)( 2u + oparg); + pushed_frame->previous = frame; + gen_frame = PyStackRef_Wrap(pushed_frame); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(SETUP_ANNOTATIONS) { + #if _Py_TAIL_CALL_INTERP + int opcode = SETUP_ANNOTATIONS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SETUP_ANNOTATIONS); + PyObject *ann_dict; + if (LOCALS() == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_SystemError, + "no locals found when setting up annotations"); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + if (ann_dict == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + ann_dict = PyDict_New(); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (ann_dict == NULL) { + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), + ann_dict); + Py_DECREF(ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(SET_ADD) { + #if _Py_TAIL_CALL_INTERP + int opcode = SET_ADD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_ADD); + _PyStackRef set; + _PyStackRef v; + v = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PySet_AddTakeRef((PySetObject *)PyStackRef_AsPyObjectBorrow(set), + PyStackRef_AsPyObjectSteal(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(pop_1_error); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(SET_FUNCTION_ATTRIBUTE) { + #if _Py_TAIL_CALL_INTERP + int opcode = SET_FUNCTION_ATTRIBUTE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); + _PyStackRef attr_st; + _PyStackRef func_in; + _PyStackRef func_out; + func_in = stack_pointer[-1]; + attr_st = stack_pointer[-2]; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_in); + PyObject *attr = PyStackRef_AsPyObjectSteal(attr_st); + func_out = func_in; + assert(PyFunction_Check(func)); + size_t offset = _Py_FunctionAttributeOffsets[oparg]; + assert(offset != 0); + PyObject **ptr = (PyObject **)(((char *)func) + offset); + assert(*ptr == NULL); + *ptr = attr; + stack_pointer[-2] = func_out; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(SET_UPDATE) { + #if _Py_TAIL_CALL_INTERP + int opcode = SET_UPDATE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_UPDATE); + _PyStackRef set; + _PyStackRef iterable; + iterable = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), + PyStackRef_AsPyObjectBorrow(iterable)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_ATTR) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_ATTR; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR); + PREDICTED_TRACING_STORE_ATTR:; + _Py_CODEUNIT* const this_instr = next_instr - 5; + (void)this_instr; + _PyStackRef owner; + _PyStackRef v; + // _SPECIALIZE_STORE_ATTR + { + owner = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_StoreAttr(owner, next_instr, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(STORE_ATTR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 3 cache entries */ + // _STORE_ATTR + { + v = stack_pointer[-2]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), + name, PyStackRef_AsPyObjectBorrow(v)); + _PyStackRef tmp = owner; + owner = PyStackRef_NULL; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + tmp = v; + v = PyStackRef_NULL; + stack_pointer[-2] = v; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (err) { + JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_ATTR_INSTANCE_VALUE) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_ATTR_INSTANCE_VALUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION_AND_LOCK + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(type_version != 0); + if (!LOCK_OBJECT(owner_o)) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } + PyTypeObject *tp = Py_TYPE(owner_o); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UNLOCK_OBJECT(owner_o); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } + } + } + // _GUARD_DORV_NO_DICT + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (_PyObject_GetManagedDict(owner_o) || + !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UNLOCK_OBJECT(owner_o); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } + } + } + // _STORE_ATTR_INSTANCE_VALUE + { + value = stack_pointer[-2]; + uint16_t offset = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + STAT_INC(STORE_ATTR, hit); + assert(_PyObject_GetManagedDict(owner_o) == NULL); + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *old_value = *value_ptr; + FT_ATOMIC_STORE_PTR_RELEASE(*value_ptr, PyStackRef_AsPyObjectSteal(value)); + if (old_value == NULL) { + PyDictValues *values = _PyObject_InlineValues(owner_o); + Py_ssize_t index = value_ptr - values->values; + _PyDictValues_AddToInsertionOrder(values, index); + } + UNLOCK_OBJECT(owner_o); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + Py_XDECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_ATTR_SLOT) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_ATTR_SLOT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_SLOT); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } + } + // _STORE_ATTR_SLOT + { + value = stack_pointer[-2]; + uint16_t index = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (!LOCK_OBJECT(owner_o)) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } + char *addr = (char *)owner_o + index; + STAT_INC(STORE_ATTR, hit); + PyObject *old_value = *(PyObject **)addr; + FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(owner_o); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + Py_XDECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_ATTR_WITH_HINT) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_ATTR_WITH_HINT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } + } + // _STORE_ATTR_WITH_HINT + { + value = stack_pointer[-2]; + uint16_t hint = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); + if (dict == NULL) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } + if (!LOCK_OBJECT(dict)) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } + assert(PyDict_CheckExact((PyObject *)dict)); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (hint >= (size_t)dict->ma_keys->dk_nentries || + !DK_IS_UNICODE(dict->ma_keys)) { + UNLOCK_OBJECT(dict); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } + } + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; + if (ep->me_key != name) { + UNLOCK_OBJECT(dict); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } + } + PyObject *old_value = ep->me_value; + if (old_value == NULL) { + UNLOCK_OBJECT(dict); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(STORE_ATTR); + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(dict); + STAT_INC(STORE_ATTR, hit); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + Py_XDECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_DEREF) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_DEREF; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_DEREF); + _PyStackRef v; + v = stack_pointer[-1]; + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_FAST) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_FAST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST); + _PyStackRef value; + value = stack_pointer[-1]; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_FAST_LOAD_FAST) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_FAST_LOAD_FAST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); + _PyStackRef value1; + _PyStackRef value2; + value1 = stack_pointer[-1]; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + _PyStackRef tmp = GETLOCAL(oparg1); + GETLOCAL(oparg1) = value1; + value2 = PyStackRef_DUP(GETLOCAL(oparg2)); + stack_pointer[-1] = value2; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_FAST_STORE_FAST) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_FAST_STORE_FAST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST_STORE_FAST); + _PyStackRef value2; + _PyStackRef value1; + value1 = stack_pointer[-1]; + value2 = stack_pointer[-2]; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + _PyStackRef tmp = GETLOCAL(oparg1); + GETLOCAL(oparg1) = value1; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + tmp = GETLOCAL(oparg2); + GETLOCAL(oparg2) = value2; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_GLOBAL) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_GLOBAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_GLOBAL); + _PyStackRef v; + v = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_NAME) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_NAME; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_NAME); + _PyStackRef v; + v = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *ns = LOCALS(); + int err; + if (ns == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_SystemError, + "no locals found when storing %R", name); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + if (PyDict_CheckExact(ns)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_SLICE) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_SLICE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_SLICE); + _PyStackRef v; + _PyStackRef container; + _PyStackRef start; + _PyStackRef stop; + // _SPECIALIZE_STORE_SLICE + { + #if ENABLE_SPECIALIZATION + OPCODE_DEFERRED_INC(STORE_SLICE); + #endif /* ENABLE_SPECIALIZATION */ + } + // _STORE_SLICE + { + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + v = stack_pointer[-4]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), + PyStackRef_AsPyObjectSteal(stop)); + stack_pointer = _PyFrame_GetStackPointer(frame); + int err; + if (slice == NULL) { + err = 1; + } + else { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); + Py_DECREF(slice); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += 2; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = container; + container = PyStackRef_NULL; + stack_pointer[-3] = container; + PyStackRef_CLOSE(tmp); + tmp = v; + v = PyStackRef_NULL; + stack_pointer[-4] = v; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -4; + assert(WITHIN_STACK_BOUNDS()); + if (err) { + JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_SUBSCR) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_SUBSCR; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR); + PREDICTED_TRACING_STORE_SUBSCR:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef container; + _PyStackRef sub; + _PyStackRef v; + // _SPECIALIZE_STORE_SUBSCR + { + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_StoreSubscr(container, sub, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(STORE_SUBSCR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _STORE_SUBSCR + { + v = stack_pointer[-3]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); + _PyStackRef tmp = sub; + sub = PyStackRef_NULL; + stack_pointer[-1] = sub; + PyStackRef_CLOSE(tmp); + tmp = container; + container = PyStackRef_NULL; + stack_pointer[-2] = container; + PyStackRef_CLOSE(tmp); + tmp = v; + v = PyStackRef_NULL; + stack_pointer[-3] = v; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + if (err) { + JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_SUBSCR_DICT) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_SUBSCR_DICT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_DICT); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); + _PyStackRef nos; + _PyStackRef value; + _PyStackRef dict_st; + _PyStackRef sub; + // _GUARD_NOS_DICT + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyDict_CheckExact(o)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + } + /* Skip 1 cache entry */ + // _STORE_SUBSCR_DICT + { + sub = stack_pointer[-1]; + dict_st = nos; + value = stack_pointer[-3]; + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + assert(PyDict_CheckExact(dict)); + STAT_INC(STORE_SUBSCR, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, + PyStackRef_AsPyObjectSteal(sub), + PyStackRef_AsPyObjectSteal(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(dict_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_SUBSCR_LIST_INT) { + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_SUBSCR_LIST_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef list_st; + _PyStackRef sub_st; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + } + // _GUARD_NOS_LIST + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyList_CheckExact(o)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + } + /* Skip 1 cache entry */ + // _STORE_SUBSCR_LIST_INT + { + sub_st = value; + list_st = nos; + value = stack_pointer[-3]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + assert(PyLong_CheckExact(sub)); + assert(PyList_CheckExact(list)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (!LOCK_OBJECT(list)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + if (index >= PyList_GET_SIZE(list)) { + UNLOCK_OBJECT(list); + if (true) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(STORE_SUBSCR); + } + } + STAT_INC(STORE_SUBSCR, hit); + PyObject *old_value = PyList_GET_ITEM(list, index); + FT_ATOMIC_STORE_PTR_RELEASE(_PyList_ITEMS(list)[index], + PyStackRef_AsPyObjectSteal(value)); + assert(old_value != NULL); + UNLOCK_OBJECT(list); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(list_st); + Py_DECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(SWAP) { + #if _Py_TAIL_CALL_INTERP + int opcode = SWAP; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SWAP); + _PyStackRef bottom; + _PyStackRef top; + top = stack_pointer[-1]; + bottom = stack_pointer[-2 - (oparg-2)]; + _PyStackRef temp = bottom; + bottom = top; + top = temp; + stack_pointer[-2 - (oparg-2)] = bottom; + stack_pointer[-1] = top; + TRACING_DISPATCH(); + } + + TRACING_TARGET(TO_BOOL) { + #if _Py_TAIL_CALL_INTERP + int opcode = TO_BOOL; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL); + PREDICTED_TRACING_TO_BOOL:; + _Py_CODEUNIT* const this_instr = next_instr - 4; + (void)this_instr; + _PyStackRef value; + _PyStackRef res; + // _SPECIALIZE_TO_BOOL + { + value = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_ToBool(value, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(TO_BOOL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 2 cache entries */ + // _TO_BOOL + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + res = err ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(TO_BOOL_ALWAYS_TRUE) { + #if _Py_TAIL_CALL_INTERP + int opcode = TO_BOOL_ALWAYS_TRUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TO_BOOL); + } + } + // _REPLACE_WITH_TRUE + { + value = owner; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_True; + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(TO_BOOL_BOOL) { + #if _Py_TAIL_CALL_INTERP + int opcode = TO_BOOL_BOOL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_BOOL); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + if (!PyStackRef_BoolCheck(value)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TO_BOOL); + } + STAT_INC(TO_BOOL, hit); + TRACING_DISPATCH(); + } + + TRACING_TARGET(TO_BOOL_INT) { + #if _Py_TAIL_CALL_INTERP + int opcode = TO_BOOL_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_INT); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyLong_CheckExact(value_o)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TO_BOOL); + } + STAT_INC(TO_BOOL, hit); + if (_PyLong_IsZero((PyLongObject *)value_o)) { + assert(_Py_IsImmortal(value_o)); + res = PyStackRef_False; + } + else { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_True; + stack_pointer += 1; + } + stack_pointer[-1] = res; + TRACING_DISPATCH(); + } + + TRACING_TARGET(TO_BOOL_LIST) { + #if _Py_TAIL_CALL_INTERP + int opcode = TO_BOOL_LIST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_LIST); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef value; + _PyStackRef res; + // _GUARD_TOS_LIST + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyList_CheckExact(o)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TO_BOOL); + } + } + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _TO_BOOL_LIST + { + value = tos; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + assert(PyList_CheckExact(value_o)); + STAT_INC(TO_BOOL, hit); + res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = value; + value = res; + stack_pointer[-1] = value; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(TO_BOOL_NONE) { + #if _Py_TAIL_CALL_INTERP + int opcode = TO_BOOL_NONE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_NONE); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + if (!PyStackRef_IsNone(value)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TO_BOOL); + } + STAT_INC(TO_BOOL, hit); + res = PyStackRef_False; + stack_pointer[-1] = res; + TRACING_DISPATCH(); + } + + TRACING_TARGET(TO_BOOL_STR) { + #if _Py_TAIL_CALL_INTERP + int opcode = TO_BOOL_STR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_STR); + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + // _GUARD_TOS_UNICODE + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TO_BOOL); + } + } + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _TO_BOOL_STR + { + STAT_INC(TO_BOOL, hit); + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (value_o == &_Py_STR(empty)) { + assert(_Py_IsImmortal(value_o)); + res = PyStackRef_False; + } + else { + assert(Py_SIZE(value_o)); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_True; + stack_pointer += 1; + } + } + stack_pointer[-1] = res; + TRACING_DISPATCH(); + } + + TRACING_TARGET(UNARY_INVERT) { + #if _Py_TAIL_CALL_INTERP + int opcode = UNARY_INVERT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_INVERT); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(UNARY_NEGATIVE) { + #if _Py_TAIL_CALL_INTERP + int opcode = UNARY_NEGATIVE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_NEGATIVE); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(UNARY_NOT) { + #if _Py_TAIL_CALL_INTERP + int opcode = UNARY_NOT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_NOT); + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(value)); + res = PyStackRef_IsFalse(value) + ? PyStackRef_True : PyStackRef_False; + stack_pointer[-1] = res; + TRACING_DISPATCH(); + } + + TRACING_TARGET(UNPACK_EX) { + #if _Py_TAIL_CALL_INTERP + int opcode = UNPACK_EX; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNPACK_EX); + _PyStackRef seq; + _PyStackRef *top; + seq = stack_pointer[-1]; + top = &stack_pointer[(oparg & 0xFF) + (oparg >> 8)]; + PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg & 0xFF, oparg >> 8, top); + Py_DECREF(seq_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res == 0) { + JUMP_TO_LABEL(error); + } + stack_pointer += 1 + (oparg & 0xFF) + (oparg >> 8); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(UNPACK_SEQUENCE) { + #if _Py_TAIL_CALL_INTERP + int opcode = UNPACK_SEQUENCE; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE); + PREDICTED_TRACING_UNPACK_SEQUENCE:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + _PyStackRef seq; + _PyStackRef *top; + // _SPECIALIZE_UNPACK_SEQUENCE + { + seq = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_UnpackSequence(seq, next_instr, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(UNPACK_SEQUENCE); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + (void)seq; + (void)counter; + } + // _UNPACK_SEQUENCE + { + top = &stack_pointer[-1 + oparg]; + PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg, -1, top); + Py_DECREF(seq_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res == 0) { + JUMP_TO_LABEL(error); + } + } + stack_pointer += oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(UNPACK_SEQUENCE_LIST) { + #if _Py_TAIL_CALL_INTERP + int opcode = UNPACK_SEQUENCE_LIST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef seq; + _PyStackRef *values; + // _GUARD_TOS_LIST + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyList_CheckExact(o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } + } + /* Skip 1 cache entry */ + // _UNPACK_SEQUENCE_LIST + { + seq = tos; + values = &stack_pointer[-1]; + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + assert(PyList_CheckExact(seq_o)); + if (!LOCK_OBJECT(seq_o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } + if (PyList_GET_SIZE(seq_o) != oparg) { + UNLOCK_OBJECT(seq_o); + if (true) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } + } + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyList_ITEMS(seq_o); + for (int i = oparg; --i >= 0; ) { + *values++ = PyStackRef_FromPyObjectNew(items[i]); + } + UNLOCK_OBJECT(seq_o); + stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(seq); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(UNPACK_SEQUENCE_TUPLE) { + #if _Py_TAIL_CALL_INTERP + int opcode = UNPACK_SEQUENCE_TUPLE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef seq; + _PyStackRef *values; + // _GUARD_TOS_TUPLE + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyTuple_CheckExact(o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } + } + /* Skip 1 cache entry */ + // _UNPACK_SEQUENCE_TUPLE + { + seq = tos; + values = &stack_pointer[-1]; + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + assert(PyTuple_CheckExact(seq_o)); + if (PyTuple_GET_SIZE(seq_o) != oparg) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyTuple_ITEMS(seq_o); + for (int i = oparg; --i >= 0; ) { + *values++ = PyStackRef_FromPyObjectNew(items[i]); + } + stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(seq); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { + #if _Py_TAIL_CALL_INTERP + int opcode = UNPACK_SEQUENCE_TWO_TUPLE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef seq; + _PyStackRef val1; + _PyStackRef val0; + // _GUARD_TOS_TUPLE + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyTuple_CheckExact(o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } + } + /* Skip 1 cache entry */ + // _UNPACK_SEQUENCE_TWO_TUPLE + { + seq = tos; + assert(oparg == 2); + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + assert(PyTuple_CheckExact(seq_o)); + if (PyTuple_GET_SIZE(seq_o) != 2) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + } + STAT_INC(UNPACK_SEQUENCE, hit); + val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); + val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); + stack_pointer[-1] = val1; + stack_pointer[0] = val0; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(seq); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(WITH_EXCEPT_START) { + #if _Py_TAIL_CALL_INTERP + int opcode = WITH_EXCEPT_START; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(WITH_EXCEPT_START); + _PyStackRef exit_func; + _PyStackRef exit_self; + _PyStackRef lasti; + _PyStackRef val; + _PyStackRef res; + val = stack_pointer[-1]; + lasti = stack_pointer[-3]; + exit_self = stack_pointer[-4]; + exit_func = stack_pointer[-5]; + PyObject *exc, *tb; + PyObject *val_o = PyStackRef_AsPyObjectBorrow(val); + PyObject *exit_func_o = PyStackRef_AsPyObjectBorrow(exit_func); + assert(val_o && PyExceptionInstance_Check(val_o)); + exc = PyExceptionInstance_Class(val_o); + PyObject *original_tb = tb = PyException_GetTraceback(val_o); + if (tb == NULL) { + tb = Py_None; + } + assert(PyStackRef_IsTaggedInt(lasti)); + (void)lasti; + PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; + int has_self = !PyStackRef_IsNull(exit_self); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, + (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + Py_XDECREF(original_tb); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(YIELD_VALUE) { + #if _Py_TAIL_CALL_INTERP + int opcode = YIELD_VALUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(YIELD_VALUE); + _PyStackRef retval; + _PyStackRef value; + retval = stack_pointer[-1]; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + frame->instr_ptr++; + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + _PyStackRef temp = retval; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + #if TIER_ONE + assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || + frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #endif + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + value = PyStackRef_MakeHeapSafe(temp); + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + #endif /* END TRACING INSTRUCTIONS */ /* END INSTRUCTIONS */ #if !_Py_TAIL_CALL_INTERP diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 6dd443e1655ed0..299739764919bb 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -257,8 +257,268 @@ static void *opcode_targets_table[256] = { &&TARGET_INSTRUMENTED_LINE, &&TARGET_ENTER_EXECUTOR, }; +static void *opcode_tracing_targets_table[256] = { + &&TARGET_TRACING_CACHE, + &&TARGET_TRACING_BINARY_SLICE, + &&TARGET_TRACING_BUILD_TEMPLATE, + &&TARGET_TRACING_BINARY_OP_INPLACE_ADD_UNICODE, + &&TARGET_TRACING_CALL_FUNCTION_EX, + &&TARGET_TRACING_CHECK_EG_MATCH, + &&TARGET_TRACING_CHECK_EXC_MATCH, + &&TARGET_TRACING_CLEANUP_THROW, + &&TARGET_TRACING_DELETE_SUBSCR, + &&TARGET_TRACING_END_FOR, + &&TARGET_TRACING_END_SEND, + &&TARGET_TRACING_EXIT_INIT_CHECK, + &&TARGET_TRACING_FORMAT_SIMPLE, + &&TARGET_TRACING_FORMAT_WITH_SPEC, + &&TARGET_TRACING_GET_AITER, + &&TARGET_TRACING_GET_ANEXT, + &&TARGET_TRACING_GET_ITER, + &&TARGET_TRACING_RESERVED, + &&TARGET_TRACING_GET_LEN, + &&TARGET_TRACING_GET_YIELD_FROM_ITER, + &&TARGET_TRACING_INTERPRETER_EXIT, + &&TARGET_TRACING_LOAD_BUILD_CLASS, + &&TARGET_TRACING_LOAD_LOCALS, + &&TARGET_TRACING_MAKE_FUNCTION, + &&TARGET_TRACING_MATCH_KEYS, + &&TARGET_TRACING_MATCH_MAPPING, + &&TARGET_TRACING_MATCH_SEQUENCE, + &&TARGET_TRACING_NOP, + &&TARGET_TRACING_NOT_TAKEN, + &&TARGET_TRACING_POP_EXCEPT, + &&TARGET_TRACING_POP_ITER, + &&TARGET_TRACING_POP_TOP, + &&TARGET_TRACING_PUSH_EXC_INFO, + &&TARGET_TRACING_PUSH_NULL, + &&TARGET_TRACING_RETURN_GENERATOR, + &&TARGET_TRACING_RETURN_VALUE, + &&TARGET_TRACING_SETUP_ANNOTATIONS, + &&TARGET_TRACING_STORE_SLICE, + &&TARGET_TRACING_STORE_SUBSCR, + &&TARGET_TRACING_TO_BOOL, + &&TARGET_TRACING_UNARY_INVERT, + &&TARGET_TRACING_UNARY_NEGATIVE, + &&TARGET_TRACING_UNARY_NOT, + &&TARGET_TRACING_WITH_EXCEPT_START, + &&TARGET_TRACING_BINARY_OP, + &&TARGET_TRACING_BUILD_INTERPOLATION, + &&TARGET_TRACING_BUILD_LIST, + &&TARGET_TRACING_BUILD_MAP, + &&TARGET_TRACING_BUILD_SET, + &&TARGET_TRACING_BUILD_SLICE, + &&TARGET_TRACING_BUILD_STRING, + &&TARGET_TRACING_BUILD_TUPLE, + &&TARGET_TRACING_CALL, + &&TARGET_TRACING_CALL_INTRINSIC_1, + &&TARGET_TRACING_CALL_INTRINSIC_2, + &&TARGET_TRACING_CALL_KW, + &&TARGET_TRACING_COMPARE_OP, + &&TARGET_TRACING_CONTAINS_OP, + &&TARGET_TRACING_CONVERT_VALUE, + &&TARGET_TRACING_COPY, + &&TARGET_TRACING_COPY_FREE_VARS, + &&TARGET_TRACING_DELETE_ATTR, + &&TARGET_TRACING_DELETE_DEREF, + &&TARGET_TRACING_DELETE_FAST, + &&TARGET_TRACING_DELETE_GLOBAL, + &&TARGET_TRACING_DELETE_NAME, + &&TARGET_TRACING_DICT_MERGE, + &&TARGET_TRACING_DICT_UPDATE, + &&TARGET_TRACING_END_ASYNC_FOR, + &&TARGET_TRACING_EXTENDED_ARG, + &&TARGET_TRACING_FOR_ITER, + &&TARGET_TRACING_GET_AWAITABLE, + &&TARGET_TRACING_IMPORT_FROM, + &&TARGET_TRACING_IMPORT_NAME, + &&TARGET_TRACING_IS_OP, + &&TARGET_TRACING_JUMP_BACKWARD, + &&TARGET_TRACING_JUMP_BACKWARD_NO_INTERRUPT, + &&TARGET_TRACING_JUMP_FORWARD, + &&TARGET_TRACING_LIST_APPEND, + &&TARGET_TRACING_LIST_EXTEND, + &&TARGET_TRACING_LOAD_ATTR, + &&TARGET_TRACING_LOAD_COMMON_CONSTANT, + &&TARGET_TRACING_LOAD_CONST, + &&TARGET_TRACING_LOAD_DEREF, + &&TARGET_TRACING_LOAD_FAST, + &&TARGET_TRACING_LOAD_FAST_AND_CLEAR, + &&TARGET_TRACING_LOAD_FAST_BORROW, + &&TARGET_TRACING_LOAD_FAST_BORROW_LOAD_FAST_BORROW, + &&TARGET_TRACING_LOAD_FAST_CHECK, + &&TARGET_TRACING_LOAD_FAST_LOAD_FAST, + &&TARGET_TRACING_LOAD_FROM_DICT_OR_DEREF, + &&TARGET_TRACING_LOAD_FROM_DICT_OR_GLOBALS, + &&TARGET_TRACING_LOAD_GLOBAL, + &&TARGET_TRACING_LOAD_NAME, + &&TARGET_TRACING_LOAD_SMALL_INT, + &&TARGET_TRACING_LOAD_SPECIAL, + &&TARGET_TRACING_LOAD_SUPER_ATTR, + &&TARGET_TRACING_MAKE_CELL, + &&TARGET_TRACING_MAP_ADD, + &&TARGET_TRACING_MATCH_CLASS, + &&TARGET_TRACING_POP_JUMP_IF_FALSE, + &&TARGET_TRACING_POP_JUMP_IF_NONE, + &&TARGET_TRACING_POP_JUMP_IF_NOT_NONE, + &&TARGET_TRACING_POP_JUMP_IF_TRUE, + &&TARGET_TRACING_RAISE_VARARGS, + &&TARGET_TRACING_RERAISE, + &&TARGET_TRACING_SEND, + &&TARGET_TRACING_SET_ADD, + &&TARGET_TRACING_SET_FUNCTION_ATTRIBUTE, + &&TARGET_TRACING_SET_UPDATE, + &&TARGET_TRACING_STORE_ATTR, + &&TARGET_TRACING_STORE_DEREF, + &&TARGET_TRACING_STORE_FAST, + &&TARGET_TRACING_STORE_FAST_LOAD_FAST, + &&TARGET_TRACING_STORE_FAST_STORE_FAST, + &&TARGET_TRACING_STORE_GLOBAL, + &&TARGET_TRACING_STORE_NAME, + &&TARGET_TRACING_SWAP, + &&TARGET_TRACING_UNPACK_EX, + &&TARGET_TRACING_UNPACK_SEQUENCE, + &&TARGET_TRACING_YIELD_VALUE, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&TARGET_TRACING_RESUME, + &&TARGET_TRACING_BINARY_OP_ADD_FLOAT, + &&TARGET_TRACING_BINARY_OP_ADD_INT, + &&TARGET_TRACING_BINARY_OP_ADD_UNICODE, + &&TARGET_TRACING_BINARY_OP_EXTEND, + &&TARGET_TRACING_BINARY_OP_MULTIPLY_FLOAT, + &&TARGET_TRACING_BINARY_OP_MULTIPLY_INT, + &&TARGET_TRACING_BINARY_OP_SUBSCR_DICT, + &&TARGET_TRACING_BINARY_OP_SUBSCR_GETITEM, + &&TARGET_TRACING_BINARY_OP_SUBSCR_LIST_INT, + &&TARGET_TRACING_BINARY_OP_SUBSCR_LIST_SLICE, + &&TARGET_TRACING_BINARY_OP_SUBSCR_STR_INT, + &&TARGET_TRACING_BINARY_OP_SUBSCR_TUPLE_INT, + &&TARGET_TRACING_BINARY_OP_SUBTRACT_FLOAT, + &&TARGET_TRACING_BINARY_OP_SUBTRACT_INT, + &&TARGET_TRACING_CALL_ALLOC_AND_ENTER_INIT, + &&TARGET_TRACING_CALL_BOUND_METHOD_EXACT_ARGS, + &&TARGET_TRACING_CALL_BOUND_METHOD_GENERAL, + &&TARGET_TRACING_CALL_BUILTIN_CLASS, + &&TARGET_TRACING_CALL_BUILTIN_FAST, + &&TARGET_TRACING_CALL_BUILTIN_FAST_WITH_KEYWORDS, + &&TARGET_TRACING_CALL_BUILTIN_O, + &&TARGET_TRACING_CALL_ISINSTANCE, + &&TARGET_TRACING_CALL_KW_BOUND_METHOD, + &&TARGET_TRACING_CALL_KW_NON_PY, + &&TARGET_TRACING_CALL_KW_PY, + &&TARGET_TRACING_CALL_LEN, + &&TARGET_TRACING_CALL_LIST_APPEND, + &&TARGET_TRACING_CALL_METHOD_DESCRIPTOR_FAST, + &&TARGET_TRACING_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, + &&TARGET_TRACING_CALL_METHOD_DESCRIPTOR_NOARGS, + &&TARGET_TRACING_CALL_METHOD_DESCRIPTOR_O, + &&TARGET_TRACING_CALL_NON_PY_GENERAL, + &&TARGET_TRACING_CALL_PY_EXACT_ARGS, + &&TARGET_TRACING_CALL_PY_GENERAL, + &&TARGET_TRACING_CALL_STR_1, + &&TARGET_TRACING_CALL_TUPLE_1, + &&TARGET_TRACING_CALL_TYPE_1, + &&TARGET_TRACING_COMPARE_OP_FLOAT, + &&TARGET_TRACING_COMPARE_OP_INT, + &&TARGET_TRACING_COMPARE_OP_STR, + &&TARGET_TRACING_CONTAINS_OP_DICT, + &&TARGET_TRACING_CONTAINS_OP_SET, + &&TARGET_TRACING_FOR_ITER_GEN, + &&TARGET_TRACING_FOR_ITER_LIST, + &&TARGET_TRACING_FOR_ITER_RANGE, + &&TARGET_TRACING_FOR_ITER_TUPLE, + &&TARGET_TRACING_JUMP_BACKWARD_JIT, + &&TARGET_TRACING_JUMP_BACKWARD_NO_JIT, + &&TARGET_TRACING_LOAD_ATTR_CLASS, + &&TARGET_TRACING_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK, + &&TARGET_TRACING_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, + &&TARGET_TRACING_LOAD_ATTR_INSTANCE_VALUE, + &&TARGET_TRACING_LOAD_ATTR_METHOD_LAZY_DICT, + &&TARGET_TRACING_LOAD_ATTR_METHOD_NO_DICT, + &&TARGET_TRACING_LOAD_ATTR_METHOD_WITH_VALUES, + &&TARGET_TRACING_LOAD_ATTR_MODULE, + &&TARGET_TRACING_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, + &&TARGET_TRACING_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, + &&TARGET_TRACING_LOAD_ATTR_PROPERTY, + &&TARGET_TRACING_LOAD_ATTR_SLOT, + &&TARGET_TRACING_LOAD_ATTR_WITH_HINT, + &&TARGET_TRACING_LOAD_GLOBAL_BUILTIN, + &&TARGET_TRACING_LOAD_GLOBAL_MODULE, + &&TARGET_TRACING_LOAD_SUPER_ATTR_ATTR, + &&TARGET_TRACING_LOAD_SUPER_ATTR_METHOD, + &&TARGET_TRACING_RESUME_CHECK, + &&TARGET_TRACING_SEND_GEN, + &&TARGET_TRACING_STORE_ATTR_INSTANCE_VALUE, + &&TARGET_TRACING_STORE_ATTR_SLOT, + &&TARGET_TRACING_STORE_ATTR_WITH_HINT, + &&TARGET_TRACING_STORE_SUBSCR_DICT, + &&TARGET_TRACING_STORE_SUBSCR_LIST_INT, + &&TARGET_TRACING_TO_BOOL_ALWAYS_TRUE, + &&TARGET_TRACING_TO_BOOL_BOOL, + &&TARGET_TRACING_TO_BOOL_INT, + &&TARGET_TRACING_TO_BOOL_LIST, + &&TARGET_TRACING_TO_BOOL_NONE, + &&TARGET_TRACING_TO_BOOL_STR, + &&TARGET_TRACING_UNPACK_SEQUENCE_LIST, + &&TARGET_TRACING_UNPACK_SEQUENCE_TUPLE, + &&TARGET_TRACING_UNPACK_SEQUENCE_TWO_TUPLE, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, + &&TARGET_TRACING_INSTRUMENTED_END_FOR, + &&TARGET_TRACING_INSTRUMENTED_POP_ITER, + &&TARGET_TRACING_INSTRUMENTED_END_SEND, + &&TARGET_TRACING_INSTRUMENTED_FOR_ITER, + &&TARGET_TRACING_INSTRUMENTED_INSTRUCTION, + &&TARGET_TRACING_INSTRUMENTED_JUMP_FORWARD, + &&TARGET_TRACING_INSTRUMENTED_NOT_TAKEN, + &&TARGET_TRACING_INSTRUMENTED_POP_JUMP_IF_TRUE, + &&TARGET_TRACING_INSTRUMENTED_POP_JUMP_IF_FALSE, + &&TARGET_TRACING_INSTRUMENTED_POP_JUMP_IF_NONE, + &&TARGET_TRACING_INSTRUMENTED_POP_JUMP_IF_NOT_NONE, + &&TARGET_TRACING_INSTRUMENTED_RESUME, + &&TARGET_TRACING_INSTRUMENTED_RETURN_VALUE, + &&TARGET_TRACING_INSTRUMENTED_YIELD_VALUE, + &&TARGET_TRACING_INSTRUMENTED_END_ASYNC_FOR, + &&TARGET_TRACING_INSTRUMENTED_LOAD_SUPER_ATTR, + &&TARGET_TRACING_INSTRUMENTED_CALL, + &&TARGET_TRACING_INSTRUMENTED_CALL_KW, + &&TARGET_TRACING_INSTRUMENTED_CALL_FUNCTION_EX, + &&TARGET_TRACING_INSTRUMENTED_JUMP_BACKWARD, + &&TARGET_TRACING_INSTRUMENTED_LINE, + &&TARGET_TRACING_ENTER_EXECUTOR, +}; #else /* _Py_TAIL_CALL_INTERP */ -static py_tail_call_funcptr instruction_funcptr_table[256]; +static py_tail_call_funcptr instruction_funcptr_handler_table[256]; + +static py_tail_call_funcptr instruction_funcptr_tracing_table[256]; Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_2_error(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_pop_1_error(TAIL_CALL_PARAMS); @@ -268,230 +528,455 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_start_frame(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_ADD_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_ADD_UNICODE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_EXTEND(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_EXTEND(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_INPLACE_ADD_UNICODE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_MULTIPLY_FLOAT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_MULTIPLY_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBSCR_DICT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_SUBSCR_DICT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBSCR_GETITEM(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_SUBSCR_GETITEM(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBSCR_LIST_SLICE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_SUBSCR_LIST_SLICE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBSCR_STR_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_SUBSCR_STR_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBSCR_TUPLE_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_SUBSCR_TUPLE_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_SUBTRACT_FLOAT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_SUBTRACT_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SLICE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_SLICE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_INTERPOLATION(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BUILD_INTERPOLATION(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_LIST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BUILD_LIST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_MAP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BUILD_MAP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SET(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BUILD_SET(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SLICE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BUILD_SLICE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_STRING(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BUILD_STRING(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_TEMPLATE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BUILD_TEMPLATE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_TUPLE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BUILD_TUPLE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CACHE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CACHE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_ALLOC_AND_ENTER_INIT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_BUILTIN_CLASS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_BUILTIN_FAST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_BUILTIN_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_BUILTIN_O(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_1(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_INTRINSIC_1(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_INTRINSIC_2(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_ISINSTANCE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_KW(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_KW_NON_PY(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_KW_PY(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_LEN(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_LIST_APPEND(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_METHOD_DESCRIPTOR_FAST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_METHOD_DESCRIPTOR_NOARGS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_METHOD_DESCRIPTOR_O(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_NON_PY_GENERAL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_PY_GENERAL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_STR_1(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_TUPLE_1(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_TYPE_1(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EG_MATCH(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CHECK_EG_MATCH(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EXC_MATCH(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CHECK_EXC_MATCH(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CLEANUP_THROW(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CLEANUP_THROW(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_COMPARE_OP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_COMPARE_OP_FLOAT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_COMPARE_OP_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_COMPARE_OP_STR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CONTAINS_OP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CONTAINS_OP_DICT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CONTAINS_OP_SET(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONVERT_VALUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CONVERT_VALUE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_COPY(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY_FREE_VARS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_COPY_FREE_VARS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_ATTR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_DELETE_ATTR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_DEREF(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_DELETE_DEREF(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_DELETE_FAST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_GLOBAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_DELETE_GLOBAL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_NAME(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_DELETE_NAME(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_SUBSCR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_DELETE_SUBSCR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_MERGE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_DICT_MERGE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_UPDATE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_DICT_UPDATE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_ASYNC_FOR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_END_ASYNC_FOR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_FOR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_END_FOR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_SEND(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_END_SEND(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_ENTER_EXECUTOR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXIT_INIT_CHECK(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_EXIT_INIT_CHECK(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXTENDED_ARG(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_EXTENDED_ARG(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_SIMPLE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_FORMAT_SIMPLE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_WITH_SPEC(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_FORMAT_WITH_SPEC(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_FOR_ITER(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_FOR_ITER_GEN(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_FOR_ITER_LIST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_FOR_ITER_RANGE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_FOR_ITER_TUPLE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AITER(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_GET_AITER(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ANEXT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_GET_ANEXT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AWAITABLE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_GET_AWAITABLE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ITER(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_GET_ITER(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_GET_LEN(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_YIELD_FROM_ITER(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_GET_YIELD_FROM_ITER(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_IMPORT_FROM(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_IMPORT_NAME(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_CALL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_CALL_KW(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_ASYNC_FOR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_END_ASYNC_FOR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_FOR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_END_FOR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_SEND(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_END_SEND(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_FOR_ITER(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_FOR_ITER(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_INSTRUCTION(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_INSTRUCTION(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_JUMP_BACKWARD(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_JUMP_FORWARD(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LINE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_LINE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_NOT_TAKEN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_NOT_TAKEN(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_ITER(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_POP_ITER(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RESUME(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_RESUME(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RETURN_VALUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_RETURN_VALUE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_YIELD_VALUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_YIELD_VALUE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INTERPRETER_EXIT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INTERPRETER_EXIT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IS_OP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_IS_OP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_JUMP_BACKWARD(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_JIT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_JUMP_BACKWARD_JIT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_JUMP_BACKWARD_NO_INTERRUPT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_JIT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_JUMP_BACKWARD_NO_JIT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_FORWARD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_JUMP_FORWARD(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LIST_APPEND(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_EXTEND(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LIST_EXTEND(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_METHOD_NO_DICT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_METHOD_WITH_VALUES(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_MODULE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_PROPERTY(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_WITH_HINT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_BUILD_CLASS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_BUILD_CLASS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_COMMON_CONSTANT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_COMMON_CONSTANT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_CONST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_DEREF(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_DEREF(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_FAST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_AND_CLEAR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_FAST_AND_CLEAR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_BORROW(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_FAST_BORROW(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_BORROW_LOAD_FAST_BORROW(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_FAST_BORROW_LOAD_FAST_BORROW(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_CHECK(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_FAST_CHECK(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_LOAD_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_FAST_LOAD_FAST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_FROM_DICT_OR_DEREF(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_FROM_DICT_OR_GLOBALS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_GLOBAL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_GLOBAL_MODULE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_LOCALS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_LOCALS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_NAME(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_NAME(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SMALL_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_SMALL_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SPECIAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_SPECIAL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_SUPER_ATTR_METHOD(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_CELL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_MAKE_CELL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_FUNCTION(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_MAKE_FUNCTION(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAP_ADD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_MAP_ADD(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_CLASS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_MATCH_CLASS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_KEYS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_MATCH_KEYS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_MAPPING(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_MATCH_MAPPING(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_SEQUENCE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_MATCH_SEQUENCE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_NOP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOT_TAKEN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_NOT_TAKEN(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_EXCEPT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_POP_EXCEPT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_ITER(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_POP_ITER(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_TOP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_POP_TOP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_EXC_INFO(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_PUSH_EXC_INFO(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_NULL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_PUSH_NULL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RAISE_VARARGS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_RAISE_VARARGS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RERAISE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_RERAISE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESERVED(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_RESERVED(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_RESUME(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_RESUME_CHECK(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_RETURN_GENERATOR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_RETURN_VALUE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_SEND(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_SEND_GEN(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_SETUP_ANNOTATIONS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_SET_ADD(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_FUNCTION_ATTRIBUTE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_SET_FUNCTION_ATTRIBUTE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_UPDATE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_SET_UPDATE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_ATTR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_ATTR_SLOT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_ATTR_WITH_HINT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_DEREF(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_DEREF(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_FAST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_LOAD_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_FAST_LOAD_FAST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_STORE_FAST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_FAST_STORE_FAST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_GLOBAL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_GLOBAL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_NAME(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_NAME(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_SLICE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_SUBSCR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_SUBSCR_DICT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SWAP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_SWAP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_TO_BOOL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_TO_BOOL_BOOL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_TO_BOOL_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_TO_BOOL_LIST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_TO_BOOL_NONE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_TO_BOOL_STR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_UNARY_INVERT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_UNARY_NEGATIVE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NOT(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_UNARY_NOT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_EX(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_UNPACK_EX(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_UNPACK_SEQUENCE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_UNPACK_SEQUENCE_LIST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_WITH_EXCEPT_START(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_WITH_EXCEPT_START(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_YIELD_VALUE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS) { int opcode = next_instr->op.code; @@ -503,7 +988,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS) JUMP_TO_LABEL(error); } -static py_tail_call_funcptr instruction_funcptr_table[256] = { +static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [BINARY_OP] = _TAIL_CALL_BINARY_OP, [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_BINARY_OP_ADD_FLOAT, [BINARY_OP_ADD_INT] = _TAIL_CALL_BINARY_OP_ADD_INT, @@ -761,4 +1246,262 @@ static py_tail_call_funcptr instruction_funcptr_table[256] = { [232] = _TAIL_CALL_UNKNOWN_OPCODE, [233] = _TAIL_CALL_UNKNOWN_OPCODE, }; +static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { + [BINARY_OP] = _TAIL_CALL_TRACING_BINARY_OP, + [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_TRACING_BINARY_OP_ADD_FLOAT, + [BINARY_OP_ADD_INT] = _TAIL_CALL_TRACING_BINARY_OP_ADD_INT, + [BINARY_OP_ADD_UNICODE] = _TAIL_CALL_TRACING_BINARY_OP_ADD_UNICODE, + [BINARY_OP_EXTEND] = _TAIL_CALL_TRACING_BINARY_OP_EXTEND, + [BINARY_OP_INPLACE_ADD_UNICODE] = _TAIL_CALL_TRACING_BINARY_OP_INPLACE_ADD_UNICODE, + [BINARY_OP_MULTIPLY_FLOAT] = _TAIL_CALL_TRACING_BINARY_OP_MULTIPLY_FLOAT, + [BINARY_OP_MULTIPLY_INT] = _TAIL_CALL_TRACING_BINARY_OP_MULTIPLY_INT, + [BINARY_OP_SUBSCR_DICT] = _TAIL_CALL_TRACING_BINARY_OP_SUBSCR_DICT, + [BINARY_OP_SUBSCR_GETITEM] = _TAIL_CALL_TRACING_BINARY_OP_SUBSCR_GETITEM, + [BINARY_OP_SUBSCR_LIST_INT] = _TAIL_CALL_TRACING_BINARY_OP_SUBSCR_LIST_INT, + [BINARY_OP_SUBSCR_LIST_SLICE] = _TAIL_CALL_TRACING_BINARY_OP_SUBSCR_LIST_SLICE, + [BINARY_OP_SUBSCR_STR_INT] = _TAIL_CALL_TRACING_BINARY_OP_SUBSCR_STR_INT, + [BINARY_OP_SUBSCR_TUPLE_INT] = _TAIL_CALL_TRACING_BINARY_OP_SUBSCR_TUPLE_INT, + [BINARY_OP_SUBTRACT_FLOAT] = _TAIL_CALL_TRACING_BINARY_OP_SUBTRACT_FLOAT, + [BINARY_OP_SUBTRACT_INT] = _TAIL_CALL_TRACING_BINARY_OP_SUBTRACT_INT, + [BINARY_SLICE] = _TAIL_CALL_TRACING_BINARY_SLICE, + [BUILD_INTERPOLATION] = _TAIL_CALL_TRACING_BUILD_INTERPOLATION, + [BUILD_LIST] = _TAIL_CALL_TRACING_BUILD_LIST, + [BUILD_MAP] = _TAIL_CALL_TRACING_BUILD_MAP, + [BUILD_SET] = _TAIL_CALL_TRACING_BUILD_SET, + [BUILD_SLICE] = _TAIL_CALL_TRACING_BUILD_SLICE, + [BUILD_STRING] = _TAIL_CALL_TRACING_BUILD_STRING, + [BUILD_TEMPLATE] = _TAIL_CALL_TRACING_BUILD_TEMPLATE, + [BUILD_TUPLE] = _TAIL_CALL_TRACING_BUILD_TUPLE, + [CACHE] = _TAIL_CALL_TRACING_CACHE, + [CALL] = _TAIL_CALL_TRACING_CALL, + [CALL_ALLOC_AND_ENTER_INIT] = _TAIL_CALL_TRACING_CALL_ALLOC_AND_ENTER_INIT, + [CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_TRACING_CALL_BOUND_METHOD_EXACT_ARGS, + [CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_TRACING_CALL_BOUND_METHOD_GENERAL, + [CALL_BUILTIN_CLASS] = _TAIL_CALL_TRACING_CALL_BUILTIN_CLASS, + [CALL_BUILTIN_FAST] = _TAIL_CALL_TRACING_CALL_BUILTIN_FAST, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = _TAIL_CALL_TRACING_CALL_BUILTIN_FAST_WITH_KEYWORDS, + [CALL_BUILTIN_O] = _TAIL_CALL_TRACING_CALL_BUILTIN_O, + [CALL_FUNCTION_EX] = _TAIL_CALL_TRACING_CALL_FUNCTION_EX, + [CALL_INTRINSIC_1] = _TAIL_CALL_TRACING_CALL_INTRINSIC_1, + [CALL_INTRINSIC_2] = _TAIL_CALL_TRACING_CALL_INTRINSIC_2, + [CALL_ISINSTANCE] = _TAIL_CALL_TRACING_CALL_ISINSTANCE, + [CALL_KW] = _TAIL_CALL_TRACING_CALL_KW, + [CALL_KW_BOUND_METHOD] = _TAIL_CALL_TRACING_CALL_KW_BOUND_METHOD, + [CALL_KW_NON_PY] = _TAIL_CALL_TRACING_CALL_KW_NON_PY, + [CALL_KW_PY] = _TAIL_CALL_TRACING_CALL_KW_PY, + [CALL_LEN] = _TAIL_CALL_TRACING_CALL_LEN, + [CALL_LIST_APPEND] = _TAIL_CALL_TRACING_CALL_LIST_APPEND, + [CALL_METHOD_DESCRIPTOR_FAST] = _TAIL_CALL_TRACING_CALL_METHOD_DESCRIPTOR_FAST, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = _TAIL_CALL_TRACING_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, + [CALL_METHOD_DESCRIPTOR_NOARGS] = _TAIL_CALL_TRACING_CALL_METHOD_DESCRIPTOR_NOARGS, + [CALL_METHOD_DESCRIPTOR_O] = _TAIL_CALL_TRACING_CALL_METHOD_DESCRIPTOR_O, + [CALL_NON_PY_GENERAL] = _TAIL_CALL_TRACING_CALL_NON_PY_GENERAL, + [CALL_PY_EXACT_ARGS] = _TAIL_CALL_TRACING_CALL_PY_EXACT_ARGS, + [CALL_PY_GENERAL] = _TAIL_CALL_TRACING_CALL_PY_GENERAL, + [CALL_STR_1] = _TAIL_CALL_TRACING_CALL_STR_1, + [CALL_TUPLE_1] = _TAIL_CALL_TRACING_CALL_TUPLE_1, + [CALL_TYPE_1] = _TAIL_CALL_TRACING_CALL_TYPE_1, + [CHECK_EG_MATCH] = _TAIL_CALL_TRACING_CHECK_EG_MATCH, + [CHECK_EXC_MATCH] = _TAIL_CALL_TRACING_CHECK_EXC_MATCH, + [CLEANUP_THROW] = _TAIL_CALL_TRACING_CLEANUP_THROW, + [COMPARE_OP] = _TAIL_CALL_TRACING_COMPARE_OP, + [COMPARE_OP_FLOAT] = _TAIL_CALL_TRACING_COMPARE_OP_FLOAT, + [COMPARE_OP_INT] = _TAIL_CALL_TRACING_COMPARE_OP_INT, + [COMPARE_OP_STR] = _TAIL_CALL_TRACING_COMPARE_OP_STR, + [CONTAINS_OP] = _TAIL_CALL_TRACING_CONTAINS_OP, + [CONTAINS_OP_DICT] = _TAIL_CALL_TRACING_CONTAINS_OP_DICT, + [CONTAINS_OP_SET] = _TAIL_CALL_TRACING_CONTAINS_OP_SET, + [CONVERT_VALUE] = _TAIL_CALL_TRACING_CONVERT_VALUE, + [COPY] = _TAIL_CALL_TRACING_COPY, + [COPY_FREE_VARS] = _TAIL_CALL_TRACING_COPY_FREE_VARS, + [DELETE_ATTR] = _TAIL_CALL_TRACING_DELETE_ATTR, + [DELETE_DEREF] = _TAIL_CALL_TRACING_DELETE_DEREF, + [DELETE_FAST] = _TAIL_CALL_TRACING_DELETE_FAST, + [DELETE_GLOBAL] = _TAIL_CALL_TRACING_DELETE_GLOBAL, + [DELETE_NAME] = _TAIL_CALL_TRACING_DELETE_NAME, + [DELETE_SUBSCR] = _TAIL_CALL_TRACING_DELETE_SUBSCR, + [DICT_MERGE] = _TAIL_CALL_TRACING_DICT_MERGE, + [DICT_UPDATE] = _TAIL_CALL_TRACING_DICT_UPDATE, + [END_ASYNC_FOR] = _TAIL_CALL_TRACING_END_ASYNC_FOR, + [END_FOR] = _TAIL_CALL_TRACING_END_FOR, + [END_SEND] = _TAIL_CALL_TRACING_END_SEND, + [ENTER_EXECUTOR] = _TAIL_CALL_TRACING_ENTER_EXECUTOR, + [EXIT_INIT_CHECK] = _TAIL_CALL_TRACING_EXIT_INIT_CHECK, + [EXTENDED_ARG] = _TAIL_CALL_TRACING_EXTENDED_ARG, + [FORMAT_SIMPLE] = _TAIL_CALL_TRACING_FORMAT_SIMPLE, + [FORMAT_WITH_SPEC] = _TAIL_CALL_TRACING_FORMAT_WITH_SPEC, + [FOR_ITER] = _TAIL_CALL_TRACING_FOR_ITER, + [FOR_ITER_GEN] = _TAIL_CALL_TRACING_FOR_ITER_GEN, + [FOR_ITER_LIST] = _TAIL_CALL_TRACING_FOR_ITER_LIST, + [FOR_ITER_RANGE] = _TAIL_CALL_TRACING_FOR_ITER_RANGE, + [FOR_ITER_TUPLE] = _TAIL_CALL_TRACING_FOR_ITER_TUPLE, + [GET_AITER] = _TAIL_CALL_TRACING_GET_AITER, + [GET_ANEXT] = _TAIL_CALL_TRACING_GET_ANEXT, + [GET_AWAITABLE] = _TAIL_CALL_TRACING_GET_AWAITABLE, + [GET_ITER] = _TAIL_CALL_TRACING_GET_ITER, + [GET_LEN] = _TAIL_CALL_TRACING_GET_LEN, + [GET_YIELD_FROM_ITER] = _TAIL_CALL_TRACING_GET_YIELD_FROM_ITER, + [IMPORT_FROM] = _TAIL_CALL_TRACING_IMPORT_FROM, + [IMPORT_NAME] = _TAIL_CALL_TRACING_IMPORT_NAME, + [INSTRUMENTED_CALL] = _TAIL_CALL_TRACING_INSTRUMENTED_CALL, + [INSTRUMENTED_CALL_FUNCTION_EX] = _TAIL_CALL_TRACING_INSTRUMENTED_CALL_FUNCTION_EX, + [INSTRUMENTED_CALL_KW] = _TAIL_CALL_TRACING_INSTRUMENTED_CALL_KW, + [INSTRUMENTED_END_ASYNC_FOR] = _TAIL_CALL_TRACING_INSTRUMENTED_END_ASYNC_FOR, + [INSTRUMENTED_END_FOR] = _TAIL_CALL_TRACING_INSTRUMENTED_END_FOR, + [INSTRUMENTED_END_SEND] = _TAIL_CALL_TRACING_INSTRUMENTED_END_SEND, + [INSTRUMENTED_FOR_ITER] = _TAIL_CALL_TRACING_INSTRUMENTED_FOR_ITER, + [INSTRUMENTED_INSTRUCTION] = _TAIL_CALL_TRACING_INSTRUMENTED_INSTRUCTION, + [INSTRUMENTED_JUMP_BACKWARD] = _TAIL_CALL_TRACING_INSTRUMENTED_JUMP_BACKWARD, + [INSTRUMENTED_JUMP_FORWARD] = _TAIL_CALL_TRACING_INSTRUMENTED_JUMP_FORWARD, + [INSTRUMENTED_LINE] = _TAIL_CALL_TRACING_INSTRUMENTED_LINE, + [INSTRUMENTED_LOAD_SUPER_ATTR] = _TAIL_CALL_TRACING_INSTRUMENTED_LOAD_SUPER_ATTR, + [INSTRUMENTED_NOT_TAKEN] = _TAIL_CALL_TRACING_INSTRUMENTED_NOT_TAKEN, + [INSTRUMENTED_POP_ITER] = _TAIL_CALL_TRACING_INSTRUMENTED_POP_ITER, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = _TAIL_CALL_TRACING_INSTRUMENTED_POP_JUMP_IF_FALSE, + [INSTRUMENTED_POP_JUMP_IF_NONE] = _TAIL_CALL_TRACING_INSTRUMENTED_POP_JUMP_IF_NONE, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_TRACING_INSTRUMENTED_POP_JUMP_IF_NOT_NONE, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = _TAIL_CALL_TRACING_INSTRUMENTED_POP_JUMP_IF_TRUE, + [INSTRUMENTED_RESUME] = _TAIL_CALL_TRACING_INSTRUMENTED_RESUME, + [INSTRUMENTED_RETURN_VALUE] = _TAIL_CALL_TRACING_INSTRUMENTED_RETURN_VALUE, + [INSTRUMENTED_YIELD_VALUE] = _TAIL_CALL_TRACING_INSTRUMENTED_YIELD_VALUE, + [INTERPRETER_EXIT] = _TAIL_CALL_TRACING_INTERPRETER_EXIT, + [IS_OP] = _TAIL_CALL_TRACING_IS_OP, + [JUMP_BACKWARD] = _TAIL_CALL_TRACING_JUMP_BACKWARD, + [JUMP_BACKWARD_JIT] = _TAIL_CALL_TRACING_JUMP_BACKWARD_JIT, + [JUMP_BACKWARD_NO_INTERRUPT] = _TAIL_CALL_TRACING_JUMP_BACKWARD_NO_INTERRUPT, + [JUMP_BACKWARD_NO_JIT] = _TAIL_CALL_TRACING_JUMP_BACKWARD_NO_JIT, + [JUMP_FORWARD] = _TAIL_CALL_TRACING_JUMP_FORWARD, + [LIST_APPEND] = _TAIL_CALL_TRACING_LIST_APPEND, + [LIST_EXTEND] = _TAIL_CALL_TRACING_LIST_EXTEND, + [LOAD_ATTR] = _TAIL_CALL_TRACING_LOAD_ATTR, + [LOAD_ATTR_CLASS] = _TAIL_CALL_TRACING_LOAD_ATTR_CLASS, + [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = _TAIL_CALL_TRACING_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = _TAIL_CALL_TRACING_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, + [LOAD_ATTR_INSTANCE_VALUE] = _TAIL_CALL_TRACING_LOAD_ATTR_INSTANCE_VALUE, + [LOAD_ATTR_METHOD_LAZY_DICT] = _TAIL_CALL_TRACING_LOAD_ATTR_METHOD_LAZY_DICT, + [LOAD_ATTR_METHOD_NO_DICT] = _TAIL_CALL_TRACING_LOAD_ATTR_METHOD_NO_DICT, + [LOAD_ATTR_METHOD_WITH_VALUES] = _TAIL_CALL_TRACING_LOAD_ATTR_METHOD_WITH_VALUES, + [LOAD_ATTR_MODULE] = _TAIL_CALL_TRACING_LOAD_ATTR_MODULE, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = _TAIL_CALL_TRACING_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = _TAIL_CALL_TRACING_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, + [LOAD_ATTR_PROPERTY] = _TAIL_CALL_TRACING_LOAD_ATTR_PROPERTY, + [LOAD_ATTR_SLOT] = _TAIL_CALL_TRACING_LOAD_ATTR_SLOT, + [LOAD_ATTR_WITH_HINT] = _TAIL_CALL_TRACING_LOAD_ATTR_WITH_HINT, + [LOAD_BUILD_CLASS] = _TAIL_CALL_TRACING_LOAD_BUILD_CLASS, + [LOAD_COMMON_CONSTANT] = _TAIL_CALL_TRACING_LOAD_COMMON_CONSTANT, + [LOAD_CONST] = _TAIL_CALL_TRACING_LOAD_CONST, + [LOAD_DEREF] = _TAIL_CALL_TRACING_LOAD_DEREF, + [LOAD_FAST] = _TAIL_CALL_TRACING_LOAD_FAST, + [LOAD_FAST_AND_CLEAR] = _TAIL_CALL_TRACING_LOAD_FAST_AND_CLEAR, + [LOAD_FAST_BORROW] = _TAIL_CALL_TRACING_LOAD_FAST_BORROW, + [LOAD_FAST_BORROW_LOAD_FAST_BORROW] = _TAIL_CALL_TRACING_LOAD_FAST_BORROW_LOAD_FAST_BORROW, + [LOAD_FAST_CHECK] = _TAIL_CALL_TRACING_LOAD_FAST_CHECK, + [LOAD_FAST_LOAD_FAST] = _TAIL_CALL_TRACING_LOAD_FAST_LOAD_FAST, + [LOAD_FROM_DICT_OR_DEREF] = _TAIL_CALL_TRACING_LOAD_FROM_DICT_OR_DEREF, + [LOAD_FROM_DICT_OR_GLOBALS] = _TAIL_CALL_TRACING_LOAD_FROM_DICT_OR_GLOBALS, + [LOAD_GLOBAL] = _TAIL_CALL_TRACING_LOAD_GLOBAL, + [LOAD_GLOBAL_BUILTIN] = _TAIL_CALL_TRACING_LOAD_GLOBAL_BUILTIN, + [LOAD_GLOBAL_MODULE] = _TAIL_CALL_TRACING_LOAD_GLOBAL_MODULE, + [LOAD_LOCALS] = _TAIL_CALL_TRACING_LOAD_LOCALS, + [LOAD_NAME] = _TAIL_CALL_TRACING_LOAD_NAME, + [LOAD_SMALL_INT] = _TAIL_CALL_TRACING_LOAD_SMALL_INT, + [LOAD_SPECIAL] = _TAIL_CALL_TRACING_LOAD_SPECIAL, + [LOAD_SUPER_ATTR] = _TAIL_CALL_TRACING_LOAD_SUPER_ATTR, + [LOAD_SUPER_ATTR_ATTR] = _TAIL_CALL_TRACING_LOAD_SUPER_ATTR_ATTR, + [LOAD_SUPER_ATTR_METHOD] = _TAIL_CALL_TRACING_LOAD_SUPER_ATTR_METHOD, + [MAKE_CELL] = _TAIL_CALL_TRACING_MAKE_CELL, + [MAKE_FUNCTION] = _TAIL_CALL_TRACING_MAKE_FUNCTION, + [MAP_ADD] = _TAIL_CALL_TRACING_MAP_ADD, + [MATCH_CLASS] = _TAIL_CALL_TRACING_MATCH_CLASS, + [MATCH_KEYS] = _TAIL_CALL_TRACING_MATCH_KEYS, + [MATCH_MAPPING] = _TAIL_CALL_TRACING_MATCH_MAPPING, + [MATCH_SEQUENCE] = _TAIL_CALL_TRACING_MATCH_SEQUENCE, + [NOP] = _TAIL_CALL_TRACING_NOP, + [NOT_TAKEN] = _TAIL_CALL_TRACING_NOT_TAKEN, + [POP_EXCEPT] = _TAIL_CALL_TRACING_POP_EXCEPT, + [POP_ITER] = _TAIL_CALL_TRACING_POP_ITER, + [POP_JUMP_IF_FALSE] = _TAIL_CALL_TRACING_POP_JUMP_IF_FALSE, + [POP_JUMP_IF_NONE] = _TAIL_CALL_TRACING_POP_JUMP_IF_NONE, + [POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_TRACING_POP_JUMP_IF_NOT_NONE, + [POP_JUMP_IF_TRUE] = _TAIL_CALL_TRACING_POP_JUMP_IF_TRUE, + [POP_TOP] = _TAIL_CALL_TRACING_POP_TOP, + [PUSH_EXC_INFO] = _TAIL_CALL_TRACING_PUSH_EXC_INFO, + [PUSH_NULL] = _TAIL_CALL_TRACING_PUSH_NULL, + [RAISE_VARARGS] = _TAIL_CALL_TRACING_RAISE_VARARGS, + [RERAISE] = _TAIL_CALL_TRACING_RERAISE, + [RESERVED] = _TAIL_CALL_TRACING_RESERVED, + [RESUME] = _TAIL_CALL_TRACING_RESUME, + [RESUME_CHECK] = _TAIL_CALL_TRACING_RESUME_CHECK, + [RETURN_GENERATOR] = _TAIL_CALL_TRACING_RETURN_GENERATOR, + [RETURN_VALUE] = _TAIL_CALL_TRACING_RETURN_VALUE, + [SEND] = _TAIL_CALL_TRACING_SEND, + [SEND_GEN] = _TAIL_CALL_TRACING_SEND_GEN, + [SETUP_ANNOTATIONS] = _TAIL_CALL_TRACING_SETUP_ANNOTATIONS, + [SET_ADD] = _TAIL_CALL_TRACING_SET_ADD, + [SET_FUNCTION_ATTRIBUTE] = _TAIL_CALL_TRACING_SET_FUNCTION_ATTRIBUTE, + [SET_UPDATE] = _TAIL_CALL_TRACING_SET_UPDATE, + [STORE_ATTR] = _TAIL_CALL_TRACING_STORE_ATTR, + [STORE_ATTR_INSTANCE_VALUE] = _TAIL_CALL_TRACING_STORE_ATTR_INSTANCE_VALUE, + [STORE_ATTR_SLOT] = _TAIL_CALL_TRACING_STORE_ATTR_SLOT, + [STORE_ATTR_WITH_HINT] = _TAIL_CALL_TRACING_STORE_ATTR_WITH_HINT, + [STORE_DEREF] = _TAIL_CALL_TRACING_STORE_DEREF, + [STORE_FAST] = _TAIL_CALL_TRACING_STORE_FAST, + [STORE_FAST_LOAD_FAST] = _TAIL_CALL_TRACING_STORE_FAST_LOAD_FAST, + [STORE_FAST_STORE_FAST] = _TAIL_CALL_TRACING_STORE_FAST_STORE_FAST, + [STORE_GLOBAL] = _TAIL_CALL_TRACING_STORE_GLOBAL, + [STORE_NAME] = _TAIL_CALL_TRACING_STORE_NAME, + [STORE_SLICE] = _TAIL_CALL_TRACING_STORE_SLICE, + [STORE_SUBSCR] = _TAIL_CALL_TRACING_STORE_SUBSCR, + [STORE_SUBSCR_DICT] = _TAIL_CALL_TRACING_STORE_SUBSCR_DICT, + [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_TRACING_STORE_SUBSCR_LIST_INT, + [SWAP] = _TAIL_CALL_TRACING_SWAP, + [TO_BOOL] = _TAIL_CALL_TRACING_TO_BOOL, + [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_TRACING_TO_BOOL_ALWAYS_TRUE, + [TO_BOOL_BOOL] = _TAIL_CALL_TRACING_TO_BOOL_BOOL, + [TO_BOOL_INT] = _TAIL_CALL_TRACING_TO_BOOL_INT, + [TO_BOOL_LIST] = _TAIL_CALL_TRACING_TO_BOOL_LIST, + [TO_BOOL_NONE] = _TAIL_CALL_TRACING_TO_BOOL_NONE, + [TO_BOOL_STR] = _TAIL_CALL_TRACING_TO_BOOL_STR, + [UNARY_INVERT] = _TAIL_CALL_TRACING_UNARY_INVERT, + [UNARY_NEGATIVE] = _TAIL_CALL_TRACING_UNARY_NEGATIVE, + [UNARY_NOT] = _TAIL_CALL_TRACING_UNARY_NOT, + [UNPACK_EX] = _TAIL_CALL_TRACING_UNPACK_EX, + [UNPACK_SEQUENCE] = _TAIL_CALL_TRACING_UNPACK_SEQUENCE, + [UNPACK_SEQUENCE_LIST] = _TAIL_CALL_TRACING_UNPACK_SEQUENCE_LIST, + [UNPACK_SEQUENCE_TUPLE] = _TAIL_CALL_TRACING_UNPACK_SEQUENCE_TUPLE, + [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_TRACING_UNPACK_SEQUENCE_TWO_TUPLE, + [WITH_EXCEPT_START] = _TAIL_CALL_TRACING_WITH_EXCEPT_START, + [YIELD_VALUE] = _TAIL_CALL_TRACING_YIELD_VALUE, + [121] = _TAIL_CALL_UNKNOWN_OPCODE, + [122] = _TAIL_CALL_UNKNOWN_OPCODE, + [123] = _TAIL_CALL_UNKNOWN_OPCODE, + [124] = _TAIL_CALL_UNKNOWN_OPCODE, + [125] = _TAIL_CALL_UNKNOWN_OPCODE, + [126] = _TAIL_CALL_UNKNOWN_OPCODE, + [127] = _TAIL_CALL_UNKNOWN_OPCODE, + [210] = _TAIL_CALL_UNKNOWN_OPCODE, + [211] = _TAIL_CALL_UNKNOWN_OPCODE, + [212] = _TAIL_CALL_UNKNOWN_OPCODE, + [213] = _TAIL_CALL_UNKNOWN_OPCODE, + [214] = _TAIL_CALL_UNKNOWN_OPCODE, + [215] = _TAIL_CALL_UNKNOWN_OPCODE, + [216] = _TAIL_CALL_UNKNOWN_OPCODE, + [217] = _TAIL_CALL_UNKNOWN_OPCODE, + [218] = _TAIL_CALL_UNKNOWN_OPCODE, + [219] = _TAIL_CALL_UNKNOWN_OPCODE, + [220] = _TAIL_CALL_UNKNOWN_OPCODE, + [221] = _TAIL_CALL_UNKNOWN_OPCODE, + [222] = _TAIL_CALL_UNKNOWN_OPCODE, + [223] = _TAIL_CALL_UNKNOWN_OPCODE, + [224] = _TAIL_CALL_UNKNOWN_OPCODE, + [225] = _TAIL_CALL_UNKNOWN_OPCODE, + [226] = _TAIL_CALL_UNKNOWN_OPCODE, + [227] = _TAIL_CALL_UNKNOWN_OPCODE, + [228] = _TAIL_CALL_UNKNOWN_OPCODE, + [229] = _TAIL_CALL_UNKNOWN_OPCODE, + [230] = _TAIL_CALL_UNKNOWN_OPCODE, + [231] = _TAIL_CALL_UNKNOWN_OPCODE, + [232] = _TAIL_CALL_UNKNOWN_OPCODE, + [233] = _TAIL_CALL_UNKNOWN_OPCODE, +}; #endif /* _Py_TAIL_CALL_INTERP */ diff --git a/Python/pystate.c b/Python/pystate.c index 29c713dccc9fe8..9fe946023741ad 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -555,6 +555,8 @@ init_interpreter(PyInterpreterState *interp, #ifdef _Py_TIER2 // Ensure the buffer is to be set as NULL. interp->jit_uop_buffer = NULL; + interp->jit_tracer_code_buffer = NULL; + interp->jit_tracer_initial_instr = NULL; #endif llist_init(&interp->mem_free_queue.head); llist_init(&interp->asyncio_tasks_head); @@ -808,6 +810,10 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) _PyObject_VirtualFree(interp->jit_uop_buffer, UOP_BUFFER_SIZE); interp->jit_uop_buffer = NULL; } + if (interp->jit_tracer_code_buffer != NULL) { + _PyObject_VirtualFree(interp->jit_tracer_code_buffer, TRACER_BUFFER_SIZE); + interp->jit_tracer_code_buffer = NULL; + } #endif _PyAST_Fini(interp); _PyAtExit_Fini(interp); diff --git a/Tools/cases_generator/target_generator.py b/Tools/cases_generator/target_generator.py index 324ef2773abe28..b6570d202d847d 100644 --- a/Tools/cases_generator/target_generator.py +++ b/Tools/cases_generator/target_generator.py @@ -31,6 +31,14 @@ def write_opcode_targets(analysis: Analysis, out: CWriter) -> None: for target in targets: out.emit(target) out.emit("};\n") + targets = ["&&_unknown_opcode,\n"] * 256 + for name, op in analysis.opmap.items(): + if op < 256: + targets[op] = f"&&TARGET_TRACING_{name},\n" + out.emit("static void *opcode_tracing_targets_table[256] = {\n") + for target in targets: + out.emit(target) + out.emit("};\n") out.emit("#else /* _Py_TAIL_CALL_INTERP */\n") def function_proto(name: str) -> str: @@ -38,7 +46,9 @@ def function_proto(name: str) -> str: def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: - out.emit("static py_tail_call_funcptr instruction_funcptr_table[256];\n") + out.emit("static py_tail_call_funcptr instruction_funcptr_handler_table[256];\n") + out.emit("\n") + out.emit("static py_tail_call_funcptr instruction_funcptr_tracing_table[256];\n") out.emit("\n") # Emit function prototypes for labels. @@ -49,6 +59,7 @@ def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: # Emit function prototypes for opcode handlers. for name in sorted(analysis.instructions.keys()): out.emit(f"{function_proto(name)};\n") + out.emit(f"{function_proto('TRACING_' + name)};\n") out.emit("\n") # Emit unknown opcode handler. @@ -60,7 +71,7 @@ def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: out.emit("\n") # Emit the dispatch table. - out.emit("static py_tail_call_funcptr instruction_funcptr_table[256] = {\n") + out.emit("static py_tail_call_funcptr instruction_funcptr_handler_table[256] = {\n") for name in sorted(analysis.instructions.keys()): out.emit(f"[{name}] = _TAIL_CALL_{name},\n") named_values = analysis.opmap.values() @@ -68,6 +79,16 @@ def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: if rest not in named_values: out.emit(f"[{rest}] = _TAIL_CALL_UNKNOWN_OPCODE,\n") out.emit("};\n") + + # Emit the tracing dispatch table. + out.emit("static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = {\n") + for name in sorted(analysis.instructions.keys()): + out.emit(f"[{name}] = _TAIL_CALL_TRACING_{name},\n") + named_values = analysis.opmap.values() + for rest in range(256): + if rest not in named_values: + out.emit(f"[{rest}] = _TAIL_CALL_UNKNOWN_OPCODE,\n") + out.emit("};\n") outfile.write("#endif /* _Py_TAIL_CALL_INTERP */\n") arg_parser = argparse.ArgumentParser( diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 94ffb0118f0786..349deeacd1b2e6 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -5,6 +5,7 @@ import argparse +from tracer_generator import generate_tracer_cases from analyzer import ( Analysis, Instruction, @@ -167,7 +168,10 @@ def generate_tier1( {INSTRUCTION_START_MARKER} """ ) - generate_tier1_cases(analysis, outfile, lines) + out = CWriter(outfile, 2, lines) + emitter = Emitter(out, analysis.labels) + generate_tier1_cases(analysis, out, emitter) + generate_tracer_cases(analysis, out) outfile.write(f""" {INSTRUCTION_END_MARKER} #if !_Py_TAIL_CALL_INTERP @@ -215,14 +219,13 @@ def get_popped(inst: Instruction, analysis: Analysis) -> str: return (-stack.base_offset).to_c() def generate_tier1_cases( - analysis: Analysis, outfile: TextIO, lines: bool + analysis: Analysis, out: CWriter, emitter: Emitter, is_tracing: bool = False ) -> None: - out = CWriter(outfile, 2, lines) - emitter = Emitter(out, analysis.labels) + tracing_prepend = "TRACING_" if is_tracing else "" out.emit("\n") for name, inst in sorted(analysis.instructions.items()): out.emit("\n") - out.emit(f"TARGET({name}) {{\n") + out.emit(f"{tracing_prepend}TARGET({name}) {{\n") popped = get_popped(inst, analysis) # We need to ifdef it because this breaks platforms # without computed gotos/tail calling. @@ -230,7 +233,7 @@ def generate_tier1_cases( out.emit(f"int opcode = {name};\n") out.emit(f"(void)(opcode);\n") out.emit(f"#endif\n") - needs_this = uses_this(inst) + needs_this = is_tracing or uses_this(inst) unused_guard = "(void)this_instr;\n" if inst.properties.needs_prev: out.emit(f"_Py_CODEUNIT* const prev_instr = frame->instr_ptr;\n") @@ -244,7 +247,7 @@ def generate_tier1_cases( out.emit(f"next_instr += {inst.size};\n") out.emit(f"INSTRUCTION_STATS({name});\n") if inst.is_target: - out.emit(f"PREDICTED_{name}:;\n") + out.emit(f"PREDICTED_{tracing_prepend}{name}:;\n") if needs_this: out.emit(f"_Py_CODEUNIT* const this_instr = next_instr - {inst.size};\n") out.emit(unused_guard) @@ -265,7 +268,7 @@ def generate_tier1_cases( out.start_line() if reachable: # type: ignore[possibly-undefined] stack.flush(out) - out.emit("DISPATCH();\n") + out.emit(f"{tracing_prepend}DISPATCH();\n") out.start_line() out.emit("}") out.emit("\n") diff --git a/Tools/cases_generator/tracer_generator.py b/Tools/cases_generator/tracer_generator.py new file mode 100644 index 00000000000000..0df435a9e32766 --- /dev/null +++ b/Tools/cases_generator/tracer_generator.py @@ -0,0 +1,66 @@ +import argparse + +from analyzer import ( + Analysis, + Instruction, + Uop, + Label, + CodeSection, + Part, + analyze_files, + Skip, + Flush, + analysis_error, + StackItem, +) +from generators_common import ( + DEFAULT_INPUT, + ROOT, + write_header, + type_and_null, + Emitter, + TokenIterator, + always_true, + emit_to, + ReplacementFunctionType, +) +from cwriter import CWriter +from typing import TextIO +from lexer import Token +from stack import Local, Stack, StackError, get_stack_effect, Storage + +class TracerEmitter(Emitter): + out: CWriter + labels: dict[str, Label] + _replacers: dict[str, ReplacementFunctionType] + cannot_escape: bool + + def __init__(self, out: CWriter, labels: dict[str, Label], cannot_escape: bool = False): + super().__init__(out, labels, cannot_escape) + self._replacers = { + **self._replacers, + "DISPATCH": self.dispatch, + } + + def dispatch( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ) -> bool: + if storage.spilled: + raise analysis_error("stack_pointer needs reloading before dispatch", tkn) + storage.stack.flush(self.out) + self.emit("TRACING_DISPATCH") + return False + +def generate_tracer_cases( + analysis: Analysis, out: CWriter +): + # Import inside to avoid circular imports. + from tier1_generator import generate_tier1_cases + out.emit(f"#ifdef _Py_TIER2 /* BEGIN TRACING INSTRUCTIONS */\n") + generate_tier1_cases(analysis, out, TracerEmitter(out, analysis.labels), is_tracing=True) + out.emit(f"#endif /* END TRACING INSTRUCTIONS */\n") \ No newline at end of file From 40bf6c1964d5870f8fe79c5d431417cc8967c133 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 19 Sep 2025 15:19:38 +0100 Subject: [PATCH 002/190] WIP generators --- Include/internal/pycore_interp_structs.h | 7 + Include/internal/pycore_magic_number.h | 3 +- Include/internal/pycore_opcode_metadata.h | 72 +- Include/internal/pycore_optimizer.h | 2 +- Include/internal/pycore_uop.h | 2 +- Include/internal/pycore_uop_ids.h | 351 +++--- Include/internal/pycore_uop_metadata.h | 26 +- Include/opcode_ids.h | 168 +-- Lib/_opcode_metadata.py | 168 +-- Python/bytecodes.c | 110 +- Python/ceval.c | 98 +- Python/ceval_macros.h | 55 +- Python/executor_cases.c.h | 124 +- Python/generated_cases.c.h | 202 ++- Python/opcode_targets.h | 20 +- Python/optimizer.c | 414 ++----- Python/optimizer_analysis.c | 14 +- Python/optimizer_cases.c.h | 33 +- Python/pystate.c | 4 + Tools/cases_generator/analyzer.py | 4 + .../opcode_metadata_generator.py | 11 + Tools/cases_generator/tier2_generator.py | 44 + Tools/cases_generator/tracer_generator.py | 1 + out.txt | 1089 +++++++++++++++++ 24 files changed, 2248 insertions(+), 774 deletions(-) create mode 100644 out.txt diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index a0a0e8665398a1..2cd61c215ce01d 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -943,9 +943,16 @@ struct _is { struct types_state types; struct callable_cache callable_cache; PyObject *common_consts[NUM_COMMON_CONSTANTS]; + // JIT tracing state int jit_tracer_code_curr_size; _Py_CODEUNIT *jit_tracer_code_buffer; _Py_CODEUNIT *jit_tracer_initial_instr; + int jit_tracer_initial_stack_depth; + int jit_tracer_initial_chain_depth; + PyCodeObject *jit_tracer_initial_code; // Borrowed + PyFunctionObject *jit_tracer_initial_func; // Borrowed + int jit_tracer_seen_initial_before; + bool jit_completed_loop; bool jit; bool compiling; struct _PyUOpInstruction *jit_uop_buffer; diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index 7ec7bd1c695516..e1f5d29d2b8cc1 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -286,6 +286,7 @@ Known values: Python 3.15a1 3653 (Fix handling of opcodes that may leave operands on the stack when optimizing LOAD_FAST) Python 3.15a1 3654 (Fix missing exception handlers in logical expression) Python 3.15a1 3655 (Fix miscompilation of some module-level annotations) + Python 3.15a1 3656 (Add GUARD_IP instruction) Python 3.16 will start with 3700 @@ -299,7 +300,7 @@ PC/launcher.c must also be updated. */ -#define PYC_MAGIC_NUMBER 3655 +#define PYC_MAGIC_NUMBER 3656 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index bd6b84ec7fd908..97642c2ff4678b 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -474,6 +474,10 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 3; case SWAP: return 2 + (oparg-2); + case TIER1_GUARD_IP: + return 0; + case TIER1_SET_IP: + return 0; case TO_BOOL: return 1; case TO_BOOL_ALWAYS_TRUE: @@ -957,6 +961,10 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 0; case SWAP: return 2 + (oparg-2); + case TIER1_GUARD_IP: + return 0; + case TIER1_SET_IP: + return 0; case TO_BOOL: return 1; case TO_BOOL_ALWAYS_TRUE: @@ -1276,6 +1284,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [STORE_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [SWAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, + [TIER1_GUARD_IP] = { true, INSTR_FMT_IXC000, HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG }, + [TIER1_SET_IP] = { true, INSTR_FMT_IXC000, HAS_ESCAPES_FLAG }, [TO_BOOL] = { true, INSTR_FMT_IXC00, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, @@ -1394,7 +1404,7 @@ _PyOpcode_macro_expansion[256] = { [FORMAT_WITH_SPEC] = { .nuops = 1, .uops = { { _FORMAT_WITH_SPEC, OPARG_SIMPLE, 0 } } }, [FOR_ITER] = { .nuops = 1, .uops = { { _FOR_ITER, OPARG_REPLACED, 0 } } }, [FOR_ITER_GEN] = { .nuops = 3, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _FOR_ITER_GEN_FRAME, OPARG_SIMPLE, 1 }, { _PUSH_FRAME, OPARG_SIMPLE, 1 } } }, - [FOR_ITER_LIST] = { .nuops = 3, .uops = { { _ITER_CHECK_LIST, OPARG_SIMPLE, 1 }, { _ITER_JUMP_LIST, OPARG_REPLACED, 1 }, { _ITER_NEXT_LIST, OPARG_REPLACED, 1 } } }, + [FOR_ITER_LIST] = { .nuops = 3, .uops = { { _ITER_CHECK_LIST, OPARG_SIMPLE, 1 }, { _ITER_JUMP_LIST, OPARG_REPLACED, 1 }, { _ITER_NEXT_LIST, OPARG_SIMPLE, 1 } } }, [FOR_ITER_RANGE] = { .nuops = 3, .uops = { { _ITER_CHECK_RANGE, OPARG_SIMPLE, 1 }, { _ITER_JUMP_RANGE, OPARG_REPLACED, 1 }, { _ITER_NEXT_RANGE, OPARG_SIMPLE, 1 } } }, [FOR_ITER_TUPLE] = { .nuops = 3, .uops = { { _ITER_CHECK_TUPLE, OPARG_SIMPLE, 1 }, { _ITER_JUMP_TUPLE, OPARG_REPLACED, 1 }, { _ITER_NEXT_TUPLE, OPARG_SIMPLE, 1 } } }, [GET_AITER] = { .nuops = 1, .uops = { { _GET_AITER, OPARG_SIMPLE, 0 } } }, @@ -1406,6 +1416,8 @@ _PyOpcode_macro_expansion[256] = { [IMPORT_FROM] = { .nuops = 1, .uops = { { _IMPORT_FROM, OPARG_SIMPLE, 0 } } }, [IMPORT_NAME] = { .nuops = 1, .uops = { { _IMPORT_NAME, OPARG_SIMPLE, 0 } } }, [IS_OP] = { .nuops = 1, .uops = { { _IS_OP, OPARG_SIMPLE, 0 } } }, + [JUMP_BACKWARD_NO_INTERRUPT] = { .nuops = 1, .uops = { { _JUMP_BACKWARD_NO_INTERRUPT, OPARG_SIMPLE, 0 } } }, + [JUMP_BACKWARD_NO_JIT] = { .nuops = 2, .uops = { { _CHECK_PERIODIC, OPARG_SIMPLE, 1 }, { _JUMP_BACKWARD_NO_INTERRUPT, OPARG_SIMPLE, 1 } } }, [LIST_APPEND] = { .nuops = 1, .uops = { { _LIST_APPEND, OPARG_SIMPLE, 0 } } }, [LIST_EXTEND] = { .nuops = 1, .uops = { { _LIST_EXTEND, OPARG_SIMPLE, 0 } } }, [LOAD_ATTR] = { .nuops = 1, .uops = { { _LOAD_ATTR, OPARG_SIMPLE, 8 } } }, @@ -1452,10 +1464,10 @@ _PyOpcode_macro_expansion[256] = { [NOT_TAKEN] = { .nuops = 1, .uops = { { _NOP, OPARG_SIMPLE, 0 } } }, [POP_EXCEPT] = { .nuops = 1, .uops = { { _POP_EXCEPT, OPARG_SIMPLE, 0 } } }, [POP_ITER] = { .nuops = 1, .uops = { { _POP_ITER, OPARG_SIMPLE, 0 } } }, - [POP_JUMP_IF_FALSE] = { .nuops = 1, .uops = { { _POP_JUMP_IF_FALSE, OPARG_REPLACED, 1 } } }, - [POP_JUMP_IF_NONE] = { .nuops = 2, .uops = { { _IS_NONE, OPARG_SIMPLE, 1 }, { _POP_JUMP_IF_TRUE, OPARG_REPLACED, 1 } } }, - [POP_JUMP_IF_NOT_NONE] = { .nuops = 2, .uops = { { _IS_NONE, OPARG_SIMPLE, 1 }, { _POP_JUMP_IF_FALSE, OPARG_REPLACED, 1 } } }, - [POP_JUMP_IF_TRUE] = { .nuops = 1, .uops = { { _POP_JUMP_IF_TRUE, OPARG_REPLACED, 1 } } }, + [POP_JUMP_IF_FALSE] = { .nuops = 1, .uops = { { _POP_JUMP_IF_FALSE, OPARG_SIMPLE, 1 } } }, + [POP_JUMP_IF_NONE] = { .nuops = 2, .uops = { { _IS_NONE, OPARG_SIMPLE, 1 }, { _POP_JUMP_IF_TRUE, OPARG_SIMPLE, 1 } } }, + [POP_JUMP_IF_NOT_NONE] = { .nuops = 2, .uops = { { _IS_NONE, OPARG_SIMPLE, 1 }, { _POP_JUMP_IF_FALSE, OPARG_SIMPLE, 1 } } }, + [POP_JUMP_IF_TRUE] = { .nuops = 1, .uops = { { _POP_JUMP_IF_TRUE, OPARG_SIMPLE, 1 } } }, [POP_TOP] = { .nuops = 1, .uops = { { _POP_TOP, OPARG_SIMPLE, 0 } } }, [PUSH_EXC_INFO] = { .nuops = 1, .uops = { { _PUSH_EXC_INFO, OPARG_SIMPLE, 0 } } }, [PUSH_NULL] = { .nuops = 1, .uops = { { _PUSH_NULL, OPARG_SIMPLE, 0 } } }, @@ -1724,6 +1736,8 @@ const char *_PyOpcode_OpName[267] = { [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", [SWAP] = "SWAP", + [TIER1_GUARD_IP] = "TIER1_GUARD_IP", + [TIER1_SET_IP] = "TIER1_SET_IP", [TO_BOOL] = "TO_BOOL", [TO_BOOL_ALWAYS_TRUE] = "TO_BOOL_ALWAYS_TRUE", [TO_BOOL_BOOL] = "TO_BOOL_BOOL", @@ -1747,6 +1761,8 @@ const char *_PyOpcode_OpName[267] = { extern const uint8_t _PyOpcode_Caches[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Caches[256] = { + [TIER1_GUARD_IP] = 4, + [TIER1_SET_IP] = 4, [TO_BOOL] = 3, [STORE_SUBSCR] = 1, [SEND] = 1, @@ -1769,11 +1785,49 @@ const uint8_t _PyOpcode_Caches[256] = { }; #endif +extern const uint8_t _PyOpcode_NeedsGuardIp[256]; +#ifdef NEED_OPCODE_METADATA +const uint8_t _PyOpcode_NeedsGuardIp[256] = { + [INTERPRETER_EXIT] = 1, + [RETURN_VALUE] = 1, + [YIELD_VALUE] = 1, + [JUMP_FORWARD] = 1, + [INSTRUMENTED_FOR_ITER] = 1, + [RETURN_GENERATOR] = 1, + [BINARY_OP_SUBSCR_GETITEM] = 1, + [INSTRUMENTED_RETURN_VALUE] = 1, + [SEND] = 1, + [SEND_GEN] = 1, + [INSTRUMENTED_YIELD_VALUE] = 1, + [INSTRUMENTED_END_ASYNC_FOR] = 1, + [END_ASYNC_FOR] = 1, + [LOAD_ATTR_PROPERTY] = 1, + [JUMP_BACKWARD] = 1, + [JUMP_BACKWARD_NO_JIT] = 1, + [JUMP_BACKWARD_JIT] = 1, + [POP_JUMP_IF_TRUE] = 1, + [POP_JUMP_IF_FALSE] = 1, + [POP_JUMP_IF_NONE] = 1, + [POP_JUMP_IF_NOT_NONE] = 1, + [JUMP_BACKWARD_NO_INTERRUPT] = 1, + [FOR_ITER] = 1, + [FOR_ITER_LIST] = 1, + [FOR_ITER_TUPLE] = 1, + [FOR_ITER_RANGE] = 1, + [FOR_ITER_GEN] = 1, + [CALL_PY_GENERAL] = 1, + [CALL_BOUND_METHOD_GENERAL] = 1, + [CALL_BOUND_METHOD_EXACT_ARGS] = 1, + [CALL_PY_EXACT_ARGS] = 1, + [CALL_ALLOC_AND_ENTER_INIT] = 1, + [CALL_KW_PY] = 1, + [CALL_KW_BOUND_METHOD] = 1, +}; +#endif + extern const uint8_t _PyOpcode_Deopt[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Deopt[256] = { - [121] = 121, - [122] = 122, [123] = 123, [124] = 124, [125] = 125, @@ -2011,6 +2065,8 @@ const uint8_t _PyOpcode_Deopt[256] = { [STORE_SUBSCR_DICT] = STORE_SUBSCR, [STORE_SUBSCR_LIST_INT] = STORE_SUBSCR, [SWAP] = SWAP, + [TIER1_GUARD_IP] = TIER1_GUARD_IP, + [TIER1_SET_IP] = TIER1_SET_IP, [TO_BOOL] = TO_BOOL, [TO_BOOL_ALWAYS_TRUE] = TO_BOOL, [TO_BOOL_BOOL] = TO_BOOL, @@ -2033,8 +2089,6 @@ const uint8_t _PyOpcode_Deopt[256] = { #endif // NEED_OPCODE_METADATA #define EXTRA_CASES \ - case 121: \ - case 122: \ case 123: \ case 124: \ case 125: \ diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 685c39dcd65fb9..9bfd301efecc9c 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -336,7 +336,7 @@ extern int _Py_uop_frame_pop(JitOptContext *ctx); PyAPI_FUNC(PyObject *) _Py_uop_symbols_test(PyObject *self, PyObject *ignored); -PyAPI_FUNC(int) _PyOptimizer_Optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *start, _PyExecutorObject **exec_ptr, int chain_depth); +PyAPI_FUNC(int) _PyOptimizer_Optimize(_PyInterpreterFrame *frame, PyThreadState *tstate); static inline _PyExecutorObject *_PyExecutor_FromExit(_PyExitData *exit) { diff --git a/Include/internal/pycore_uop.h b/Include/internal/pycore_uop.h index f364c01ba92083..301779905cf4c5 100644 --- a/Include/internal/pycore_uop.h +++ b/Include/internal/pycore_uop.h @@ -41,7 +41,7 @@ typedef struct _PyUOpInstruction{ // This is the length of the trace we record. // This includes the inline caches. -#define TRACE_MAX_TRACE_LENGTH 1000 +#define TRACE_MAX_TRACE_LENGTH 2000 #define TRACER_BUFFER_SIZE ((int)(TRACE_MAX_TRACE_LENGTH * sizeof(_Py_CODEUNIT))) #ifdef __cplusplus diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index ff1d75c0cb1938..2ef1fccc8d8ee2 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -108,74 +108,76 @@ extern "C" { #define _DO_CALL 374 #define _DO_CALL_FUNCTION_EX 375 #define _DO_CALL_KW 376 +#define _DYNAMIC_EXIT 377 #define _END_FOR END_FOR #define _END_SEND END_SEND -#define _ERROR_POP_N 377 +#define _ERROR_POP_N 378 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _EXPAND_METHOD 378 -#define _EXPAND_METHOD_KW 379 -#define _FATAL_ERROR 380 +#define _EXPAND_METHOD 379 +#define _EXPAND_METHOD_KW 380 +#define _FATAL_ERROR 381 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 381 -#define _FOR_ITER_GEN_FRAME 382 -#define _FOR_ITER_TIER_TWO 383 +#define _FOR_ITER 382 +#define _FOR_ITER_GEN_FRAME 383 +#define _FOR_ITER_TIER_TWO 384 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BINARY_OP_EXTEND 384 -#define _GUARD_CALLABLE_ISINSTANCE 385 -#define _GUARD_CALLABLE_LEN 386 -#define _GUARD_CALLABLE_LIST_APPEND 387 -#define _GUARD_CALLABLE_STR_1 388 -#define _GUARD_CALLABLE_TUPLE_1 389 -#define _GUARD_CALLABLE_TYPE_1 390 -#define _GUARD_DORV_NO_DICT 391 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 392 -#define _GUARD_GLOBALS_VERSION 393 -#define _GUARD_IS_FALSE_POP 394 -#define _GUARD_IS_NONE_POP 395 -#define _GUARD_IS_NOT_NONE_POP 396 -#define _GUARD_IS_TRUE_POP 397 -#define _GUARD_KEYS_VERSION 398 -#define _GUARD_NOS_DICT 399 -#define _GUARD_NOS_FLOAT 400 -#define _GUARD_NOS_INT 401 -#define _GUARD_NOS_LIST 402 -#define _GUARD_NOS_NOT_NULL 403 -#define _GUARD_NOS_NULL 404 -#define _GUARD_NOS_OVERFLOWED 405 -#define _GUARD_NOS_TUPLE 406 -#define _GUARD_NOS_UNICODE 407 -#define _GUARD_NOT_EXHAUSTED_LIST 408 -#define _GUARD_NOT_EXHAUSTED_RANGE 409 -#define _GUARD_NOT_EXHAUSTED_TUPLE 410 -#define _GUARD_THIRD_NULL 411 -#define _GUARD_TOS_ANY_SET 412 -#define _GUARD_TOS_DICT 413 -#define _GUARD_TOS_FLOAT 414 -#define _GUARD_TOS_INT 415 -#define _GUARD_TOS_LIST 416 -#define _GUARD_TOS_OVERFLOWED 417 -#define _GUARD_TOS_SLICE 418 -#define _GUARD_TOS_TUPLE 419 -#define _GUARD_TOS_UNICODE 420 -#define _GUARD_TYPE_VERSION 421 -#define _GUARD_TYPE_VERSION_AND_LOCK 422 -#define _HANDLE_PENDING_AND_DEOPT 423 +#define _GUARD_BINARY_OP_EXTEND 385 +#define _GUARD_CALLABLE_ISINSTANCE 386 +#define _GUARD_CALLABLE_LEN 387 +#define _GUARD_CALLABLE_LIST_APPEND 388 +#define _GUARD_CALLABLE_STR_1 389 +#define _GUARD_CALLABLE_TUPLE_1 390 +#define _GUARD_CALLABLE_TYPE_1 391 +#define _GUARD_DORV_NO_DICT 392 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 393 +#define _GUARD_GLOBALS_VERSION 394 +#define _GUARD_IP 395 +#define _GUARD_IS_FALSE_POP 396 +#define _GUARD_IS_NONE_POP 397 +#define _GUARD_IS_NOT_NONE_POP 398 +#define _GUARD_IS_TRUE_POP 399 +#define _GUARD_KEYS_VERSION 400 +#define _GUARD_NOS_DICT 401 +#define _GUARD_NOS_FLOAT 402 +#define _GUARD_NOS_INT 403 +#define _GUARD_NOS_LIST 404 +#define _GUARD_NOS_NOT_NULL 405 +#define _GUARD_NOS_NULL 406 +#define _GUARD_NOS_OVERFLOWED 407 +#define _GUARD_NOS_TUPLE 408 +#define _GUARD_NOS_UNICODE 409 +#define _GUARD_NOT_EXHAUSTED_LIST 410 +#define _GUARD_NOT_EXHAUSTED_RANGE 411 +#define _GUARD_NOT_EXHAUSTED_TUPLE 412 +#define _GUARD_THIRD_NULL 413 +#define _GUARD_TOS_ANY_SET 414 +#define _GUARD_TOS_DICT 415 +#define _GUARD_TOS_FLOAT 416 +#define _GUARD_TOS_INT 417 +#define _GUARD_TOS_LIST 418 +#define _GUARD_TOS_OVERFLOWED 419 +#define _GUARD_TOS_SLICE 420 +#define _GUARD_TOS_TUPLE 421 +#define _GUARD_TOS_UNICODE 422 +#define _GUARD_TYPE_VERSION 423 +#define _GUARD_TYPE_VERSION_AND_LOCK 424 +#define _HANDLE_PENDING_AND_DEOPT 425 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 424 -#define _INIT_CALL_PY_EXACT_ARGS 425 -#define _INIT_CALL_PY_EXACT_ARGS_0 426 -#define _INIT_CALL_PY_EXACT_ARGS_1 427 -#define _INIT_CALL_PY_EXACT_ARGS_2 428 -#define _INIT_CALL_PY_EXACT_ARGS_3 429 -#define _INIT_CALL_PY_EXACT_ARGS_4 430 -#define _INSERT_NULL 431 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 426 +#define _INIT_CALL_PY_EXACT_ARGS 427 +#define _INIT_CALL_PY_EXACT_ARGS_0 428 +#define _INIT_CALL_PY_EXACT_ARGS_1 429 +#define _INIT_CALL_PY_EXACT_ARGS_2 430 +#define _INIT_CALL_PY_EXACT_ARGS_3 431 +#define _INIT_CALL_PY_EXACT_ARGS_4 432 +#define _INSERT_NULL 433 #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER #define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION #define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD @@ -185,177 +187,178 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _IS_NONE 432 +#define _IS_NONE 434 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 433 -#define _ITER_CHECK_RANGE 434 -#define _ITER_CHECK_TUPLE 435 -#define _ITER_JUMP_LIST 436 -#define _ITER_JUMP_RANGE 437 -#define _ITER_JUMP_TUPLE 438 -#define _ITER_NEXT_LIST 439 -#define _ITER_NEXT_LIST_TIER_TWO 440 -#define _ITER_NEXT_RANGE 441 -#define _ITER_NEXT_TUPLE 442 -#define _JUMP_TO_TOP 443 +#define _ITER_CHECK_LIST 435 +#define _ITER_CHECK_RANGE 436 +#define _ITER_CHECK_TUPLE 437 +#define _ITER_JUMP_LIST 438 +#define _ITER_JUMP_RANGE 439 +#define _ITER_JUMP_TUPLE 440 +#define _ITER_NEXT_LIST 441 +#define _ITER_NEXT_LIST_TIER_TWO 442 +#define _ITER_NEXT_RANGE 443 +#define _ITER_NEXT_TUPLE 444 +#define _JUMP_BACKWARD_NO_INTERRUPT 445 +#define _JUMP_TO_TOP 446 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 444 -#define _LOAD_ATTR_CLASS 445 +#define _LOAD_ATTR 447 +#define _LOAD_ATTR_CLASS 448 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 446 -#define _LOAD_ATTR_METHOD_LAZY_DICT 447 -#define _LOAD_ATTR_METHOD_NO_DICT 448 -#define _LOAD_ATTR_METHOD_WITH_VALUES 449 -#define _LOAD_ATTR_MODULE 450 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 451 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 452 -#define _LOAD_ATTR_PROPERTY_FRAME 453 -#define _LOAD_ATTR_SLOT 454 -#define _LOAD_ATTR_WITH_HINT 455 +#define _LOAD_ATTR_INSTANCE_VALUE 449 +#define _LOAD_ATTR_METHOD_LAZY_DICT 450 +#define _LOAD_ATTR_METHOD_NO_DICT 451 +#define _LOAD_ATTR_METHOD_WITH_VALUES 452 +#define _LOAD_ATTR_MODULE 453 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 454 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 455 +#define _LOAD_ATTR_PROPERTY_FRAME 456 +#define _LOAD_ATTR_SLOT 457 +#define _LOAD_ATTR_WITH_HINT 458 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _LOAD_BYTECODE 456 +#define _LOAD_BYTECODE 459 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 457 -#define _LOAD_CONST_INLINE_BORROW 458 -#define _LOAD_CONST_UNDER_INLINE 459 -#define _LOAD_CONST_UNDER_INLINE_BORROW 460 +#define _LOAD_CONST_INLINE 460 +#define _LOAD_CONST_INLINE_BORROW 461 +#define _LOAD_CONST_UNDER_INLINE 462 +#define _LOAD_CONST_UNDER_INLINE_BORROW 463 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 461 -#define _LOAD_FAST_0 462 -#define _LOAD_FAST_1 463 -#define _LOAD_FAST_2 464 -#define _LOAD_FAST_3 465 -#define _LOAD_FAST_4 466 -#define _LOAD_FAST_5 467 -#define _LOAD_FAST_6 468 -#define _LOAD_FAST_7 469 +#define _LOAD_FAST 464 +#define _LOAD_FAST_0 465 +#define _LOAD_FAST_1 466 +#define _LOAD_FAST_2 467 +#define _LOAD_FAST_3 468 +#define _LOAD_FAST_4 469 +#define _LOAD_FAST_5 470 +#define _LOAD_FAST_6 471 +#define _LOAD_FAST_7 472 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR -#define _LOAD_FAST_BORROW 470 -#define _LOAD_FAST_BORROW_0 471 -#define _LOAD_FAST_BORROW_1 472 -#define _LOAD_FAST_BORROW_2 473 -#define _LOAD_FAST_BORROW_3 474 -#define _LOAD_FAST_BORROW_4 475 -#define _LOAD_FAST_BORROW_5 476 -#define _LOAD_FAST_BORROW_6 477 -#define _LOAD_FAST_BORROW_7 478 +#define _LOAD_FAST_BORROW 473 +#define _LOAD_FAST_BORROW_0 474 +#define _LOAD_FAST_BORROW_1 475 +#define _LOAD_FAST_BORROW_2 476 +#define _LOAD_FAST_BORROW_3 477 +#define _LOAD_FAST_BORROW_4 478 +#define _LOAD_FAST_BORROW_5 479 +#define _LOAD_FAST_BORROW_6 480 +#define _LOAD_FAST_BORROW_7 481 #define _LOAD_FAST_BORROW_LOAD_FAST_BORROW LOAD_FAST_BORROW_LOAD_FAST_BORROW #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 479 -#define _LOAD_GLOBAL_BUILTINS 480 -#define _LOAD_GLOBAL_MODULE 481 +#define _LOAD_GLOBAL 482 +#define _LOAD_GLOBAL_BUILTINS 483 +#define _LOAD_GLOBAL_MODULE 484 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 482 -#define _LOAD_SMALL_INT_0 483 -#define _LOAD_SMALL_INT_1 484 -#define _LOAD_SMALL_INT_2 485 -#define _LOAD_SMALL_INT_3 486 -#define _LOAD_SPECIAL 487 +#define _LOAD_SMALL_INT 485 +#define _LOAD_SMALL_INT_0 486 +#define _LOAD_SMALL_INT_1 487 +#define _LOAD_SMALL_INT_2 488 +#define _LOAD_SMALL_INT_3 489 +#define _LOAD_SPECIAL 490 #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _MAKE_CALLARGS_A_TUPLE 488 +#define _MAKE_CALLARGS_A_TUPLE 491 #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_WARM 489 +#define _MAKE_WARM 492 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 490 -#define _MAYBE_EXPAND_METHOD_KW 491 -#define _MONITOR_CALL 492 -#define _MONITOR_CALL_KW 493 -#define _MONITOR_JUMP_BACKWARD 494 -#define _MONITOR_RESUME 495 +#define _MAYBE_EXPAND_METHOD 493 +#define _MAYBE_EXPAND_METHOD_KW 494 +#define _MONITOR_CALL 495 +#define _MONITOR_CALL_KW 496 +#define _MONITOR_JUMP_BACKWARD 497 +#define _MONITOR_RESUME 498 #define _NOP NOP -#define _POP_CALL 496 -#define _POP_CALL_LOAD_CONST_INLINE_BORROW 497 -#define _POP_CALL_ONE 498 -#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 499 -#define _POP_CALL_TWO 500 -#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 501 +#define _POP_CALL 499 +#define _POP_CALL_LOAD_CONST_INLINE_BORROW 500 +#define _POP_CALL_ONE 501 +#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 502 +#define _POP_CALL_TWO 503 +#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 504 #define _POP_EXCEPT POP_EXCEPT #define _POP_ITER POP_ITER -#define _POP_JUMP_IF_FALSE 502 -#define _POP_JUMP_IF_TRUE 503 +#define _POP_JUMP_IF_FALSE 505 +#define _POP_JUMP_IF_TRUE 506 #define _POP_TOP POP_TOP -#define _POP_TOP_FLOAT 504 -#define _POP_TOP_INT 505 -#define _POP_TOP_LOAD_CONST_INLINE 506 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 507 -#define _POP_TOP_NOP 508 -#define _POP_TOP_UNICODE 509 -#define _POP_TWO 510 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW 511 +#define _POP_TOP_FLOAT 507 +#define _POP_TOP_INT 508 +#define _POP_TOP_LOAD_CONST_INLINE 509 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 510 +#define _POP_TOP_NOP 511 +#define _POP_TOP_UNICODE 512 +#define _POP_TWO 513 +#define _POP_TWO_LOAD_CONST_INLINE_BORROW 514 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 512 +#define _PUSH_FRAME 515 #define _PUSH_NULL PUSH_NULL -#define _PUSH_NULL_CONDITIONAL 513 -#define _PY_FRAME_GENERAL 514 -#define _PY_FRAME_KW 515 -#define _QUICKEN_RESUME 516 -#define _REPLACE_WITH_TRUE 517 +#define _PUSH_NULL_CONDITIONAL 516 +#define _PY_FRAME_GENERAL 517 +#define _PY_FRAME_KW 518 +#define _QUICKEN_RESUME 519 +#define _REPLACE_WITH_TRUE 520 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 518 -#define _SEND 519 -#define _SEND_GEN_FRAME 520 +#define _SAVE_RETURN_OFFSET 521 +#define _SEND 522 +#define _SEND_GEN_FRAME 523 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 521 -#define _STORE_ATTR 522 -#define _STORE_ATTR_INSTANCE_VALUE 523 -#define _STORE_ATTR_SLOT 524 -#define _STORE_ATTR_WITH_HINT 525 +#define _START_EXECUTOR 524 +#define _STORE_ATTR 525 +#define _STORE_ATTR_INSTANCE_VALUE 526 +#define _STORE_ATTR_SLOT 527 +#define _STORE_ATTR_WITH_HINT 528 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 526 -#define _STORE_FAST_0 527 -#define _STORE_FAST_1 528 -#define _STORE_FAST_2 529 -#define _STORE_FAST_3 530 -#define _STORE_FAST_4 531 -#define _STORE_FAST_5 532 -#define _STORE_FAST_6 533 -#define _STORE_FAST_7 534 +#define _STORE_FAST 529 +#define _STORE_FAST_0 530 +#define _STORE_FAST_1 531 +#define _STORE_FAST_2 532 +#define _STORE_FAST_3 533 +#define _STORE_FAST_4 534 +#define _STORE_FAST_5 535 +#define _STORE_FAST_6 536 +#define _STORE_FAST_7 537 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 535 -#define _STORE_SUBSCR 536 -#define _STORE_SUBSCR_DICT 537 -#define _STORE_SUBSCR_LIST_INT 538 -#define _SWAP 539 -#define _SWAP_2 540 -#define _SWAP_3 541 -#define _TIER2_RESUME_CHECK 542 -#define _TO_BOOL 543 +#define _STORE_SLICE 538 +#define _STORE_SUBSCR 539 +#define _STORE_SUBSCR_DICT 540 +#define _STORE_SUBSCR_LIST_INT 541 +#define _SWAP 542 +#define _SWAP_2 543 +#define _SWAP_3 544 +#define _TIER2_RESUME_CHECK 545 +#define _TO_BOOL 546 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT -#define _TO_BOOL_LIST 544 +#define _TO_BOOL_LIST 547 #define _TO_BOOL_NONE TO_BOOL_NONE -#define _TO_BOOL_STR 545 +#define _TO_BOOL_STR 548 #define _UNARY_INVERT UNARY_INVERT #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 546 -#define _UNPACK_SEQUENCE_LIST 547 -#define _UNPACK_SEQUENCE_TUPLE 548 -#define _UNPACK_SEQUENCE_TWO_TUPLE 549 +#define _UNPACK_SEQUENCE 549 +#define _UNPACK_SEQUENCE_LIST 550 +#define _UNPACK_SEQUENCE_TUPLE 551 +#define _UNPACK_SEQUENCE_TWO_TUPLE 552 #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 549 +#define MAX_UOP_ID 552 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 1248771996943b..f8099301b6b4b4 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -205,7 +205,10 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CHECK_EXC_MATCH] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_IMPORT_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_IMPORT_FROM] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_POP_JUMP_IF_FALSE] = HAS_ARG_FLAG | HAS_JUMP_FLAG, + [_POP_JUMP_IF_TRUE] = HAS_ARG_FLAG | HAS_JUMP_FLAG, [_IS_NONE] = HAS_ESCAPES_FLAG, + [_JUMP_BACKWARD_NO_INTERRUPT] = HAS_ARG_FLAG | HAS_JUMP_FLAG, [_GET_LEN] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_MATCH_CLASS] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_MATCH_MAPPING] = 0, @@ -213,9 +216,10 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_MATCH_KEYS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GET_ITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GET_YIELD_FROM_ITER] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_FOR_ITER_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_FOR_ITER_TIER_TWO] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_ITER_CHECK_LIST] = HAS_EXIT_FLAG, [_GUARD_NOT_EXHAUSTED_LIST] = HAS_EXIT_FLAG, + [_ITER_NEXT_LIST] = HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_ITER_NEXT_LIST_TIER_TWO] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_ITER_CHECK_TUPLE] = HAS_EXIT_FLAG, [_GUARD_NOT_EXHAUSTED_TUPLE] = HAS_EXIT_FLAG, @@ -337,6 +341,8 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_ERROR_POP_N] = HAS_ARG_FLAG, [_TIER2_RESUME_CHECK] = HAS_PERIODIC_FLAG, [_COLD_EXIT] = HAS_ESCAPES_FLAG, + [_GUARD_IP] = HAS_ESCAPES_FLAG, + [_DYNAMIC_EXIT] = 0, }; const ReplicationRange _PyUop_Replication[MAX_UOP_ID+1] = { @@ -443,6 +449,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_DEOPT] = "_DEOPT", [_DICT_MERGE] = "_DICT_MERGE", [_DICT_UPDATE] = "_DICT_UPDATE", + [_DYNAMIC_EXIT] = "_DYNAMIC_EXIT", [_END_FOR] = "_END_FOR", [_END_SEND] = "_END_SEND", [_ERROR_POP_N] = "_ERROR_POP_N", @@ -471,6 +478,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_GUARD_DORV_NO_DICT] = "_GUARD_DORV_NO_DICT", [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT", [_GUARD_GLOBALS_VERSION] = "_GUARD_GLOBALS_VERSION", + [_GUARD_IP] = "_GUARD_IP", [_GUARD_IS_FALSE_POP] = "_GUARD_IS_FALSE_POP", [_GUARD_IS_NONE_POP] = "_GUARD_IS_NONE_POP", [_GUARD_IS_NOT_NONE_POP] = "_GUARD_IS_NOT_NONE_POP", @@ -516,9 +524,11 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_ITER_CHECK_LIST] = "_ITER_CHECK_LIST", [_ITER_CHECK_RANGE] = "_ITER_CHECK_RANGE", [_ITER_CHECK_TUPLE] = "_ITER_CHECK_TUPLE", + [_ITER_NEXT_LIST] = "_ITER_NEXT_LIST", [_ITER_NEXT_LIST_TIER_TWO] = "_ITER_NEXT_LIST_TIER_TWO", [_ITER_NEXT_RANGE] = "_ITER_NEXT_RANGE", [_ITER_NEXT_TUPLE] = "_ITER_NEXT_TUPLE", + [_JUMP_BACKWARD_NO_INTERRUPT] = "_JUMP_BACKWARD_NO_INTERRUPT", [_JUMP_TO_TOP] = "_JUMP_TO_TOP", [_LIST_APPEND] = "_LIST_APPEND", [_LIST_EXTEND] = "_LIST_EXTEND", @@ -598,6 +608,8 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW] = "_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW", [_POP_EXCEPT] = "_POP_EXCEPT", [_POP_ITER] = "_POP_ITER", + [_POP_JUMP_IF_FALSE] = "_POP_JUMP_IF_FALSE", + [_POP_JUMP_IF_TRUE] = "_POP_JUMP_IF_TRUE", [_POP_TOP] = "_POP_TOP", [_POP_TOP_FLOAT] = "_POP_TOP_FLOAT", [_POP_TOP_INT] = "_POP_TOP_INT", @@ -1041,8 +1053,14 @@ int _PyUop_num_popped(int opcode, int oparg) return 2; case _IMPORT_FROM: return 0; + case _POP_JUMP_IF_FALSE: + return 1; + case _POP_JUMP_IF_TRUE: + return 1; case _IS_NONE: return 1; + case _JUMP_BACKWARD_NO_INTERRUPT: + return 0; case _GET_LEN: return 0; case _MATCH_CLASS: @@ -1063,6 +1081,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _GUARD_NOT_EXHAUSTED_LIST: return 0; + case _ITER_NEXT_LIST: + return 0; case _ITER_NEXT_LIST_TIER_TWO: return 0; case _ITER_CHECK_TUPLE: @@ -1305,6 +1325,10 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _COLD_EXIT: return 0; + case _GUARD_IP: + return 0; + case _DYNAMIC_EXIT: + return 0; default: return -1; } diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 1d5c74adefcd35..4021dc0ad72e14 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -49,88 +49,90 @@ extern "C" { #define SETUP_ANNOTATIONS 36 #define STORE_SLICE 37 #define STORE_SUBSCR 38 -#define TO_BOOL 39 -#define UNARY_INVERT 40 -#define UNARY_NEGATIVE 41 -#define UNARY_NOT 42 -#define WITH_EXCEPT_START 43 -#define BINARY_OP 44 -#define BUILD_INTERPOLATION 45 -#define BUILD_LIST 46 -#define BUILD_MAP 47 -#define BUILD_SET 48 -#define BUILD_SLICE 49 -#define BUILD_STRING 50 -#define BUILD_TUPLE 51 -#define CALL 52 -#define CALL_INTRINSIC_1 53 -#define CALL_INTRINSIC_2 54 -#define CALL_KW 55 -#define COMPARE_OP 56 -#define CONTAINS_OP 57 -#define CONVERT_VALUE 58 -#define COPY 59 -#define COPY_FREE_VARS 60 -#define DELETE_ATTR 61 -#define DELETE_DEREF 62 -#define DELETE_FAST 63 -#define DELETE_GLOBAL 64 -#define DELETE_NAME 65 -#define DICT_MERGE 66 -#define DICT_UPDATE 67 -#define END_ASYNC_FOR 68 -#define EXTENDED_ARG 69 -#define FOR_ITER 70 -#define GET_AWAITABLE 71 -#define IMPORT_FROM 72 -#define IMPORT_NAME 73 -#define IS_OP 74 -#define JUMP_BACKWARD 75 -#define JUMP_BACKWARD_NO_INTERRUPT 76 -#define JUMP_FORWARD 77 -#define LIST_APPEND 78 -#define LIST_EXTEND 79 -#define LOAD_ATTR 80 -#define LOAD_COMMON_CONSTANT 81 -#define LOAD_CONST 82 -#define LOAD_DEREF 83 -#define LOAD_FAST 84 -#define LOAD_FAST_AND_CLEAR 85 -#define LOAD_FAST_BORROW 86 -#define LOAD_FAST_BORROW_LOAD_FAST_BORROW 87 -#define LOAD_FAST_CHECK 88 -#define LOAD_FAST_LOAD_FAST 89 -#define LOAD_FROM_DICT_OR_DEREF 90 -#define LOAD_FROM_DICT_OR_GLOBALS 91 -#define LOAD_GLOBAL 92 -#define LOAD_NAME 93 -#define LOAD_SMALL_INT 94 -#define LOAD_SPECIAL 95 -#define LOAD_SUPER_ATTR 96 -#define MAKE_CELL 97 -#define MAP_ADD 98 -#define MATCH_CLASS 99 -#define POP_JUMP_IF_FALSE 100 -#define POP_JUMP_IF_NONE 101 -#define POP_JUMP_IF_NOT_NONE 102 -#define POP_JUMP_IF_TRUE 103 -#define RAISE_VARARGS 104 -#define RERAISE 105 -#define SEND 106 -#define SET_ADD 107 -#define SET_FUNCTION_ATTRIBUTE 108 -#define SET_UPDATE 109 -#define STORE_ATTR 110 -#define STORE_DEREF 111 -#define STORE_FAST 112 -#define STORE_FAST_LOAD_FAST 113 -#define STORE_FAST_STORE_FAST 114 -#define STORE_GLOBAL 115 -#define STORE_NAME 116 -#define SWAP 117 -#define UNPACK_EX 118 -#define UNPACK_SEQUENCE 119 -#define YIELD_VALUE 120 +#define TIER1_GUARD_IP 39 +#define TIER1_SET_IP 40 +#define TO_BOOL 41 +#define UNARY_INVERT 42 +#define UNARY_NEGATIVE 43 +#define UNARY_NOT 44 +#define WITH_EXCEPT_START 45 +#define BINARY_OP 46 +#define BUILD_INTERPOLATION 47 +#define BUILD_LIST 48 +#define BUILD_MAP 49 +#define BUILD_SET 50 +#define BUILD_SLICE 51 +#define BUILD_STRING 52 +#define BUILD_TUPLE 53 +#define CALL 54 +#define CALL_INTRINSIC_1 55 +#define CALL_INTRINSIC_2 56 +#define CALL_KW 57 +#define COMPARE_OP 58 +#define CONTAINS_OP 59 +#define CONVERT_VALUE 60 +#define COPY 61 +#define COPY_FREE_VARS 62 +#define DELETE_ATTR 63 +#define DELETE_DEREF 64 +#define DELETE_FAST 65 +#define DELETE_GLOBAL 66 +#define DELETE_NAME 67 +#define DICT_MERGE 68 +#define DICT_UPDATE 69 +#define END_ASYNC_FOR 70 +#define EXTENDED_ARG 71 +#define FOR_ITER 72 +#define GET_AWAITABLE 73 +#define IMPORT_FROM 74 +#define IMPORT_NAME 75 +#define IS_OP 76 +#define JUMP_BACKWARD 77 +#define JUMP_BACKWARD_NO_INTERRUPT 78 +#define JUMP_FORWARD 79 +#define LIST_APPEND 80 +#define LIST_EXTEND 81 +#define LOAD_ATTR 82 +#define LOAD_COMMON_CONSTANT 83 +#define LOAD_CONST 84 +#define LOAD_DEREF 85 +#define LOAD_FAST 86 +#define LOAD_FAST_AND_CLEAR 87 +#define LOAD_FAST_BORROW 88 +#define LOAD_FAST_BORROW_LOAD_FAST_BORROW 89 +#define LOAD_FAST_CHECK 90 +#define LOAD_FAST_LOAD_FAST 91 +#define LOAD_FROM_DICT_OR_DEREF 92 +#define LOAD_FROM_DICT_OR_GLOBALS 93 +#define LOAD_GLOBAL 94 +#define LOAD_NAME 95 +#define LOAD_SMALL_INT 96 +#define LOAD_SPECIAL 97 +#define LOAD_SUPER_ATTR 98 +#define MAKE_CELL 99 +#define MAP_ADD 100 +#define MATCH_CLASS 101 +#define POP_JUMP_IF_FALSE 102 +#define POP_JUMP_IF_NONE 103 +#define POP_JUMP_IF_NOT_NONE 104 +#define POP_JUMP_IF_TRUE 105 +#define RAISE_VARARGS 106 +#define RERAISE 107 +#define SEND 108 +#define SET_ADD 109 +#define SET_FUNCTION_ATTRIBUTE 110 +#define SET_UPDATE 111 +#define STORE_ATTR 112 +#define STORE_DEREF 113 +#define STORE_FAST 114 +#define STORE_FAST_LOAD_FAST 115 +#define STORE_FAST_STORE_FAST 116 +#define STORE_GLOBAL 117 +#define STORE_NAME 118 +#define SWAP 119 +#define UNPACK_EX 120 +#define UNPACK_SEQUENCE 121 +#define YIELD_VALUE 122 #define RESUME 128 #define BINARY_OP_ADD_FLOAT 129 #define BINARY_OP_ADD_INT 130 @@ -247,7 +249,7 @@ extern "C" { #define SETUP_WITH 265 #define STORE_FAST_MAYBE_NULL 266 -#define HAVE_ARGUMENT 43 +#define HAVE_ARGUMENT 45 #define MIN_SPECIALIZED_OPCODE 129 #define MIN_INSTRUMENTED_OPCODE 234 diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index f168d169a32948..ca7fc579b3aadd 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -246,88 +246,90 @@ 'SETUP_ANNOTATIONS': 36, 'STORE_SLICE': 37, 'STORE_SUBSCR': 38, - 'TO_BOOL': 39, - 'UNARY_INVERT': 40, - 'UNARY_NEGATIVE': 41, - 'UNARY_NOT': 42, - 'WITH_EXCEPT_START': 43, - 'BINARY_OP': 44, - 'BUILD_INTERPOLATION': 45, - 'BUILD_LIST': 46, - 'BUILD_MAP': 47, - 'BUILD_SET': 48, - 'BUILD_SLICE': 49, - 'BUILD_STRING': 50, - 'BUILD_TUPLE': 51, - 'CALL': 52, - 'CALL_INTRINSIC_1': 53, - 'CALL_INTRINSIC_2': 54, - 'CALL_KW': 55, - 'COMPARE_OP': 56, - 'CONTAINS_OP': 57, - 'CONVERT_VALUE': 58, - 'COPY': 59, - 'COPY_FREE_VARS': 60, - 'DELETE_ATTR': 61, - 'DELETE_DEREF': 62, - 'DELETE_FAST': 63, - 'DELETE_GLOBAL': 64, - 'DELETE_NAME': 65, - 'DICT_MERGE': 66, - 'DICT_UPDATE': 67, - 'END_ASYNC_FOR': 68, - 'EXTENDED_ARG': 69, - 'FOR_ITER': 70, - 'GET_AWAITABLE': 71, - 'IMPORT_FROM': 72, - 'IMPORT_NAME': 73, - 'IS_OP': 74, - 'JUMP_BACKWARD': 75, - 'JUMP_BACKWARD_NO_INTERRUPT': 76, - 'JUMP_FORWARD': 77, - 'LIST_APPEND': 78, - 'LIST_EXTEND': 79, - 'LOAD_ATTR': 80, - 'LOAD_COMMON_CONSTANT': 81, - 'LOAD_CONST': 82, - 'LOAD_DEREF': 83, - 'LOAD_FAST': 84, - 'LOAD_FAST_AND_CLEAR': 85, - 'LOAD_FAST_BORROW': 86, - 'LOAD_FAST_BORROW_LOAD_FAST_BORROW': 87, - 'LOAD_FAST_CHECK': 88, - 'LOAD_FAST_LOAD_FAST': 89, - 'LOAD_FROM_DICT_OR_DEREF': 90, - 'LOAD_FROM_DICT_OR_GLOBALS': 91, - 'LOAD_GLOBAL': 92, - 'LOAD_NAME': 93, - 'LOAD_SMALL_INT': 94, - 'LOAD_SPECIAL': 95, - 'LOAD_SUPER_ATTR': 96, - 'MAKE_CELL': 97, - 'MAP_ADD': 98, - 'MATCH_CLASS': 99, - 'POP_JUMP_IF_FALSE': 100, - 'POP_JUMP_IF_NONE': 101, - 'POP_JUMP_IF_NOT_NONE': 102, - 'POP_JUMP_IF_TRUE': 103, - 'RAISE_VARARGS': 104, - 'RERAISE': 105, - 'SEND': 106, - 'SET_ADD': 107, - 'SET_FUNCTION_ATTRIBUTE': 108, - 'SET_UPDATE': 109, - 'STORE_ATTR': 110, - 'STORE_DEREF': 111, - 'STORE_FAST': 112, - 'STORE_FAST_LOAD_FAST': 113, - 'STORE_FAST_STORE_FAST': 114, - 'STORE_GLOBAL': 115, - 'STORE_NAME': 116, - 'SWAP': 117, - 'UNPACK_EX': 118, - 'UNPACK_SEQUENCE': 119, - 'YIELD_VALUE': 120, + 'TIER1_GUARD_IP': 39, + 'TIER1_SET_IP': 40, + 'TO_BOOL': 41, + 'UNARY_INVERT': 42, + 'UNARY_NEGATIVE': 43, + 'UNARY_NOT': 44, + 'WITH_EXCEPT_START': 45, + 'BINARY_OP': 46, + 'BUILD_INTERPOLATION': 47, + 'BUILD_LIST': 48, + 'BUILD_MAP': 49, + 'BUILD_SET': 50, + 'BUILD_SLICE': 51, + 'BUILD_STRING': 52, + 'BUILD_TUPLE': 53, + 'CALL': 54, + 'CALL_INTRINSIC_1': 55, + 'CALL_INTRINSIC_2': 56, + 'CALL_KW': 57, + 'COMPARE_OP': 58, + 'CONTAINS_OP': 59, + 'CONVERT_VALUE': 60, + 'COPY': 61, + 'COPY_FREE_VARS': 62, + 'DELETE_ATTR': 63, + 'DELETE_DEREF': 64, + 'DELETE_FAST': 65, + 'DELETE_GLOBAL': 66, + 'DELETE_NAME': 67, + 'DICT_MERGE': 68, + 'DICT_UPDATE': 69, + 'END_ASYNC_FOR': 70, + 'EXTENDED_ARG': 71, + 'FOR_ITER': 72, + 'GET_AWAITABLE': 73, + 'IMPORT_FROM': 74, + 'IMPORT_NAME': 75, + 'IS_OP': 76, + 'JUMP_BACKWARD': 77, + 'JUMP_BACKWARD_NO_INTERRUPT': 78, + 'JUMP_FORWARD': 79, + 'LIST_APPEND': 80, + 'LIST_EXTEND': 81, + 'LOAD_ATTR': 82, + 'LOAD_COMMON_CONSTANT': 83, + 'LOAD_CONST': 84, + 'LOAD_DEREF': 85, + 'LOAD_FAST': 86, + 'LOAD_FAST_AND_CLEAR': 87, + 'LOAD_FAST_BORROW': 88, + 'LOAD_FAST_BORROW_LOAD_FAST_BORROW': 89, + 'LOAD_FAST_CHECK': 90, + 'LOAD_FAST_LOAD_FAST': 91, + 'LOAD_FROM_DICT_OR_DEREF': 92, + 'LOAD_FROM_DICT_OR_GLOBALS': 93, + 'LOAD_GLOBAL': 94, + 'LOAD_NAME': 95, + 'LOAD_SMALL_INT': 96, + 'LOAD_SPECIAL': 97, + 'LOAD_SUPER_ATTR': 98, + 'MAKE_CELL': 99, + 'MAP_ADD': 100, + 'MATCH_CLASS': 101, + 'POP_JUMP_IF_FALSE': 102, + 'POP_JUMP_IF_NONE': 103, + 'POP_JUMP_IF_NOT_NONE': 104, + 'POP_JUMP_IF_TRUE': 105, + 'RAISE_VARARGS': 106, + 'RERAISE': 107, + 'SEND': 108, + 'SET_ADD': 109, + 'SET_FUNCTION_ATTRIBUTE': 110, + 'SET_UPDATE': 111, + 'STORE_ATTR': 112, + 'STORE_DEREF': 113, + 'STORE_FAST': 114, + 'STORE_FAST_LOAD_FAST': 115, + 'STORE_FAST_STORE_FAST': 116, + 'STORE_GLOBAL': 117, + 'STORE_NAME': 118, + 'SWAP': 119, + 'UNPACK_EX': 120, + 'UNPACK_SEQUENCE': 121, + 'YIELD_VALUE': 122, 'INSTRUMENTED_END_FOR': 234, 'INSTRUMENTED_POP_ITER': 235, 'INSTRUMENTED_END_SEND': 236, @@ -361,5 +363,5 @@ 'STORE_FAST_MAYBE_NULL': 266, } -HAVE_ARGUMENT = 43 +HAVE_ARGUMENT = 45 MIN_INSTRUMENTED_OPCODE = 234 diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 99b3bc849b6040..a495e332d97886 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -45,6 +45,8 @@ #define USE_COMPUTED_GOTOS 0 #include "ceval_macros.h" +#include "../Include/internal/pycore_code.h" +#include "../Include/internal/pycore_stackref.h" /* Flow control macros */ @@ -1250,6 +1252,9 @@ dummy_func( _PyEval_FrameClearAndPop(tstate, dying); RELOAD_STACK(); LOAD_IP(frame->return_offset); +#if TIER_TWO + frame->instr_ptr += frame->return_offset; +#endif res = temp; LLTRACE_RESUME_FRAME(); } @@ -1437,6 +1442,9 @@ dummy_func( #endif RELOAD_STACK(); LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); +#if TIER_TWO + frame->instr_ptr += (1 + INLINE_CACHE_ENTRIES_SEND); +#endif value = PyStackRef_MakeHeapSafe(temp); LLTRACE_RESUME_FRAME(); } @@ -2942,7 +2950,7 @@ dummy_func( tier1 op(_SPECIALIZE_JUMP_BACKWARD, (--)) { #if ENABLE_SPECIALIZATION if (this_instr->op.code == JUMP_BACKWARD) { - this_instr->op.code = tstate->interp->jit ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; + this_instr->op.code = (tstate->interp->jit && !PyStackRef_IsNull(frame->f_funcobj)) ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; // Need to re-dispatch so the warmup counter isn't off by one: next_instr = this_instr; DISPATCH_SAME_OPARG(); @@ -2953,32 +2961,34 @@ dummy_func( tier1 op(_JIT, (--)) { #ifdef _Py_TIER2 _Py_BackoffCounter counter = this_instr[1].counter; - if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD_JIT) { + if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD_JIT) { _Py_CODEUNIT *start = this_instr; /* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */ - while (oparg > 255) { - oparg >>= 8; + int curr_oparg = oparg; + while (curr_oparg > 255) { + curr_oparg >>= 8; start--; } if (tstate->interp->jit_tracer_code_buffer == NULL) { tstate->interp->jit_tracer_code_buffer = (_Py_CODEUNIT *)_PyObject_VirtualAlloc(TRACER_BUFFER_SIZE); - tstate->interp->jit_tracer_code_curr_size = 0; if (tstate->interp->jit_tracer_code_buffer == NULL) { // Don't error, just go to next instruction. DISPATCH(); } - } - if (this_instr == tstate->interp->jit_tracer_initial_instr) { - // Looped back to initial instr. End tracing. - LEAVE_TRACING(); - DISPATCH(); + tstate->interp->jit_tracer_code_curr_size = 0; } ENTER_TRACING(); // Nothing in the buffer, begin tracing! if (tstate->interp->jit_tracer_code_curr_size == 0) { - tstate->interp->jit_tracer_initial_instr = this_instr; + tstate->interp->jit_tracer_initial_instr = next_instr; + tstate->interp->jit_tracer_initial_code = _PyFrame_GetCode(frame); + tstate->interp->jit_tracer_initial_func = _PyFrame_GetFunction(frame); + tstate->interp->jit_tracer_seen_initial_before = 0; + tstate->interp->jit_completed_loop = false; + tstate->interp->jit_tracer_initial_stack_depth = (int)STACK_LEVEL(); + tstate->interp->jit_tracer_initial_chain_depth = 0; } - // Not tracing dispatch, normal dispatch because we don't record the current instruction. + // Don't add the JUMP_BACKWARD_JIT instruction to the trace. DISPATCH(); } else { @@ -2991,17 +3001,17 @@ dummy_func( unused/1 + _SPECIALIZE_JUMP_BACKWARD + _CHECK_PERIODIC + - JUMP_BACKWARD_NO_INTERRUPT; + _JUMP_BACKWARD_NO_INTERRUPT; macro(JUMP_BACKWARD_NO_JIT) = unused/1 + _CHECK_PERIODIC + - JUMP_BACKWARD_NO_INTERRUPT; + _JUMP_BACKWARD_NO_INTERRUPT; macro(JUMP_BACKWARD_JIT) = unused/1 + _CHECK_PERIODIC + - JUMP_BACKWARD_NO_INTERRUPT + + _JUMP_BACKWARD_NO_INTERRUPT + _JIT; pseudo(JUMP, (--)) = { @@ -3033,7 +3043,7 @@ dummy_func( /* If the eval breaker is set then stay in tier 1. * This avoids any potentially infinite loops * involving _RESUME_CHECK */ - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + if (IS_JIT_TRACING() || _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { opcode = executor->vm_data.opcode; oparg = (oparg & ~255) | executor->vm_data.oparg; next_instr = this_instr; @@ -3050,20 +3060,18 @@ dummy_func( #endif /* _Py_TIER2 */ } - replaced op(_POP_JUMP_IF_FALSE, (cond -- )) { + op(_POP_JUMP_IF_FALSE, (cond -- )) { assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsFalse(cond); DEAD(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + JUMPBY(flag ? oparg : 0); } - replaced op(_POP_JUMP_IF_TRUE, (cond -- )) { + op(_POP_JUMP_IF_TRUE, (cond -- )) { assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsTrue(cond); DEAD(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + JUMPBY(flag ? oparg : 0); } op(_IS_NONE, (value -- b)) { @@ -3085,13 +3093,18 @@ dummy_func( macro(POP_JUMP_IF_NOT_NONE) = unused/1 + _IS_NONE + _POP_JUMP_IF_FALSE; - tier1 inst(JUMP_BACKWARD_NO_INTERRUPT, (--)) { + // This actually has 1 cache entry, but for some reason it's not factored in the oparg. + macro(JUMP_BACKWARD_NO_INTERRUPT) = _JUMP_BACKWARD_NO_INTERRUPT; + + op(_JUMP_BACKWARD_NO_INTERRUPT, (--)) { /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. * (see bpo-30039). */ +#if TIER_ONE assert(oparg <= INSTR_OFFSET()); +#endif JUMPBY(-oparg); } @@ -3237,7 +3250,8 @@ dummy_func( ERROR_NO_POP(); } /* iterator ended normally */ - /* The translator sets the deopt target just past the matching END_FOR */ + // Jump forward by oparg and skip the following END_FOR + TIER2_JUMPBY(oparg + 1); EXIT_IF(true); } next = item; @@ -3298,7 +3312,7 @@ dummy_func( #endif } - replaced op(_ITER_NEXT_LIST, (iter, null_or_index -- iter, null_or_index, next)) { + op(_ITER_NEXT_LIST, (iter, null_or_index -- iter, null_or_index, next)) { PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); assert(PyList_CheckExact(list_o)); #ifdef Py_GIL_DISABLED @@ -4986,6 +5000,9 @@ dummy_func( _PyThreadState_PopFrame(tstate, frame); frame = tstate->current_frame = prev; LOAD_IP(frame->return_offset); +#if TIER_TWO + frame->instr_ptr += (frame->return_offset); +#endif RELOAD_STACK(); res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); LLTRACE_RESUME_FRAME(); @@ -5380,19 +5397,19 @@ dummy_func( } tier2 op(_DEOPT, (--)) { - GOTO_TIER_ONE(_PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + GOTO_TIER_ONE(_PyFrame_GetBytecode(frame) + CURRENT_TARGET(), 0); } tier2 op(_HANDLE_PENDING_AND_DEOPT, (--)) { int err = _Py_HandlePending(tstate); - GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET(), 0); } tier2 op(_ERROR_POP_N, (target/2 --)) { assert(oparg == 0); frame->instr_ptr = _PyFrame_GetBytecode(frame) + target; SYNC_SP(); - GOTO_TIER_ONE(NULL); + GOTO_TIER_ONE(NULL, 0); } /* Progress is guaranteed if we DEOPT on the eval breaker, because @@ -5414,7 +5431,7 @@ dummy_func( _Py_BackoffCounter temperature = exit->temperature; if (!backoff_counter_triggers(temperature)) { exit->temperature = advance_backoff_counter(temperature); - GOTO_TIER_ONE(target); + GOTO_TIER_ONE(target, 0); } _PyExecutorObject *executor; if (target->op.code == ENTER_EXECUTOR) { @@ -5426,18 +5443,41 @@ dummy_func( _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); assert(tstate->current_executor == (PyObject *)previous_executor); int chain_depth = previous_executor->vm_data.chain_depth + 1; - int optimized = _PyOptimizer_Optimize(frame, target, &executor, chain_depth); - if (optimized <= 0) { - exit->temperature = restart_backoff_counter(temperature); - GOTO_TIER_ONE(optimized < 0 ? NULL : target); - } - exit->temperature = initial_temperature_backoff_counter(); + tstate->interp->jit_tracer_initial_chain_depth = chain_depth; + tstate->interp->jit_tracer_initial_instr = target; + tstate->interp->jit_tracer_initial_code = _PyFrame_GetCode(frame); + tstate->interp->jit_tracer_initial_func = _PyFrame_GetFunction(frame); + tstate->interp->jit_tracer_seen_initial_before = 0; + tstate->interp->jit_completed_loop = false; + exit->temperature = restart_backoff_counter(temperature); + GOTO_TIER_ONE(target, 1); } assert(tstate->jit_exit == exit); exit->executor = executor; TIER2_TO_TIER2(exit->executor); } + + no_save_ip tier1 inst(TIER1_GUARD_IP, (func_ptr/4 --)) { + (void)(func_ptr); + } + + tier2 op(_GUARD_IP, (ip/4 --)) { + if (frame->instr_ptr != (_Py_CODEUNIT *)ip) { + fprintf(stdout, "d:%p:%p\n", frame->instr_ptr, ip); + GOTO_TIER_ONE(frame->instr_ptr, 1); + } + } + + tier2 op(_DYNAMIC_EXIT, (ip/4 --)) { + GOTO_TIER_ONE(frame->instr_ptr, 1); + } + + tier1 inst(TIER1_SET_IP, (ip/4 --)) { + (void)(ip); + } + + label(pop_2_error) { stack_pointer -= 2; assert(WITHIN_STACK_BOUNDS()); diff --git a/Python/ceval.c b/Python/ceval.c index 453ec103484b5a..4378afebc6ab2e 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -974,18 +974,99 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch) } // 1 for trace full, 0 for successful write. -static inline int -add_to_code_trace(PyThreadState *tstate, _Py_CODEUNIT *this_instr) +static int +add_to_code_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *this_instr, int oparg) { + assert(frame != NULL); assert(tstate->interp->jit_tracer_code_curr_size < TRACE_MAX_TRACE_LENGTH); int curr_size = tstate->interp->jit_tracer_code_curr_size; - int nsize = _PyOpcode_Caches[this_instr->op.code] + 1; - if (curr_size + nsize > TRACE_MAX_TRACE_LENGTH) { + int opcode = this_instr->op.code; + if (opcode == ENTER_EXECUTOR) { return 1; + // PyCodeObject *code = _PyFrame_GetCode(frame); + // assert(code != NULL); + // _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; + // assert(executor->vm_data.code == code); + // assert(executor->vm_data.valid); + // assert(tstate->current_executor == NULL); + // opcode = executor->vm_data.opcode; + // oparg = (oparg & ~255) | executor->vm_data.oparg; + } + else { + oparg = this_instr->op.arg; + } + assert(opcode != 0); + // Check if we looped back to the start. + if (this_instr == tstate->interp->jit_tracer_initial_instr) { + if (tstate->interp->jit_tracer_seen_initial_before >= 1) { + tstate->interp->jit_completed_loop = true; + return 1; + } + tstate->interp->jit_tracer_seen_initial_before++; + } +#ifdef Py_DEBUG + char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); + int lltrace = 3; + if (python_lltrace != NULL && *python_lltrace >= '0') { + lltrace = *python_lltrace - '0'; } - for (int i = 0; i < nsize; i++) { + if (lltrace >= 3) { + printf("%d ADD_TO_BYTECODE_TRACE: %s %d\n", curr_size, _PyOpcode_OpName[opcode], oparg); + } +#endif + // JUMP_BACKWARD_NO_INTERRUPT is not registered with cache due to a bug in the compiler. + int caches = opcode == JUMP_BACKWARD_NO_INTERRUPT ? 1 : _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; + int nsize = caches + 1; + int needs_guard_ip = _PyOpcode_NeedsGuardIp[opcode]; + int total_size = nsize + (needs_guard_ip ? 5 : 0) + 5; + if (curr_size + total_size >= TRACE_MAX_TRACE_LENGTH) { +#ifdef Py_DEBUG + if (lltrace >= 3) { + printf("END TRACE LENGTH: %d\n", curr_size); + } +#endif + return 1; + } + tstate->interp->jit_tracer_code_buffer[curr_size].op.code = opcode; + tstate->interp->jit_tracer_code_buffer[curr_size].op.arg = oparg; + for (int i = 1; i < nsize; i++) { tstate->interp->jit_tracer_code_buffer[curr_size + i] = *(this_instr + i); } + if (needs_guard_ip) { +#ifdef Py_DEBUG + if (lltrace >= 3) { + printf("%d ADD_TO_BYTECODE_TRACE: %s %d\n", curr_size + nsize, _PyOpcode_OpName[TIER1_GUARD_IP], 0); + } +#endif + tstate->interp->jit_tracer_code_buffer[curr_size + nsize].op.code = TIER1_GUARD_IP; + PyObject *func = PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + if (Py_IsNone(func)) { + func = NULL; // Trampoline frames don't set their func object field. + } + assert(func == NULL || PyFunction_Check(func)); + write_ptr(&tstate->interp->jit_tracer_code_buffer[curr_size + nsize + 1].cache, func); + +#ifdef Py_DEBUG + if (lltrace >= 3) { + printf("%d ADD_TO_BYTECODE_TRACE: %s %p\n", curr_size + nsize + 5, _PyOpcode_OpName[TIER1_SET_IP], frame->instr_ptr); + } +#endif + tstate->interp->jit_tracer_code_buffer[curr_size + nsize + 5].op.code = TIER1_SET_IP; + write_ptr(&tstate->interp->jit_tracer_code_buffer[curr_size + nsize + 6].cache, frame->instr_ptr); + + } + else { +#ifdef Py_DEBUG + if (lltrace >= 3) { + printf("%d ADD_TO_BYTECODE_TRACE: %s %p\n", curr_size + nsize, _PyOpcode_OpName[TIER1_SET_IP], frame->instr_ptr); + } +#endif + tstate->interp->jit_tracer_code_buffer[curr_size + nsize].op.code = TIER1_SET_IP; + write_ptr(&tstate->interp->jit_tracer_code_buffer[curr_size + nsize + 1].cache, frame->instr_ptr); + + } + + tstate->interp->jit_tracer_code_curr_size += total_size; return 0; } @@ -1061,11 +1142,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int _Py_CODEUNIT *next_instr; _PyStackRef *stack_pointer; entry.stack[0] = PyStackRef_NULL; + entry.frame.f_funcobj = PyStackRef_None; #ifdef Py_STACKREF_DEBUG entry.frame.f_funcobj = PyStackRef_None; #elif defined(Py_DEBUG) /* Set these to invalid but identifiable values for debugging. */ - entry.frame.f_funcobj = (_PyStackRef){.bits = 0xaaa0}; entry.frame.f_locals = (PyObject*)0xaaa1; entry.frame.frame_obj = (PyFrameObject*)0xaaa2; entry.frame.f_globals = (PyObject*)0xaaa3; @@ -1199,10 +1280,11 @@ _PyTier2Interpreter( assert(next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT); tier2_dispatch: for (;;) { + frame->lltrace = 4; uopcode = next_uop->opcode; #ifdef Py_DEBUG - if (frame->lltrace >= 3) { - dump_stack(frame, stack_pointer); + if (1 && next_uop->opcode != _YIELD_VALUE) { + // dump_stack(frame, stack_pointer); if (next_uop->opcode == _START_EXECUTOR) { printf("%4d uop: ", 0); } diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 1afe5d4003792b..c0f33dd5b31f9c 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -78,12 +78,6 @@ # define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, instruction_funcptr_table, oparg #endif -# define TRACING_DISPATCH_GOTO() do { \ - if (add_to_code_trace(tstate, this_instr)) { \ - LEAVE_TRACING(); \ - } \ - DISPATCH_GOTO(); \ - } while (0); #if _Py_TAIL_CALL_INTERP // Note: [[clang::musttail]] works for GCC 15, but not __attribute__((musttail)) at the moment. @@ -91,10 +85,12 @@ # define Py_PRESERVE_NONE_CC __attribute__((preserve_none)) Py_PRESERVE_NONE_CC typedef PyObject* (*py_tail_call_funcptr)(TAIL_CALL_PARAMS); +# define DISPATCH_TABLE_VAR instruction_funcptr_table +# define DISPATCH_TABLE instruction_funcptr_handler_table +# define TRACING_DISPATCH_TABLE instruction_funcptr_tracing_table # define TARGET(op) Py_PRESERVE_NONE_CC PyObject *_TAIL_CALL_##op(TAIL_CALL_PARAMS) # define TRACING_TARGET(op) Py_PRESERVE_NONE_CC PyObject *_TAIL_CALL_TRACING_##op(TAIL_CALL_PARAMS) -# define LEAVE_TRACING() instruction_funcptr_table = instruction_funcptr_handler_table; -# define ENTER_TRACING() instruction_funcptr_table = instruction_funcptr_tracing_table + # define DISPATCH_GOTO() \ do { \ Py_MUSTTAIL return (((py_tail_call_funcptr *)instruction_funcptr_table)[opcode])(TAIL_CALL_ARGS); \ @@ -116,11 +112,13 @@ # endif # define LABEL(name) TARGET(name) #elif USE_COMPUTED_GOTOS +# define DISPATCH_TABLE_VAR opcode_targets +# define DISPATCH_TABLE opcode_targets_table +# define TRACING_DISPATCH_TABLE opcode_tracing_targets_table # define TARGET(op) TARGET_##op: # define TRACING_TARGET(op) TARGET_TRACING_##op: # define DISPATCH_GOTO() goto *opcode_targets[opcode] -# define LEAVE_TRACING() opcode_targets = opcode_targets_table; -# define ENTER_TRACING() opcode_targets = opcode_tracing_targets_table; + # define JUMP_TO_LABEL(name) goto name; # define JUMP_TO_PREDICTED(name) goto PREDICTED_##name; # define LABEL(name) name: @@ -132,6 +130,27 @@ # define LABEL(name) name: #endif +#if _Py_TAIL_CALL_INTERP || USE_COMPUTED_GOTOS +# define IS_JIT_TRACING() (DISPATCH_TABLE_VAR == TRACING_DISPATCH_TABLE) +# define ENTER_TRACING() DISPATCH_TABLE_VAR = TRACING_DISPATCH_TABLE; +# define LEAVE_TRACING() DISPATCH_TABLE_VAR = DISPATCH_TABLE; +# define BAIL_TRACING() \ + LEAVE_TRACING(); \ + int err = _PyOptimizer_Optimize(frame, tstate); \ + tstate->interp->jit_tracer_code_curr_size = 0; \ + if (err < 0) { \ + JUMP_TO_LABEL(error); \ + } \ + DISPATCH(); +# define RECORD_TRACE() do { \ + frame->instr_ptr = next_instr; \ + if (add_to_code_trace(tstate, frame, this_instr, oparg)) { \ + BAIL_TRACING(); \ + } \ + } while (0); +#endif + + /* PRE_DISPATCH_GOTO() does lltrace if enabled. Normally a no-op */ #ifdef Py_DEBUG #define PRE_DISPATCH_GOTO() if (frame->lltrace >= 5) { \ @@ -188,9 +207,10 @@ do { \ #define TRACING_DISPATCH() \ { \ assert(frame->stackpointer == NULL); \ + RECORD_TRACE(); \ NEXTOPARG(); \ PRE_DISPATCH_GOTO(); \ - TRACING_DISPATCH_GOTO(); \ + DISPATCH_GOTO(); \ } /* Tuple access macros */ @@ -222,6 +242,7 @@ GETITEM(PyObject *v, Py_ssize_t i) { * and skipped instructions. */ #define JUMPBY(x) (next_instr += (x)) +#define TIER2_JUMPBY(x) (frame->instr_ptr += (x)) #define SKIP_OVER(x) (next_instr += (x)) #define STACK_LEVEL() ((int)(stack_pointer - _PyFrame_Stackbase(frame))) @@ -378,10 +399,18 @@ do { \ next_instr = _Py_jit_entry((EXECUTOR), frame, stack_pointer, tstate); \ frame = tstate->current_frame; \ stack_pointer = _PyFrame_GetStackPointer(frame); \ + int keep_tracing_bit = (uintptr_t)next_instr & 1; \ + next_instr = (_Py_CODEUNIT *)(((uintptr_t)next_instr) >> 1 << 1); \ if (next_instr == NULL) { \ next_instr = frame->instr_ptr; \ JUMP_TO_LABEL(error); \ } \ + if (keep_tracing_bit) { \ + ENTER_TRACING(); \ + } \ + else { \ + LEAVE_TRACING(); \ + } \ DISPATCH(); \ } while (0) @@ -392,13 +421,13 @@ do { \ goto tier2_start; \ } while (0) -#define GOTO_TIER_ONE(TARGET) \ +#define GOTO_TIER_ONE(TARGET, SHOULD_CONTINUE_TRACING) \ do \ { \ tstate->current_executor = NULL; \ OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); \ _PyFrame_SetStackPointer(frame, stack_pointer); \ - return TARGET; \ + return (_Py_CODEUNIT *)(((uintptr_t)(TARGET)) | SHOULD_CONTINUE_TRACING); \ } while (0) #define CURRENT_OPARG() (next_uop[-1].oparg) diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 0e4d86463761a0..0b5aee504012cb 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1931,6 +1931,9 @@ _PyEval_FrameClearAndPop(tstate, dying); stack_pointer = _PyFrame_GetStackPointer(frame); LOAD_IP(frame->return_offset); + #if TIER_TWO + frame->instr_ptr += frame->return_offset; + #endif res = temp; LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; @@ -2097,6 +2100,9 @@ #endif stack_pointer = _PyFrame_GetStackPointer(frame); LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + #if TIER_TWO + frame->instr_ptr += (1 + INLINE_CACHE_ENTRIES_SEND); + #endif value = PyStackRef_MakeHeapSafe(temp); LLTRACE_RESUME_FRAME(); stack_pointer[0] = value; @@ -4165,9 +4171,31 @@ break; } - /* _POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 because it is replaced */ + case _POP_JUMP_IF_FALSE: { + TIER2_JUMPBY(2); + _PyStackRef cond; + oparg = CURRENT_OPARG(); + cond = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsFalse(cond); + TIER2_JUMPBY(flag ? oparg : 0); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } - /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 because it is replaced */ + case _POP_JUMP_IF_TRUE: { + TIER2_JUMPBY(2); + _PyStackRef cond; + oparg = CURRENT_OPARG(); + cond = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsTrue(cond); + TIER2_JUMPBY(flag ? oparg : 0); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } case _IS_NONE: { _PyStackRef value; @@ -4189,6 +4217,16 @@ break; } + case _JUMP_BACKWARD_NO_INTERRUPT: { + TIER2_JUMPBY(2); + oparg = CURRENT_OPARG(); + #if TIER_ONE + assert(oparg <= INSTR_OFFSET()); + #endif + TIER2_JUMPBY(-oparg); + break; + } + case _GET_LEN: { _PyStackRef obj; _PyStackRef len; @@ -4383,6 +4421,7 @@ _PyStackRef null_or_index; _PyStackRef iter; _PyStackRef next; + oparg = CURRENT_OPARG(); null_or_index = stack_pointer[-1]; iter = stack_pointer[-2]; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -4392,6 +4431,7 @@ if (PyStackRef_IsError(item)) { JUMP_TO_ERROR(); } + TIER2_JUMPBY(oparg + 1); if (true) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -4445,7 +4485,43 @@ break; } - /* _ITER_NEXT_LIST is not a viable micro-op for tier 2 because it is replaced */ + case _ITER_NEXT_LIST: { + TIER2_JUMPBY(2); + _PyStackRef null_or_index; + _PyStackRef iter; + _PyStackRef next; + oparg = CURRENT_OPARG(); + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(PyList_CheckExact(list_o)); + #ifdef Py_GIL_DISABLED + assert(_Py_IsOwnedByCurrentThread(list_o) || + _PyObject_GC_IS_SHARED(list_o)); + STAT_INC(FOR_ITER, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (result < 0) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (result == 0) { + null_or_index = PyStackRef_TagInt(-1); + stack_pointer[-1] = null_or_index; + TIER2_JUMPBY(oparg + 1); + break; + } + #else + next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index))); + #endif + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); + stack_pointer[-1] = null_or_index; + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } case _ITER_NEXT_LIST_TIER_TWO: { _PyStackRef null_or_index; @@ -6749,6 +6825,9 @@ _PyThreadState_PopFrame(tstate, frame); frame = tstate->current_frame = prev; LOAD_IP(frame->return_offset); + #if TIER_TWO + frame->instr_ptr += (frame->return_offset); + #endif stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); LLTRACE_RESUME_FRAME(); @@ -7422,7 +7501,7 @@ } case _DEOPT: { - GOTO_TIER_ONE(_PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + GOTO_TIER_ONE(_PyFrame_GetBytecode(frame) + CURRENT_TARGET(), 0); break; } @@ -7430,7 +7509,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET(), 0); break; } @@ -7439,7 +7518,7 @@ uint32_t target = (uint32_t)CURRENT_OPERAND0(); assert(oparg == 0); frame->instr_ptr = _PyFrame_GetBytecode(frame) + target; - GOTO_TIER_ONE(NULL); + GOTO_TIER_ONE(NULL, 0); break; } @@ -7467,7 +7546,7 @@ _Py_BackoffCounter temperature = exit->temperature; if (!backoff_counter_triggers(temperature)) { exit->temperature = advance_backoff_counter(temperature); - GOTO_TIER_ONE(target); + GOTO_TIER_ONE(target, 0); } _PyExecutorObject *executor; if (target->op.code == ENTER_EXECUTOR) { @@ -7481,14 +7560,16 @@ stack_pointer = _PyFrame_GetStackPointer(frame); assert(tstate->current_executor == (PyObject *)previous_executor); int chain_depth = previous_executor->vm_data.chain_depth + 1; + tstate->interp->jit_tracer_initial_chain_depth = chain_depth; + tstate->interp->jit_tracer_initial_instr = target; + tstate->interp->jit_tracer_initial_code = _PyFrame_GetCode(frame); _PyFrame_SetStackPointer(frame, stack_pointer); - int optimized = _PyOptimizer_Optimize(frame, target, &executor, chain_depth); + tstate->interp->jit_tracer_initial_func = _PyFrame_GetFunction(frame); stack_pointer = _PyFrame_GetStackPointer(frame); - if (optimized <= 0) { - exit->temperature = restart_backoff_counter(temperature); - GOTO_TIER_ONE(optimized < 0 ? NULL : target); - } - exit->temperature = initial_temperature_backoff_counter(); + tstate->interp->jit_tracer_seen_initial_before = 0; + tstate->interp->jit_completed_loop = false; + exit->temperature = restart_backoff_counter(temperature); + GOTO_TIER_ONE(target, 1); } assert(tstate->jit_exit == exit); exit->executor = executor; @@ -7496,4 +7577,21 @@ break; } + case _GUARD_IP: { + PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + if (frame->instr_ptr != (_Py_CODEUNIT *)ip) { + _PyFrame_SetStackPointer(frame, stack_pointer); + fprintf(stdout, "d:%p:%p\n", frame->instr_ptr, ip); + stack_pointer = _PyFrame_GetStackPointer(frame); + GOTO_TIER_ONE(frame->instr_ptr, 1); + } + break; + } + + case _DYNAMIC_EXIT: { + PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + GOTO_TIER_ONE(frame->instr_ptr, 1); + break; + } + #undef TIER_TWO diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 884f2e928733d9..a9749abb5fad87 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5482,7 +5482,7 @@ assert(executor->vm_data.code == code); assert(executor->vm_data.valid); assert(tstate->current_executor == NULL); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + if (IS_JIT_TRACING() || _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { opcode = executor->vm_data.opcode; oparg = (oparg & ~255) | executor->vm_data.oparg; next_instr = this_instr; @@ -7430,6 +7430,9 @@ _PyEval_FrameClearAndPop(tstate, dying); stack_pointer = _PyFrame_GetStackPointer(frame); LOAD_IP(frame->return_offset); + #if TIER_TWO + frame->instr_ptr += frame->return_offset; + #endif res = temp; LLTRACE_RESUME_FRAME(); } @@ -7498,6 +7501,9 @@ #endif stack_pointer = _PyFrame_GetStackPointer(frame); LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + #if TIER_TWO + frame->instr_ptr += (1 + INLINE_CACHE_ENTRIES_SEND); + #endif value = PyStackRef_MakeHeapSafe(temp); LLTRACE_RESUME_FRAME(); } @@ -7591,7 +7597,7 @@ { #if ENABLE_SPECIALIZATION if (this_instr->op.code == JUMP_BACKWARD) { - this_instr->op.code = tstate->interp->jit ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; + this_instr->op.code = (tstate->interp->jit && !PyStackRef_IsNull(frame->f_funcobj)) ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; next_instr = this_instr; DISPATCH_SAME_OPARG(); } @@ -7608,7 +7614,9 @@ } // _JUMP_BACKWARD_NO_INTERRUPT { + #if TIER_ONE assert(oparg <= INSTR_OFFSET()); + #endif JUMPBY(-oparg); } DISPATCH(); @@ -7637,35 +7645,42 @@ } // _JUMP_BACKWARD_NO_INTERRUPT { + #if TIER_ONE assert(oparg <= INSTR_OFFSET()); + #endif JUMPBY(-oparg); } // _JIT { #ifdef _Py_TIER2 _Py_BackoffCounter counter = this_instr[1].counter; - if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD_JIT) { + if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD_JIT) { _Py_CODEUNIT *start = this_instr; - while (oparg > 255) { - oparg >>= 8; + int curr_oparg = oparg; + while (curr_oparg > 255) { + curr_oparg >>= 8; start--; } if (tstate->interp->jit_tracer_code_buffer == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); tstate->interp->jit_tracer_code_buffer = (_Py_CODEUNIT *)_PyObject_VirtualAlloc(TRACER_BUFFER_SIZE); stack_pointer = _PyFrame_GetStackPointer(frame); - tstate->interp->jit_tracer_code_curr_size = 0; if (tstate->interp->jit_tracer_code_buffer == NULL) { DISPATCH(); } - } - if (this_instr == tstate->interp->jit_tracer_initial_instr) { - LEAVE_TRACING(); - DISPATCH(); + tstate->interp->jit_tracer_code_curr_size = 0; } ENTER_TRACING(); if (tstate->interp->jit_tracer_code_curr_size == 0) { - tstate->interp->jit_tracer_initial_instr = this_instr; + tstate->interp->jit_tracer_initial_instr = next_instr; + tstate->interp->jit_tracer_initial_code = _PyFrame_GetCode(frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + tstate->interp->jit_tracer_initial_func = _PyFrame_GetFunction(frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + tstate->interp->jit_tracer_seen_initial_before = 0; + tstate->interp->jit_completed_loop = false; + tstate->interp->jit_tracer_initial_stack_depth = (int)STACK_LEVEL(); + tstate->interp->jit_tracer_initial_chain_depth = 0; } DISPATCH(); } @@ -7685,7 +7700,9 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); + #if TIER_ONE assert(oparg <= INSTR_OFFSET()); + #endif JUMPBY(-oparg); DISPATCH(); } @@ -7711,7 +7728,9 @@ } // _JUMP_BACKWARD_NO_INTERRUPT { + #if TIER_ONE assert(oparg <= INSTR_OFFSET()); + #endif JUMPBY(-oparg); } DISPATCH(); @@ -9967,8 +9986,6 @@ int opcode = POP_JUMP_IF_FALSE; (void)(opcode); #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_FALSE); @@ -9977,8 +9994,7 @@ cond = stack_pointer[-1]; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsFalse(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + JUMPBY(flag ? oparg : 0); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -9989,8 +10005,6 @@ int opcode = POP_JUMP_IF_NONE; (void)(opcode); #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_NONE); @@ -10019,8 +10033,7 @@ cond = b; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsTrue(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + JUMPBY(flag ? oparg : 0); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -10032,8 +10045,6 @@ int opcode = POP_JUMP_IF_NOT_NONE; (void)(opcode); #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); @@ -10062,8 +10073,7 @@ cond = b; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsFalse(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + JUMPBY(flag ? oparg : 0); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -10075,8 +10085,6 @@ int opcode = POP_JUMP_IF_TRUE; (void)(opcode); #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_TRUE); @@ -10085,8 +10093,7 @@ cond = stack_pointer[-1]; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsTrue(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + JUMPBY(flag ? oparg : 0); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -10369,6 +10376,9 @@ _PyThreadState_PopFrame(tstate, frame); frame = tstate->current_frame = prev; LOAD_IP(frame->return_offset); + #if TIER_TWO + frame->instr_ptr += (frame->return_offset); + #endif stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); LLTRACE_RESUME_FRAME(); @@ -10401,6 +10411,9 @@ _PyEval_FrameClearAndPop(tstate, dying); stack_pointer = _PyFrame_GetStackPointer(frame); LOAD_IP(frame->return_offset); + #if TIER_TWO + frame->instr_ptr += frame->return_offset; + #endif res = temp; LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; @@ -11415,6 +11428,41 @@ DISPATCH(); } + TARGET(TIER1_GUARD_IP) { + #if _Py_TAIL_CALL_INTERP + int opcode = TIER1_GUARD_IP; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + next_instr += 5; + INSTRUCTION_STATS(TIER1_GUARD_IP); + PyObject *func_ptr = read_obj(&this_instr[1].cache); + (void)func_ptr; + _PyFrame_SetStackPointer(frame, stack_pointer); + (void)(func_ptr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + + TARGET(TIER1_SET_IP) { + #if _Py_TAIL_CALL_INTERP + int opcode = TIER1_SET_IP; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(TIER1_SET_IP); + PyObject *ip = read_obj(&this_instr[1].cache); + (void)ip; + _PyFrame_SetStackPointer(frame, stack_pointer); + (void)(ip); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH(); + } + TARGET(TO_BOOL) { #if _Py_TAIL_CALL_INTERP int opcode = TO_BOOL; @@ -12089,6 +12137,9 @@ #endif stack_pointer = _PyFrame_GetStackPointer(frame); LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + #if TIER_TWO + frame->instr_ptr += (1 + INLINE_CACHE_ENTRIES_SEND); + #endif value = PyStackRef_MakeHeapSafe(temp); LLTRACE_RESUME_FRAME(); stack_pointer[0] = value; @@ -17617,7 +17668,7 @@ assert(executor->vm_data.code == code); assert(executor->vm_data.valid); assert(tstate->current_executor == NULL); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + if (IS_JIT_TRACING() || _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { opcode = executor->vm_data.opcode; oparg = (oparg & ~255) | executor->vm_data.oparg; next_instr = this_instr; @@ -19589,6 +19640,9 @@ _PyEval_FrameClearAndPop(tstate, dying); stack_pointer = _PyFrame_GetStackPointer(frame); LOAD_IP(frame->return_offset); + #if TIER_TWO + frame->instr_ptr += frame->return_offset; + #endif res = temp; LLTRACE_RESUME_FRAME(); } @@ -19657,6 +19711,9 @@ #endif stack_pointer = _PyFrame_GetStackPointer(frame); LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + #if TIER_TWO + frame->instr_ptr += (1 + INLINE_CACHE_ENTRIES_SEND); + #endif value = PyStackRef_MakeHeapSafe(temp); LLTRACE_RESUME_FRAME(); } @@ -19754,7 +19811,7 @@ { #if ENABLE_SPECIALIZATION if (this_instr->op.code == JUMP_BACKWARD) { - this_instr->op.code = tstate->interp->jit ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; + this_instr->op.code = (tstate->interp->jit && !PyStackRef_IsNull(frame->f_funcobj)) ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; next_instr = this_instr; DISPATCH_SAME_OPARG(); } @@ -19771,7 +19828,9 @@ } // _JUMP_BACKWARD_NO_INTERRUPT { + #if TIER_ONE assert(oparg <= INSTR_OFFSET()); + #endif JUMPBY(-oparg); } TRACING_DISPATCH(); @@ -19800,35 +19859,42 @@ } // _JUMP_BACKWARD_NO_INTERRUPT { + #if TIER_ONE assert(oparg <= INSTR_OFFSET()); + #endif JUMPBY(-oparg); } // _JIT { #ifdef _Py_TIER2 _Py_BackoffCounter counter = this_instr[1].counter; - if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD_JIT) { + if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD_JIT) { _Py_CODEUNIT *start = this_instr; - while (oparg > 255) { - oparg >>= 8; + int curr_oparg = oparg; + while (curr_oparg > 255) { + curr_oparg >>= 8; start--; } if (tstate->interp->jit_tracer_code_buffer == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); tstate->interp->jit_tracer_code_buffer = (_Py_CODEUNIT *)_PyObject_VirtualAlloc(TRACER_BUFFER_SIZE); stack_pointer = _PyFrame_GetStackPointer(frame); - tstate->interp->jit_tracer_code_curr_size = 0; if (tstate->interp->jit_tracer_code_buffer == NULL) { TRACING_DISPATCH(); } - } - if (this_instr == tstate->interp->jit_tracer_initial_instr) { - LEAVE_TRACING(); - TRACING_DISPATCH(); + tstate->interp->jit_tracer_code_curr_size = 0; } ENTER_TRACING(); if (tstate->interp->jit_tracer_code_curr_size == 0) { - tstate->interp->jit_tracer_initial_instr = this_instr; + tstate->interp->jit_tracer_initial_instr = next_instr; + tstate->interp->jit_tracer_initial_code = _PyFrame_GetCode(frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + tstate->interp->jit_tracer_initial_func = _PyFrame_GetFunction(frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + tstate->interp->jit_tracer_seen_initial_before = 0; + tstate->interp->jit_completed_loop = false; + tstate->interp->jit_tracer_initial_stack_depth = (int)STACK_LEVEL(); + tstate->interp->jit_tracer_initial_chain_depth = 0; } TRACING_DISPATCH(); } @@ -19850,7 +19916,9 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); + #if TIER_ONE assert(oparg <= INSTR_OFFSET()); + #endif JUMPBY(-oparg); TRACING_DISPATCH(); } @@ -19878,7 +19946,9 @@ } // _JUMP_BACKWARD_NO_INTERRUPT { + #if TIER_ONE assert(oparg <= INSTR_OFFSET()); + #endif JUMPBY(-oparg); } TRACING_DISPATCH(); @@ -22204,8 +22274,7 @@ cond = stack_pointer[-1]; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsFalse(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + JUMPBY(flag ? oparg : 0); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); TRACING_DISPATCH(); @@ -22246,8 +22315,7 @@ cond = b; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsTrue(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + JUMPBY(flag ? oparg : 0); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -22289,8 +22357,7 @@ cond = b; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsFalse(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + JUMPBY(flag ? oparg : 0); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -22312,8 +22379,7 @@ cond = stack_pointer[-1]; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsTrue(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + JUMPBY(flag ? oparg : 0); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); TRACING_DISPATCH(); @@ -22606,6 +22672,9 @@ _PyThreadState_PopFrame(tstate, frame); frame = tstate->current_frame = prev; LOAD_IP(frame->return_offset); + #if TIER_TWO + frame->instr_ptr += (frame->return_offset); + #endif stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); LLTRACE_RESUME_FRAME(); @@ -22640,6 +22709,9 @@ _PyEval_FrameClearAndPop(tstate, dying); stack_pointer = _PyFrame_GetStackPointer(frame); LOAD_IP(frame->return_offset); + #if TIER_TWO + frame->instr_ptr += frame->return_offset; + #endif res = temp; LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; @@ -23678,6 +23750,41 @@ TRACING_DISPATCH(); } + TRACING_TARGET(TIER1_GUARD_IP) { + #if _Py_TAIL_CALL_INTERP + int opcode = TIER1_GUARD_IP; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + next_instr += 5; + INSTRUCTION_STATS(TIER1_GUARD_IP); + PyObject *func_ptr = read_obj(&this_instr[1].cache); + (void)func_ptr; + _PyFrame_SetStackPointer(frame, stack_pointer); + (void)(func_ptr); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(TIER1_SET_IP) { + #if _Py_TAIL_CALL_INTERP + int opcode = TIER1_SET_IP; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(TIER1_SET_IP); + PyObject *ip = read_obj(&this_instr[1].cache); + (void)ip; + _PyFrame_SetStackPointer(frame, stack_pointer); + (void)(ip); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + TRACING_TARGET(TO_BOOL) { #if _Py_TAIL_CALL_INTERP int opcode = TO_BOOL; @@ -24364,6 +24471,9 @@ #endif stack_pointer = _PyFrame_GetStackPointer(frame); LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + #if TIER_TWO + frame->instr_ptr += (1 + INLINE_CACHE_ENTRIES_SEND); + #endif value = PyStackRef_MakeHeapSafe(temp); LLTRACE_RESUME_FRAME(); stack_pointer[0] = value; diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 299739764919bb..47d6290b0dc1e8 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -39,6 +39,8 @@ static void *opcode_targets_table[256] = { &&TARGET_SETUP_ANNOTATIONS, &&TARGET_STORE_SLICE, &&TARGET_STORE_SUBSCR, + &&TARGET_TIER1_GUARD_IP, + &&TARGET_TIER1_SET_IP, &&TARGET_TO_BOOL, &&TARGET_UNARY_INVERT, &&TARGET_UNARY_NEGATIVE, @@ -126,8 +128,6 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_RESUME, &&TARGET_BINARY_OP_ADD_FLOAT, &&TARGET_BINARY_OP_ADD_INT, @@ -297,6 +297,8 @@ static void *opcode_tracing_targets_table[256] = { &&TARGET_TRACING_SETUP_ANNOTATIONS, &&TARGET_TRACING_STORE_SLICE, &&TARGET_TRACING_STORE_SUBSCR, + &&TARGET_TRACING_TIER1_GUARD_IP, + &&TARGET_TRACING_TIER1_SET_IP, &&TARGET_TRACING_TO_BOOL, &&TARGET_TRACING_UNARY_INVERT, &&TARGET_TRACING_UNARY_NEGATIVE, @@ -384,8 +386,6 @@ static void *opcode_tracing_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_TRACING_RESUME, &&TARGET_TRACING_BINARY_OP_ADD_FLOAT, &&TARGET_TRACING_BINARY_OP_ADD_INT, @@ -943,6 +943,10 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SWAP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_SWAP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TIER1_GUARD_IP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_TIER1_GUARD_IP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TIER1_SET_IP(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_TIER1_SET_IP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_TO_BOOL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS); @@ -1197,6 +1201,8 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [STORE_SUBSCR_DICT] = _TAIL_CALL_STORE_SUBSCR_DICT, [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_STORE_SUBSCR_LIST_INT, [SWAP] = _TAIL_CALL_SWAP, + [TIER1_GUARD_IP] = _TAIL_CALL_TIER1_GUARD_IP, + [TIER1_SET_IP] = _TAIL_CALL_TIER1_SET_IP, [TO_BOOL] = _TAIL_CALL_TO_BOOL, [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_TO_BOOL_ALWAYS_TRUE, [TO_BOOL_BOOL] = _TAIL_CALL_TO_BOOL_BOOL, @@ -1214,8 +1220,6 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE, [WITH_EXCEPT_START] = _TAIL_CALL_WITH_EXCEPT_START, [YIELD_VALUE] = _TAIL_CALL_YIELD_VALUE, - [121] = _TAIL_CALL_UNKNOWN_OPCODE, - [122] = _TAIL_CALL_UNKNOWN_OPCODE, [123] = _TAIL_CALL_UNKNOWN_OPCODE, [124] = _TAIL_CALL_UNKNOWN_OPCODE, [125] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -1455,6 +1459,8 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [STORE_SUBSCR_DICT] = _TAIL_CALL_TRACING_STORE_SUBSCR_DICT, [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_TRACING_STORE_SUBSCR_LIST_INT, [SWAP] = _TAIL_CALL_TRACING_SWAP, + [TIER1_GUARD_IP] = _TAIL_CALL_TRACING_TIER1_GUARD_IP, + [TIER1_SET_IP] = _TAIL_CALL_TRACING_TIER1_SET_IP, [TO_BOOL] = _TAIL_CALL_TRACING_TO_BOOL, [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_TRACING_TO_BOOL_ALWAYS_TRUE, [TO_BOOL_BOOL] = _TAIL_CALL_TRACING_TO_BOOL_BOOL, @@ -1472,8 +1478,6 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_TRACING_UNPACK_SEQUENCE_TWO_TUPLE, [WITH_EXCEPT_START] = _TAIL_CALL_TRACING_WITH_EXCEPT_START, [YIELD_VALUE] = _TAIL_CALL_TRACING_YIELD_VALUE, - [121] = _TAIL_CALL_UNKNOWN_OPCODE, - [122] = _TAIL_CALL_UNKNOWN_OPCODE, [123] = _TAIL_CALL_UNKNOWN_OPCODE, [124] = _TAIL_CALL_UNKNOWN_OPCODE, [125] = _TAIL_CALL_UNKNOWN_OPCODE, diff --git a/Python/optimizer.c b/Python/optimizer.c index 53f1500f3989a4..e4ccca924302ca 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -33,6 +33,9 @@ static bool has_space_for_executor(PyCodeObject *code, _Py_CODEUNIT *instr) { + if (code == (PyCodeObject *)&_Py_InitCleanup) { + return false; + } if (instr->op.code == ENTER_EXECUTOR) { return true; } @@ -102,8 +105,8 @@ static _PyExecutorObject * make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies); static int -uop_optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *instr, - _PyExecutorObject **exec_ptr, int curr_stackentries, +uop_optimize(_PyInterpreterFrame *frame, PyThreadState *tstate, + _PyExecutorObject **exec_ptr, bool progress_needed); /* Returns 1 if optimized, 0 if not optimized, and -1 for an error. @@ -112,13 +115,14 @@ uop_optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *instr, // gh-137573: inlining this function causes stack overflows Py_NO_INLINE int _PyOptimizer_Optimize( - _PyInterpreterFrame *frame, _Py_CODEUNIT *start, - _PyExecutorObject **executor_ptr, int chain_depth) + _PyInterpreterFrame *frame, PyThreadState *tstate) { _PyStackRef *stack_pointer = frame->stackpointer; PyInterpreterState *interp = _PyInterpreterState_GET(); + int chain_depth = tstate->interp->jit_tracer_initial_chain_depth; assert(interp->jit); assert(!interp->compiling); + assert(tstate->interp->jit_tracer_initial_stack_depth >= 0); interp->compiling = true; // The first executor in a chain and the MAX_CHAIN_DEPTH'th executor *must* // make progress in order to avoid infinite loops or excessively-long @@ -126,18 +130,20 @@ _PyOptimizer_Optimize( // this is true, since a deopt won't infinitely re-enter the executor: chain_depth %= MAX_CHAIN_DEPTH; bool progress_needed = chain_depth == 0; - PyCodeObject *code = _PyFrame_GetCode(frame); + PyCodeObject *code = (PyCodeObject *)tstate->interp->jit_tracer_initial_func->func_code; assert(PyCode_Check(code)); + _Py_CODEUNIT *start = tstate->interp->jit_tracer_initial_instr; if (progress_needed && !has_space_for_executor(code, start)) { interp->compiling = false; return 0; } - int err = uop_optimize(frame, start, executor_ptr, (int)(stack_pointer - _PyFrame_Stackbase(frame)), progress_needed); + _PyExecutorObject *executor; + int err = uop_optimize(frame, tstate, &executor, progress_needed); if (err <= 0) { interp->compiling = false; return err; } - assert(*executor_ptr != NULL); + assert(executor != NULL); if (progress_needed) { int index = get_index_for_executor(code, start); if (index < 0) { @@ -147,17 +153,17 @@ _PyOptimizer_Optimize( * If an optimizer has already produced an executor, * it might get confused by the executor disappearing, * but there is not much we can do about that here. */ - Py_DECREF(*executor_ptr); + Py_DECREF(executor); interp->compiling = false; return 0; } - insert_executor(code, start, index, *executor_ptr); + insert_executor(code, start, index, executor); } else { - (*executor_ptr)->vm_data.code = NULL; + executor->vm_data.code = NULL; } - (*executor_ptr)->vm_data.chain_depth = chain_depth; - assert((*executor_ptr)->vm_data.valid); + executor->vm_data.chain_depth = chain_depth; + assert(executor->vm_data.valid); interp->compiling = false; return 1; } @@ -435,12 +441,12 @@ PyTypeObject _PyUOpExecutor_Type = { /* TO DO -- Generate these tables */ static const uint16_t _PyUOp_Replacements[MAX_UOP_ID + 1] = { + [_CHECK_PERIODIC_AT_END] = _TIER2_RESUME_CHECK, [_ITER_JUMP_RANGE] = _GUARD_NOT_EXHAUSTED_RANGE, [_ITER_JUMP_LIST] = _GUARD_NOT_EXHAUSTED_LIST, [_ITER_JUMP_TUPLE] = _GUARD_NOT_EXHAUSTED_TUPLE, [_FOR_ITER] = _FOR_ITER_TIER_TWO, [_ITER_NEXT_LIST] = _ITER_NEXT_LIST_TIER_TWO, - [_CHECK_PERIODIC_AT_END] = _TIER2_RESUME_CHECK, }; static const uint8_t @@ -451,18 +457,6 @@ is_for_iter_test[MAX_UOP_ID + 1] = { [_FOR_ITER_TIER_TWO] = 1, }; -static const uint16_t -BRANCH_TO_GUARD[4][2] = { - [POP_JUMP_IF_FALSE - POP_JUMP_IF_FALSE][0] = _GUARD_IS_TRUE_POP, - [POP_JUMP_IF_FALSE - POP_JUMP_IF_FALSE][1] = _GUARD_IS_FALSE_POP, - [POP_JUMP_IF_TRUE - POP_JUMP_IF_FALSE][0] = _GUARD_IS_FALSE_POP, - [POP_JUMP_IF_TRUE - POP_JUMP_IF_FALSE][1] = _GUARD_IS_TRUE_POP, - [POP_JUMP_IF_NONE - POP_JUMP_IF_FALSE][0] = _GUARD_IS_NOT_NONE_POP, - [POP_JUMP_IF_NONE - POP_JUMP_IF_FALSE][1] = _GUARD_IS_NONE_POP, - [POP_JUMP_IF_NOT_NONE - POP_JUMP_IF_FALSE][0] = _GUARD_IS_NONE_POP, - [POP_JUMP_IF_NOT_NONE - POP_JUMP_IF_FALSE][1] = _GUARD_IS_NOT_NONE_POP, -}; - #define CONFIDENCE_RANGE 1000 #define CONFIDENCE_CUTOFF 333 @@ -525,27 +519,6 @@ add_to_trace( // Reserve space for N uops, plus 3 for _SET_IP, _CHECK_VALIDITY and _EXIT_TRACE #define RESERVE(needed) RESERVE_RAW((needed) + 3, _PyUOpName(opcode)) -// Trace stack operations (used by _PUSH_FRAME, _RETURN_VALUE) -#define TRACE_STACK_PUSH() \ - if (trace_stack_depth >= TRACE_STACK_SIZE) { \ - DPRINTF(2, "Trace stack overflow\n"); \ - OPT_STAT_INC(trace_stack_overflow); \ - return 0; \ - } \ - assert(func == NULL || func->func_code == (PyObject *)code); \ - trace_stack[trace_stack_depth].func = func; \ - trace_stack[trace_stack_depth].code = code; \ - trace_stack[trace_stack_depth].instr = instr; \ - trace_stack_depth++; -#define TRACE_STACK_POP() \ - if (trace_stack_depth <= 0) { \ - Py_FatalError("Trace stack underflow\n"); \ - } \ - trace_stack_depth--; \ - func = trace_stack[trace_stack_depth].func; \ - code = trace_stack[trace_stack_depth].code; \ - assert(func == NULL || func->func_code == (PyObject *)code); \ - instr = trace_stack[trace_stack_depth].instr; /* Returns the length of the trace on success, * 0 if it failed to produce a worthwhile trace, @@ -554,33 +527,25 @@ add_to_trace( static int translate_bytecode_to_trace( _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr, + PyThreadState *tstate, _PyUOpInstruction *trace, int buffer_size, _PyBloomFilter *dependencies, bool progress_needed) { bool first = true; - PyCodeObject *code = _PyFrame_GetCode(frame); - PyFunctionObject *func = _PyFrame_GetFunction(frame); - assert(PyFunction_Check(func)); + PyCodeObject *code = tstate->interp->jit_tracer_initial_code;; + PyFunctionObject *func = tstate->interp->jit_tracer_initial_func; PyCodeObject *initial_code = code; _Py_BloomFilter_Add(dependencies, initial_code); - _Py_CODEUNIT *initial_instr = instr; + _Py_CODEUNIT *target_instr = tstate->interp->jit_tracer_initial_instr; + _Py_CODEUNIT *initial_instr = tstate->interp->jit_tracer_code_buffer; int trace_length = 0; // Leave space for possible trailing _EXIT_TRACE int max_length = buffer_size-2; - struct { - PyFunctionObject *func; - PyCodeObject *code; - _Py_CODEUNIT *instr; - } trace_stack[TRACE_STACK_SIZE]; - int trace_stack_depth = 0; - int confidence = CONFIDENCE_RANGE; // Adjusted by branch instructions - bool jump_seen = false; #ifdef Py_DEBUG char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); - int lltrace = 0; + int lltrace = 3; if (python_lltrace != NULL && *python_lltrace >= '0') { lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that } @@ -591,55 +556,40 @@ translate_bytecode_to_trace( PyUnicode_AsUTF8(code->co_qualname), PyUnicode_AsUTF8(code->co_filename), code->co_firstlineno, - 2 * INSTR_IP(initial_instr, code)); - ADD_TO_TRACE(_START_EXECUTOR, 0, (uintptr_t)instr, INSTR_IP(instr, code)); + 2 * INSTR_IP(target_instr, code)); + ADD_TO_TRACE(_START_EXECUTOR, 0, (uintptr_t)target_instr, INSTR_IP(target_instr, code)); ADD_TO_TRACE(_MAKE_WARM, 0, 0, 0); uint32_t target = 0; + bool preceded_by_for_iter = false; - for (;;) { - target = INSTR_IP(instr, code); + for (int x = 0; x < tstate->interp->jit_tracer_code_curr_size;) { + target = INSTR_IP(target_instr, code); // One for possible _DEOPT, one because _CHECK_VALIDITY itself might _DEOPT max_length-=2; - uint32_t opcode = instr->op.code; - uint32_t oparg = instr->op.arg; - - if (!first && instr == initial_instr) { - // We have looped around to the start: - RESERVE(1); - ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0, 0); - goto done; - } + uint32_t opcode = initial_instr[x].op.code; + uint32_t oparg = initial_instr[x].op.arg; DPRINTF(2, "%d: %s(%d)\n", target, _PyOpcode_OpName[opcode], oparg); if (opcode == EXTENDED_ARG) { - instr++; - opcode = instr->op.code; - oparg = (oparg << 8) | instr->op.arg; + x++; + opcode = initial_instr[x].op.code; + oparg = (oparg << 8) | initial_instr[x].op.arg; if (opcode == EXTENDED_ARG) { - instr--; + x--; goto done; } } - if (opcode == ENTER_EXECUTOR) { - // We have a couple of options here. We *could* peek "underneath" - // this executor and continue tracing, which could give us a longer, - // more optimizeable trace (at the expense of lots of duplicated - // tier two code). Instead, we choose to just end here and stitch to - // the other trace, which allows a side-exit traces to rejoin the - // "main" trace periodically (and also helps protect us against - // pathological behavior where the amount of tier two code explodes - // for a medium-length, branchy code path). This seems to work - // better in practice, but in the future we could be smarter about - // what we do here: - goto done; - } assert(opcode != ENTER_EXECUTOR && opcode != EXTENDED_ARG); - RESERVE_RAW(2, "_CHECK_VALIDITY"); - ADD_TO_TRACE(_CHECK_VALIDITY, 0, 0, target); - if (!OPCODE_HAS_NO_SAVE_IP(opcode)) { - RESERVE_RAW(2, "_SET_IP"); - ADD_TO_TRACE(_SET_IP, 0, (uintptr_t)instr, target); + if (opcode != NOP) { + if (opcode != TIER1_GUARD_IP) { + RESERVE_RAW(1, "_CHECK_VALIDITY"); + ADD_TO_TRACE(_CHECK_VALIDITY, 0, 0, target); + } + if (!OPCODE_HAS_NO_SAVE_IP(opcode)) { + RESERVE_RAW(2, "_SET_IP"); + ADD_TO_TRACE(_SET_IP, 0, (uintptr_t)target_instr, target); + } } /* Special case the first instruction, @@ -663,69 +613,56 @@ translate_bytecode_to_trace( RESERVE_RAW(2, "_ERROR_POP_N"); max_length--; } + if (opcode == FOR_ITER_RANGE || opcode == FOR_ITER_LIST || opcode == FOR_ITER_TUPLE) { + preceded_by_for_iter = true; + // assert((initial_instr + x + 1 + _PyOpcode_Caches[FOR_ITER])->op.code == TIER1_GUARD_IP); + // assert((initial_instr + x + 6 + _PyOpcode_Caches[FOR_ITER])->op.code == TIER1_SET_IP); + // PyObject *temp = read_obj(&(initial_instr + x + 2 + _PyOpcode_Caches[FOR_ITER])->cache); + // if (temp != NULL) { + // code = ((PyFunctionObject*)temp)->func_code; + // } + // target_instr = (_Py_CODEUNIT *)(read_obj(&(initial_instr + x + 7 + _PyOpcode_Caches[FOR_ITER])->cache)); + // target = INSTR_IP(target_instr, code); + // + // (initial_instr + x + 1 + _PyOpcode_Caches[FOR_ITER])->op.code = NOP; + // (initial_instr + x + 2 + _PyOpcode_Caches[FOR_ITER])->op.code = NOP; + // (initial_instr + x + 3 + _PyOpcode_Caches[FOR_ITER])->op.code = NOP; + // (initial_instr + x + 4 + _PyOpcode_Caches[FOR_ITER])->op.code = NOP; + // (initial_instr + x + 5 + _PyOpcode_Caches[FOR_ITER])->op.code = NOP; + // + // (initial_instr + x + 6 + _PyOpcode_Caches[FOR_ITER])->op.code = NOP; + // (initial_instr + x + 7 + _PyOpcode_Caches[FOR_ITER])->op.code = NOP; + // (initial_instr + x + 8 + _PyOpcode_Caches[FOR_ITER])->op.code = NOP; + // (initial_instr + x + 9 + _PyOpcode_Caches[FOR_ITER])->op.code = NOP; + // (initial_instr + x + 10 + _PyOpcode_Caches[FOR_ITER])->op.code = NOP; + } switch (opcode) { - case POP_JUMP_IF_NONE: - case POP_JUMP_IF_NOT_NONE: - case POP_JUMP_IF_FALSE: - case POP_JUMP_IF_TRUE: - { - RESERVE(1); - int counter = instr[1].cache; - int bitcount = _Py_popcount32(counter); - int jump_likely = bitcount > 8; - /* If bitcount is 8 (half the jumps were taken), adjust confidence by 50%. - For values in between, adjust proportionally. */ - if (jump_likely) { - confidence = confidence * bitcount / 16; - } - else { - confidence = confidence * (16 - bitcount) / 16; - } - uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_likely]; - DPRINTF(2, "%d: %s(%d): counter=%04x, bitcount=%d, likely=%d, confidence=%d, uopcode=%s\n", - target, _PyOpcode_OpName[opcode], oparg, - counter, bitcount, jump_likely, confidence, _PyUOpName(uopcode)); - if (confidence < CONFIDENCE_CUTOFF) { - DPRINTF(2, "Confidence too low (%d < %d)\n", confidence, CONFIDENCE_CUTOFF); - OPT_STAT_INC(low_confidence); - goto done; - } - _Py_CODEUNIT *next_instr = instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; - _Py_CODEUNIT *target_instr = next_instr + oparg; - if (jump_likely) { - DPRINTF(2, "Jump likely (%04x = %d bits), continue at byte offset %d\n", - instr[1].cache, bitcount, 2 * INSTR_IP(target_instr, code)); - instr = target_instr; - ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(next_instr, code)); - goto top; - } - ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(target_instr, code)); - break; - } - - case JUMP_BACKWARD: case JUMP_BACKWARD_JIT: + case JUMP_BACKWARD: ADD_TO_TRACE(_CHECK_PERIODIC, 0, 0, target); _Py_FALLTHROUGH; case JUMP_BACKWARD_NO_INTERRUPT: - { - instr += 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] - (int)oparg; - if (jump_seen) { - OPT_STAT_INC(inner_loop); - DPRINTF(2, "JUMP_BACKWARD not to top ends trace\n"); - goto done; + ADD_TO_TRACE(_JUMP_BACKWARD_NO_INTERRUPT, oparg, 0, target); + break; + case TIER1_GUARD_IP: + func = (PyFunctionObject *)read_obj(&(initial_instr + x + 1)->cache); + assert(func == NULL || PyFunction_Check(func)); + if (func != NULL) { + code = (PyCodeObject *)func->func_code; } - jump_seen = true; - goto top; - } + assert((initial_instr + x + 5)->op.code == TIER1_SET_IP); + target_instr = (_Py_CODEUNIT*)read_obj(&(initial_instr + x + 6)->cache); + if (preceded_by_for_iter) {; + preceded_by_for_iter = false; + } + else { + ADD_TO_TRACE(_GUARD_IP, 0, (uintptr_t)target_instr, 0) + } + break; - case JUMP_FORWARD: - { - RESERVE(0); - // This will emit two _SET_IP instructions; leave it to the optimizer - instr += oparg; + case TIER1_SET_IP: + target_instr = (_Py_CODEUNIT*)read_obj(&(initial_instr + x + 1)->cache); break; - } case RESUME: /* Use a special tier 2 version of RESUME_CHECK to allow traces to @@ -740,17 +677,6 @@ translate_bytecode_to_trace( // Reserve space for nuops (+ _SET_IP + _EXIT_TRACE) int nuops = expansion->nuops; RESERVE(nuops + 1); /* One extra for exit */ - int16_t last_op = expansion->uops[nuops-1].uop; - if (last_op == _RETURN_VALUE || last_op == _RETURN_GENERATOR || last_op == _YIELD_VALUE) { - // Check for trace stack underflow now: - // We can't bail e.g. in the middle of - // LOAD_CONST + _RETURN_VALUE. - if (trace_stack_depth == 0) { - DPRINTF(2, "Trace stack underflow\n"); - OPT_STAT_INC(trace_stack_underflow); - return 0; - } - } uint32_t orig_oparg = oparg; // For OPARG_TOP/BOTTOM for (int i = 0; i < nuops; i++) { oparg = orig_oparg; @@ -760,16 +686,16 @@ translate_bytecode_to_trace( int offset = expansion->uops[i].offset + 1; switch (expansion->uops[i].size) { case OPARG_SIMPLE: - assert(opcode != JUMP_BACKWARD_NO_INTERRUPT && opcode != JUMP_BACKWARD); + assert(opcode != _JUMP_BACKWARD_NO_INTERRUPT && opcode != JUMP_BACKWARD); break; case OPARG_CACHE_1: - operand = read_u16(&instr[offset].cache); + operand = read_u16(&(initial_instr + x)[offset].cache); break; case OPARG_CACHE_2: - operand = read_u32(&instr[offset].cache); + operand = read_u32(&(initial_instr + x)[offset].cache); break; case OPARG_CACHE_4: - operand = read_u64(&instr[offset].cache); + operand = read_u64(&(initial_instr + x)[offset].cache); break; case OPARG_TOP: // First half of super-instr oparg = orig_oparg >> 4; @@ -784,6 +710,7 @@ translate_bytecode_to_trace( case OPARG_REPLACED: uop = _PyUOp_Replacements[uop]; assert(uop != 0); + uint32_t next_inst = target + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + (oparg > 255); if (uop == _TIER2_RESUME_CHECK) { target = next_inst; @@ -798,17 +725,17 @@ translate_bytecode_to_trace( break; case OPERAND1_1: assert(trace[trace_length-1].opcode == uop); - operand = read_u16(&instr[offset].cache); + operand = read_u16(&(initial_instr + x)[offset].cache); trace[trace_length-1].operand1 = operand; continue; case OPERAND1_2: assert(trace[trace_length-1].opcode == uop); - operand = read_u32(&instr[offset].cache); + operand = read_u32(&(initial_instr + x)[offset].cache); trace[trace_length-1].operand1 = operand; continue; case OPERAND1_4: assert(trace[trace_length-1].opcode == uop); - operand = read_u64(&instr[offset].cache); + operand = read_u64(&(initial_instr + x)[offset].cache); trace[trace_length-1].operand1 = operand; continue; default: @@ -820,116 +747,13 @@ translate_bytecode_to_trace( Py_FatalError("garbled expansion"); } - if (uop == _RETURN_VALUE || uop == _RETURN_GENERATOR || uop == _YIELD_VALUE) { - TRACE_STACK_POP(); - /* Set the operand to the function or code object returned to, - * to assist optimization passes. (See _PUSH_FRAME below.) - */ - if (func != NULL) { - operand = (uintptr_t)func; - } - else if (code != NULL) { - operand = (uintptr_t)code | 1; - } - else { - operand = 0; - } - ADD_TO_TRACE(uop, oparg, operand, target); - DPRINTF(2, - "Returning to %s (%s:%d) at byte offset %d\n", - PyUnicode_AsUTF8(code->co_qualname), - PyUnicode_AsUTF8(code->co_filename), - code->co_firstlineno, - 2 * INSTR_IP(instr, code)); - goto top; - } - - if (uop == _PUSH_FRAME) { - assert(i + 1 == nuops); - if (opcode == FOR_ITER_GEN || - opcode == LOAD_ATTR_PROPERTY || - opcode == BINARY_OP_SUBSCR_GETITEM || - opcode == SEND_GEN) - { - DPRINTF(2, "Bailing due to dynamic target\n"); - OPT_STAT_INC(unknown_callee); - return 0; - } - assert(_PyOpcode_Deopt[opcode] == CALL || _PyOpcode_Deopt[opcode] == CALL_KW); - int func_version_offset = - offsetof(_PyCallCache, func_version)/sizeof(_Py_CODEUNIT) - // Add one to account for the actual opcode/oparg pair: - + 1; - uint32_t func_version = read_u32(&instr[func_version_offset].cache); - PyCodeObject *new_code = NULL; - PyFunctionObject *new_func = - _PyFunction_LookupByVersion(func_version, (PyObject **) &new_code); - DPRINTF(2, "Function: version=%#x; new_func=%p, new_code=%p\n", - (int)func_version, new_func, new_code); - if (new_code != NULL) { - if (new_code == code) { - // Recursive call, bail (we could be here forever). - DPRINTF(2, "Bailing on recursive call to %s (%s:%d)\n", - PyUnicode_AsUTF8(new_code->co_qualname), - PyUnicode_AsUTF8(new_code->co_filename), - new_code->co_firstlineno); - OPT_STAT_INC(recursive_call); - ADD_TO_TRACE(uop, oparg, 0, target); - ADD_TO_TRACE(_EXIT_TRACE, 0, 0, 0); - goto done; - } - if (new_code->co_version != func_version) { - // func.__code__ was updated. - // Perhaps it may happen again, so don't bother tracing. - // TODO: Reason about this -- is it better to bail or not? - DPRINTF(2, "Bailing because co_version != func_version\n"); - ADD_TO_TRACE(uop, oparg, 0, target); - ADD_TO_TRACE(_EXIT_TRACE, 0, 0, 0); - goto done; - } - // Increment IP to the return address - instr += _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + 1; - TRACE_STACK_PUSH(); - _Py_BloomFilter_Add(dependencies, new_code); - /* Set the operand to the callee's function or code object, - * to assist optimization passes. - * We prefer setting it to the function - * but if that's not available but the code is available, - * use the code, setting the low bit so the optimizer knows. - */ - if (new_func != NULL) { - operand = (uintptr_t)new_func; - } - else if (new_code != NULL) { - operand = (uintptr_t)new_code | 1; - } - else { - operand = 0; - } - ADD_TO_TRACE(uop, oparg, operand, target); - code = new_code; - func = new_func; - instr = _PyCode_CODE(code); - DPRINTF(2, - "Continuing in %s (%s:%d) at byte offset %d\n", - PyUnicode_AsUTF8(code->co_qualname), - PyUnicode_AsUTF8(code->co_filename), - code->co_firstlineno, - 2 * INSTR_IP(instr, code)); - goto top; - } - DPRINTF(2, "Bail, new_code == NULL\n"); - OPT_STAT_INC(unknown_callee); - return 0; - } - if (uop == _BINARY_OP_INPLACE_ADD_UNICODE) { assert(i + 1 == nuops); - _Py_CODEUNIT *next_instr = instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; + _Py_CODEUNIT *next_instr = target_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; assert(next_instr->op.code == STORE_FAST); operand = next_instr->op.arg; // Skip the STORE_FAST: - instr++; + x++; } // All other instructions @@ -944,24 +768,27 @@ translate_bytecode_to_trace( } // End switch (opcode) - instr++; + x++; // Add cache size for opcode - instr += _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; + x += _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; if (opcode == CALL_LIST_APPEND) { - assert(instr->op.code == POP_TOP); - instr++; + assert(initial_instr[x].op.code == POP_TOP); + x++; + } + + if (x > tstate->interp->jit_tracer_code_curr_size) { + goto done; } - top: // Jump here after _PUSH_FRAME or likely branches. first = false; } // End for (;;) done: - while (trace_stack_depth > 0) { - TRACE_STACK_POP(); + // Looped back to top. + if (!first && tstate->interp->jit_completed_loop && target_instr == tstate->interp->jit_tracer_initial_instr) { + ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0, 0); } - assert(code == initial_code); // Skip short traces where we can't even translate a single instruction: if (first) { OPT_STAT_INC(trace_too_short); @@ -983,7 +810,7 @@ translate_bytecode_to_trace( PyUnicode_AsUTF8(code->co_qualname), PyUnicode_AsUTF8(code->co_filename), code->co_firstlineno, - 2 * INSTR_IP(initial_instr, code), + 2 * INSTR_IP(tstate->interp->jit_tracer_initial_instr, (PyCodeObject *)tstate->interp->jit_tracer_initial_func->func_code), trace_length); OPT_HIST(trace_length, trace_length_hist); return trace_length; @@ -1063,6 +890,9 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) else if (exit_flags & HAS_PERIODIC_FLAG) { exit_op = _HANDLE_PENDING_AND_DEOPT; } + if (opcode == _FOR_ITER_TIER_TWO) { + exit_op = _DYNAMIC_EXIT; + } int32_t jump_target = target; if (is_for_iter_test[opcode]) { /* Target the POP_TOP immediately after the END_FOR, @@ -1178,7 +1008,8 @@ sanity_check(_PyExecutorObject *executor) opcode == _DEOPT || opcode == _HANDLE_PENDING_AND_DEOPT || opcode == _EXIT_TRACE || - opcode == _ERROR_POP_N); + opcode == _ERROR_POP_N || + opcode == _DYNAMIC_EXIT); } } @@ -1214,7 +1045,6 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil int opcode = buffer[i].opcode; dest--; *dest = buffer[i]; - assert(opcode != _POP_JUMP_IF_FALSE && opcode != _POP_JUMP_IF_TRUE); if (opcode == _EXIT_TRACE) { _PyExitData *exit = &executor->exits[next_exit]; exit->target = buffer[i].target; @@ -1280,9 +1110,8 @@ int effective_trace_length(_PyUOpInstruction *buffer, int length) static int uop_optimize( _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr, + PyThreadState *tstate, _PyExecutorObject **exec_ptr, - int curr_stackentries, bool progress_needed) { _PyBloomFilter dependencies; @@ -1301,7 +1130,16 @@ uop_optimize( if (env_var == NULL || *env_var == '\0' || *env_var > '0') { is_noopt = false; } - int length = translate_bytecode_to_trace(frame, instr, buffer, UOP_MAX_TRACE_LENGTH, &dependencies, progress_needed); + int curr_stackentries = tstate->interp->jit_tracer_initial_stack_depth; +// #ifdef Py_DEBUG +// for (int x = 0; x < tstate->interp->jit_tracer_code_curr_size;) { +// int opcode = tstate->interp->jit_tracer_code_buffer[x].op.code; +// int oparg = tstate->interp->jit_tracer_code_buffer[x].op.arg; +// fprintf(stdout, "%s(%d)\n", _PyOpcode_OpName[opcode], oparg); +// x += 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; +// } +// #endif + int length = translate_bytecode_to_trace(frame, tstate, buffer, UOP_MAX_TRACE_LENGTH, &dependencies, progress_needed); if (length <= 0) { // Error or nothing translated return length; @@ -1778,7 +1616,7 @@ executor_to_gv(_PyExecutorObject *executor, FILE *out) else if (flags & HAS_EXIT_FLAG) { assert(inst->format == UOP_FORMAT_JUMP); _PyUOpInstruction const *exit_inst = &executor->trace[inst->jump_target]; - assert(exit_inst->opcode == _EXIT_TRACE); + assert(exit_inst->opcode == _EXIT_TRACE || exit_inst->opcode == _DYNAMIC_EXIT); exit = (_PyExitData *)exit_inst->operand0; } if (exit != NULL && exit->executor != NULL) { diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index a6add301ccb26c..21410f1016ad5d 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -527,13 +527,13 @@ _Py_uop_analyze_and_optimize( { OPT_STAT_INC(optimizer_attempts); - length = optimize_uops( - _PyFrame_GetFunction(frame), buffer, - length, curr_stacklen, dependencies); - - if (length == 0) { - return length; - } + // length = optimize_uops( + // _PyFrame_GetFunction(frame), buffer, + // length, curr_stacklen, dependencies); + // + // if (length == 0) { + // return length; + // } assert(length > 0); diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index b08099d8e2fc3b..1240be061b7828 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -2092,9 +2092,17 @@ break; } - /* _POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 */ + case _POP_JUMP_IF_FALSE: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } - /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ + case _POP_JUMP_IF_TRUE: { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + break; + } case _IS_NONE: { JitOptRef b; @@ -2103,6 +2111,10 @@ break; } + case _JUMP_BACKWARD_NO_INTERRUPT: { + break; + } + case _GET_LEN: { JitOptRef obj; JitOptRef len; @@ -2219,7 +2231,14 @@ break; } - /* _ITER_NEXT_LIST is not a viable micro-op for tier 2 */ + case _ITER_NEXT_LIST: { + JitOptRef next; + next = sym_new_not_null(ctx); + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + break; + } case _ITER_NEXT_LIST_TIER_TWO: { JitOptRef next; @@ -3418,3 +3437,11 @@ break; } + case _GUARD_IP: { + break; + } + + case _DYNAMIC_EXIT: { + break; + } + diff --git a/Python/pystate.c b/Python/pystate.c index 9fe946023741ad..33ffc034b4410c 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -557,6 +557,10 @@ init_interpreter(PyInterpreterState *interp, interp->jit_uop_buffer = NULL; interp->jit_tracer_code_buffer = NULL; interp->jit_tracer_initial_instr = NULL; + interp->jit_tracer_initial_stack_depth = -1; + interp->jit_tracer_initial_chain_depth = -1; + interp->jit_tracer_initial_code = NULL; + interp->jit_tracer_initial_func = NULL; #endif llist_init(&interp->mem_free_queue.head); llist_init(&interp->asyncio_tasks_head); diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 9dd7e5dbfbae7b..75bff34aa74275 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -34,6 +34,7 @@ class Properties: side_exit: bool pure: bool uses_opcode: bool + needs_guard_ip: bool tier: int | None = None const_oparg: int = -1 needs_prev: bool = False @@ -75,6 +76,7 @@ def from_list(properties: list["Properties"]) -> "Properties": pure=all(p.pure for p in properties), needs_prev=any(p.needs_prev for p in properties), no_save_ip=all(p.no_save_ip for p in properties), + needs_guard_ip=any(p.needs_guard_ip for p in properties) ) @property @@ -102,6 +104,7 @@ def infallible(self) -> bool: side_exit=False, pure=True, no_save_ip=False, + needs_guard_ip=False, ) @@ -932,6 +935,7 @@ def compute_properties(op: parser.CodeDef) -> Properties: no_save_ip=no_save_ip, tier=tier_variable(op), needs_prev=variable_used(op, "prev_instr"), + needs_guard_ip=variable_used(op, "JUMPBY") or variable_used(op, "LLTRACE_RESUME_FRAME"), ) def expand(items: list[StackItem], oparg: int) -> list[StackItem]: diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py index b649b38123388d..d17a2431c75b37 100644 --- a/Tools/cases_generator/opcode_metadata_generator.py +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -184,6 +184,16 @@ def generate_cache_table(analysis: Analysis, out: CWriter) -> None: out.emit("#endif\n\n") +def generate_needs_guard_ip_table(analysis: Analysis, out: CWriter) -> None: + out.emit("extern const uint8_t _PyOpcode_NeedsGuardIp[256];\n") + out.emit("#ifdef NEED_OPCODE_METADATA\n") + out.emit("const uint8_t _PyOpcode_NeedsGuardIp[256] = {\n") + for inst in analysis.instructions.values(): + if inst.properties.needs_guard_ip: + out.emit(f"[{inst.name}] = 1,\n") + out.emit("};\n") + out.emit("#endif\n\n") + def generate_name_table(analysis: Analysis, out: CWriter) -> None: table_size = 256 + len(analysis.pseudos) out.emit(f"extern const char *_PyOpcode_OpName[{table_size}];\n") @@ -382,6 +392,7 @@ def generate_opcode_metadata( generate_expansion_table(analysis, out) generate_name_table(analysis, out) generate_cache_table(analysis, out) + generate_needs_guard_ip_table(analysis, out) generate_deopt_table(analysis, out) generate_extra_cases(analysis, out) generate_pseudo_targets(analysis, out) diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index 1bb5f48658ddfc..35a10e7bebed1e 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -63,6 +63,8 @@ class Tier2Emitter(Emitter): def __init__(self, out: CWriter, labels: dict[str, Label]): super().__init__(out, labels) self._replacers["oparg"] = self.oparg + self._replacers["JUMPBY"] = self.jumpby + self._replacers["DISPATCH"] = self.dispatch def goto_error(self, offset: int, storage: Storage) -> str: # To do: Add jump targets for popping values. @@ -134,6 +136,38 @@ def oparg( self.out.emit_at(uop.name[-1], tkn) return True + def jumpby( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ) -> bool: + if storage.spilled: + raise analysis_error("stack_pointer needs reloading before dispatch", tkn) + storage.stack.flush(self.out) + self.emit("TIER2_JUMPBY") + emit_to(self.out, tkn_iter, "SEMI") + self.emit(";\n") + return True + + def dispatch( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ) -> bool: + if storage.spilled: + raise analysis_error("stack_pointer needs reloading before dispatch", tkn) + storage.stack.flush(self.out) + self.emit("break;\n") + next(tkn_iter) + next(tkn_iter) + next(tkn_iter) + return False def write_uop(uop: Uop, emitter: Emitter, stack: Stack) -> Stack: locals: dict[str, Local] = {} @@ -192,6 +226,16 @@ def generate_tier2( ) continue out.emit(f"case {uop.name}: {{\n") + if uop.properties.jumps: + containing_inst = None + for inst in analysis.instructions.values(): + if uop in inst.parts: + print(uop.name, inst.name) + containing_inst = inst + break + assert containing_inst is not None, uop.name + size = containing_inst.size + out.emit(f"TIER2_JUMPBY({size});\n") declare_variables(uop, out) stack = Stack() stack = write_uop(uop, emitter, stack) diff --git a/Tools/cases_generator/tracer_generator.py b/Tools/cases_generator/tracer_generator.py index 0df435a9e32766..e394807518f66c 100644 --- a/Tools/cases_generator/tracer_generator.py +++ b/Tools/cases_generator/tracer_generator.py @@ -53,6 +53,7 @@ def dispatch( if storage.spilled: raise analysis_error("stack_pointer needs reloading before dispatch", tkn) storage.stack.flush(self.out) + self.out.start_line() self.emit("TRACING_DISPATCH") return False diff --git a/out.txt b/out.txt new file mode 100644 index 00000000000000..225b73b0f38765 --- /dev/null +++ b/out.txt @@ -0,0 +1,1089 @@ +0 ADD_TO_BYTECODE_TRACE: FOR_ITER_GEN 3 +2 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +7 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5565a +12 ADD_TO_BYTECODE_TRACE: RESUME_CHECK 2 +13 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5565c +18 ADD_TO_BYTECODE_TRACE: JUMP_BACKWARD_NO_INTERRUPT 5 +19 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +24 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55654 +29 ADD_TO_BYTECODE_TRACE: SEND_GEN 3 +31 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +36 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5565a +41 ADD_TO_BYTECODE_TRACE: RESUME_CHECK 2 +42 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5565c +47 ADD_TO_BYTECODE_TRACE: JUMP_BACKWARD_NO_INTERRUPT 5 +48 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +53 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55654 +58 ADD_TO_BYTECODE_TRACE: SEND_GEN 3 +60 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +65 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5565a +70 ADD_TO_BYTECODE_TRACE: RESUME_CHECK 2 +71 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5565c +76 ADD_TO_BYTECODE_TRACE: JUMP_BACKWARD_NO_INTERRUPT 5 +77 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +82 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55654 +87 ADD_TO_BYTECODE_TRACE: SEND_GEN 3 +89 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +94 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5565a +99 ADD_TO_BYTECODE_TRACE: RESUME_CHECK 2 +100 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5565c +105 ADD_TO_BYTECODE_TRACE: JUMP_BACKWARD_NO_INTERRUPT 5 +106 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +111 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55654 +116 ADD_TO_BYTECODE_TRACE: SEND_GEN 3 +118 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +123 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556c2 +128 ADD_TO_BYTECODE_TRACE: RESUME_CHECK 2 +129 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556c4 +134 ADD_TO_BYTECODE_TRACE: JUMP_BACKWARD_NO_INTERRUPT 5 +135 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +140 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556bc +145 ADD_TO_BYTECODE_TRACE: SEND_GEN 3 +147 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +152 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5565a +157 ADD_TO_BYTECODE_TRACE: RESUME_CHECK 2 +158 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5565c +163 ADD_TO_BYTECODE_TRACE: JUMP_BACKWARD_NO_INTERRUPT 5 +164 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +169 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55654 +174 ADD_TO_BYTECODE_TRACE: SEND_GEN 3 +176 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +181 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556c2 +186 ADD_TO_BYTECODE_TRACE: RESUME_CHECK 2 +187 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556c4 +192 ADD_TO_BYTECODE_TRACE: JUMP_BACKWARD_NO_INTERRUPT 5 +193 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +198 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556bc +203 ADD_TO_BYTECODE_TRACE: SEND_GEN 3 +205 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +210 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5565a +215 ADD_TO_BYTECODE_TRACE: RESUME_CHECK 2 +216 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5565c +221 ADD_TO_BYTECODE_TRACE: JUMP_BACKWARD_NO_INTERRUPT 5 +222 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +227 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55654 +232 ADD_TO_BYTECODE_TRACE: SEND_GEN 3 +234 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +239 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5565a +244 ADD_TO_BYTECODE_TRACE: RESUME_CHECK 2 +245 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5565c +250 ADD_TO_BYTECODE_TRACE: JUMP_BACKWARD_NO_INTERRUPT 5 +251 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +256 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55654 +261 ADD_TO_BYTECODE_TRACE: SEND_GEN 3 +263 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +268 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556c2 +273 ADD_TO_BYTECODE_TRACE: RESUME_CHECK 2 +274 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556c4 +279 ADD_TO_BYTECODE_TRACE: JUMP_BACKWARD_NO_INTERRUPT 5 +280 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +285 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556bc +290 ADD_TO_BYTECODE_TRACE: SEND_GEN 3 +292 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +297 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556c2 +302 ADD_TO_BYTECODE_TRACE: RESUME_CHECK 2 +303 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556c4 +308 ADD_TO_BYTECODE_TRACE: JUMP_BACKWARD_NO_INTERRUPT 5 +309 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +314 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556bc +319 ADD_TO_BYTECODE_TRACE: SEND_GEN 3 +321 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +326 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556c2 +331 ADD_TO_BYTECODE_TRACE: RESUME_CHECK 2 +332 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556c4 +337 ADD_TO_BYTECODE_TRACE: JUMP_BACKWARD_NO_INTERRUPT 5 +338 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +343 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556bc +348 ADD_TO_BYTECODE_TRACE: SEND_GEN 3 +350 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +355 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556c2 +360 ADD_TO_BYTECODE_TRACE: RESUME_CHECK 2 +361 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556c4 +366 ADD_TO_BYTECODE_TRACE: JUMP_BACKWARD_NO_INTERRUPT 5 +367 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +372 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556bc +377 ADD_TO_BYTECODE_TRACE: SEND_GEN 3 +379 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +384 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5565a +389 ADD_TO_BYTECODE_TRACE: RESUME_CHECK 2 +390 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5565c +395 ADD_TO_BYTECODE_TRACE: JUMP_BACKWARD_NO_INTERRUPT 5 +396 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +401 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55654 +406 ADD_TO_BYTECODE_TRACE: SEND_GEN 3 +408 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +413 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556c2 +418 ADD_TO_BYTECODE_TRACE: RESUME_CHECK 2 +419 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556c4 +424 ADD_TO_BYTECODE_TRACE: JUMP_BACKWARD_NO_INTERRUPT 5 +425 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +430 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556bc +435 ADD_TO_BYTECODE_TRACE: SEND_GEN 3 +437 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +442 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5565a +447 ADD_TO_BYTECODE_TRACE: RESUME_CHECK 2 +448 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5565c +453 ADD_TO_BYTECODE_TRACE: JUMP_BACKWARD_NO_INTERRUPT 5 +454 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +459 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55654 +464 ADD_TO_BYTECODE_TRACE: SEND_GEN 3 +466 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +471 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5567a +476 ADD_TO_BYTECODE_TRACE: RESUME_CHECK 5 +477 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5567c +482 ADD_TO_BYTECODE_TRACE: POP_TOP 0 +483 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5567e +488 ADD_TO_BYTECODE_TRACE: LOAD_FAST_BORROW 0 +489 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55680 +494 ADD_TO_BYTECODE_TRACE: LOAD_ATTR_INSTANCE_VALUE 4 +504 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55694 +509 ADD_TO_BYTECODE_TRACE: TO_BOOL_NONE 0 +513 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5569c +518 ADD_TO_BYTECODE_TRACE: POP_JUMP_IF_FALSE 23 +520 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +525 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556ce +530 ADD_TO_BYTECODE_TRACE: LOAD_CONST 0 +531 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556d0 +536 ADD_TO_BYTECODE_TRACE: RETURN_VALUE 0 +537 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +542 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5565e +547 ADD_TO_BYTECODE_TRACE: END_SEND 0 +548 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55660 +553 ADD_TO_BYTECODE_TRACE: POP_TOP 0 +554 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55662 +559 ADD_TO_BYTECODE_TRACE: LOAD_FAST_BORROW 0 +560 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55664 +565 ADD_TO_BYTECODE_TRACE: LOAD_ATTR_INSTANCE_VALUE 2 +575 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55678 +580 ADD_TO_BYTECODE_TRACE: YIELD_VALUE 0 +581 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +586 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556c0 +591 ADD_TO_BYTECODE_TRACE: YIELD_VALUE 1 +592 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +597 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55658 +602 ADD_TO_BYTECODE_TRACE: YIELD_VALUE 1 +603 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +608 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556c0 +613 ADD_TO_BYTECODE_TRACE: YIELD_VALUE 1 +614 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +619 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556c0 +624 ADD_TO_BYTECODE_TRACE: YIELD_VALUE 1 +625 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +630 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556c0 +635 ADD_TO_BYTECODE_TRACE: YIELD_VALUE 1 +636 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +641 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556c0 +646 ADD_TO_BYTECODE_TRACE: YIELD_VALUE 1 +647 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +652 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55658 +657 ADD_TO_BYTECODE_TRACE: YIELD_VALUE 1 +658 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +663 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55658 +668 ADD_TO_BYTECODE_TRACE: YIELD_VALUE 1 +669 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +674 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556c0 +679 ADD_TO_BYTECODE_TRACE: YIELD_VALUE 1 +680 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +685 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55658 +690 ADD_TO_BYTECODE_TRACE: YIELD_VALUE 1 +691 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +696 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556c0 +701 ADD_TO_BYTECODE_TRACE: YIELD_VALUE 1 +702 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +707 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55658 +712 ADD_TO_BYTECODE_TRACE: YIELD_VALUE 1 +713 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +718 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55658 +723 ADD_TO_BYTECODE_TRACE: YIELD_VALUE 1 +724 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +729 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55658 +734 ADD_TO_BYTECODE_TRACE: YIELD_VALUE 1 +735 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +740 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55658 +745 ADD_TO_BYTECODE_TRACE: YIELD_VALUE 1 +746 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +751 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5630e +756 ADD_TO_BYTECODE_TRACE: STORE_FAST 3 +757 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db56310 +762 ADD_TO_BYTECODE_TRACE: JUMP_BACKWARD_JIT 5 +764 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +769 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5630a +FOR_ITER_GEN(3) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +RESUME_CHECK(2) +TIER1_SET_IP(0) +JUMP_BACKWARD_NO_INTERRUPT(5) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +SEND_GEN(3) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +RESUME_CHECK(2) +TIER1_SET_IP(0) +JUMP_BACKWARD_NO_INTERRUPT(5) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +SEND_GEN(3) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +RESUME_CHECK(2) +TIER1_SET_IP(0) +JUMP_BACKWARD_NO_INTERRUPT(5) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +SEND_GEN(3) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +RESUME_CHECK(2) +TIER1_SET_IP(0) +JUMP_BACKWARD_NO_INTERRUPT(5) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +SEND_GEN(3) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +RESUME_CHECK(2) +TIER1_SET_IP(0) +JUMP_BACKWARD_NO_INTERRUPT(5) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +SEND_GEN(3) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +RESUME_CHECK(2) +TIER1_SET_IP(0) +JUMP_BACKWARD_NO_INTERRUPT(5) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +SEND_GEN(3) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +RESUME_CHECK(2) +TIER1_SET_IP(0) +JUMP_BACKWARD_NO_INTERRUPT(5) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +SEND_GEN(3) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +RESUME_CHECK(2) +TIER1_SET_IP(0) +JUMP_BACKWARD_NO_INTERRUPT(5) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +SEND_GEN(3) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +RESUME_CHECK(2) +TIER1_SET_IP(0) +JUMP_BACKWARD_NO_INTERRUPT(5) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +SEND_GEN(3) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +RESUME_CHECK(2) +TIER1_SET_IP(0) +JUMP_BACKWARD_NO_INTERRUPT(5) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +SEND_GEN(3) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +RESUME_CHECK(2) +TIER1_SET_IP(0) +JUMP_BACKWARD_NO_INTERRUPT(5) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +SEND_GEN(3) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +RESUME_CHECK(2) +TIER1_SET_IP(0) +JUMP_BACKWARD_NO_INTERRUPT(5) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +SEND_GEN(3) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +RESUME_CHECK(2) +TIER1_SET_IP(0) +JUMP_BACKWARD_NO_INTERRUPT(5) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +SEND_GEN(3) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +RESUME_CHECK(2) +TIER1_SET_IP(0) +JUMP_BACKWARD_NO_INTERRUPT(5) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +SEND_GEN(3) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +RESUME_CHECK(2) +TIER1_SET_IP(0) +JUMP_BACKWARD_NO_INTERRUPT(5) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +SEND_GEN(3) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +RESUME_CHECK(2) +TIER1_SET_IP(0) +JUMP_BACKWARD_NO_INTERRUPT(5) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +SEND_GEN(3) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +RESUME_CHECK(5) +TIER1_SET_IP(0) +POP_TOP(0) +TIER1_SET_IP(0) +LOAD_FAST_BORROW(0) +TIER1_SET_IP(0) +LOAD_ATTR_INSTANCE_VALUE(4) +TIER1_SET_IP(0) +TO_BOOL_NONE(0) +TIER1_SET_IP(0) +POP_JUMP_IF_FALSE(23) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +LOAD_CONST(0) +TIER1_SET_IP(0) +RETURN_VALUE(0) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +END_SEND(0) +TIER1_SET_IP(0) +POP_TOP(0) +TIER1_SET_IP(0) +LOAD_FAST_BORROW(0) +TIER1_SET_IP(0) +LOAD_ATTR_INSTANCE_VALUE(2) +TIER1_SET_IP(0) +YIELD_VALUE(0) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +YIELD_VALUE(1) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +YIELD_VALUE(1) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +YIELD_VALUE(1) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +YIELD_VALUE(1) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +YIELD_VALUE(1) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +YIELD_VALUE(1) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +YIELD_VALUE(1) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +YIELD_VALUE(1) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +YIELD_VALUE(1) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +YIELD_VALUE(1) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +YIELD_VALUE(1) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +YIELD_VALUE(1) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +YIELD_VALUE(1) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +YIELD_VALUE(1) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +YIELD_VALUE(1) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +STORE_FAST(3) +TIER1_SET_IP(0) +JUMP_BACKWARD_JIT(5) +TIER1_GUARD_IP(0) +TIER1_SET_IP(0) +Optimizing bench_generators (/home/ken/Documents/GitHub/cpython/../bm_generators.py:36) at byte offset 186 + 1 ADD_TO_TRACE: _START_EXECUTOR (0, target=93, operand0=0x71448db5630a, operand1=0) + 2 ADD_TO_TRACE: _MAKE_WARM (0, target=0, operand0=0, operand1=0) +93: FOR_ITER_GEN(3) + 3 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=93, operand0=0, operand1=0) + 4 ADD_TO_TRACE: _SET_IP (0, target=93, operand0=0x71448db5630a, operand1=0) + 5 ADD_TO_TRACE: _FOR_ITER (3, target=93, operand0=0, operand1=0, error_target=0) +93: TIER1_GUARD_IP(0) + 6 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=93, operand0=0, operand1=0) + 7 ADD_TO_TRACE: _GUARD_IP (0, target=37, operand0=0x71448db5565a, operand1=0) +37: RESUME_CHECK(2) + 8 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=37, operand0=0, operand1=0) + 9 ADD_TO_TRACE: _SET_IP (0, target=37, operand0=0x71448db5565a, operand1=0) + 10 ADD_TO_TRACE: _RESUME_CHECK (2, target=37, operand0=0, operand1=0) +37: TIER1_SET_IP(0) + 11 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=37, operand0=0, operand1=0) + 12 ADD_TO_TRACE: _SET_IP (0, target=37, operand0=0x71448db5565a, operand1=0) +38: JUMP_BACKWARD_NO_INTERRUPT(5) + 13 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=38, operand0=0, operand1=0) + 14 ADD_TO_TRACE: _SET_IP (0, target=38, operand0=0x71448db5565c, operand1=0) + 15 ADD_TO_TRACE: _CHECK_PERIODIC (0, target=38, operand0=0, operand1=0, error_target=0) +38: TIER1_GUARD_IP(0) + 16 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=38, operand0=0, operand1=0) + 17 ADD_TO_TRACE: _GUARD_IP (0, target=34, operand0=0x71448db55654, operand1=0) +34: SEND_GEN(3) + 18 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=34, operand0=0, operand1=0) + 19 ADD_TO_TRACE: _SET_IP (0, target=34, operand0=0x71448db55654, operand1=0) + 20 ADD_TO_TRACE: _CHECK_PEP_523 (3, target=34, operand0=0, operand1=0) + 21 ADD_TO_TRACE: _SEND_GEN_FRAME (3, target=34, operand0=0, operand1=0) + 22 ADD_TO_TRACE: _PUSH_FRAME (3, target=34, operand0=0, operand1=0) +34: TIER1_GUARD_IP(0) + 23 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=34, operand0=0, operand1=0) + 24 ADD_TO_TRACE: _GUARD_IP (0, target=37, operand0=0x71448db5565a, operand1=0) +37: RESUME_CHECK(2) + 25 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=37, operand0=0, operand1=0) + 26 ADD_TO_TRACE: _SET_IP (0, target=37, operand0=0x71448db5565a, operand1=0) + 27 ADD_TO_TRACE: _RESUME_CHECK (2, target=37, operand0=0, operand1=0) +37: TIER1_SET_IP(0) + 28 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=37, operand0=0, operand1=0) + 29 ADD_TO_TRACE: _SET_IP (0, target=37, operand0=0x71448db5565a, operand1=0) +38: JUMP_BACKWARD_NO_INTERRUPT(5) + 30 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=38, operand0=0, operand1=0) + 31 ADD_TO_TRACE: _SET_IP (0, target=38, operand0=0x71448db5565c, operand1=0) + 32 ADD_TO_TRACE: _CHECK_PERIODIC (0, target=38, operand0=0, operand1=0, error_target=0) +38: TIER1_GUARD_IP(0) + 33 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=38, operand0=0, operand1=0) + 34 ADD_TO_TRACE: _GUARD_IP (0, target=34, operand0=0x71448db55654, operand1=0) +34: SEND_GEN(3) + 35 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=34, operand0=0, operand1=0) + 36 ADD_TO_TRACE: _SET_IP (0, target=34, operand0=0x71448db55654, operand1=0) + 37 ADD_TO_TRACE: _CHECK_PEP_523 (3, target=34, operand0=0, operand1=0) + 38 ADD_TO_TRACE: _SEND_GEN_FRAME (3, target=34, operand0=0, operand1=0) + 39 ADD_TO_TRACE: _PUSH_FRAME (3, target=34, operand0=0, operand1=0) +34: TIER1_GUARD_IP(0) + 40 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=34, operand0=0, operand1=0) + 41 ADD_TO_TRACE: _GUARD_IP (0, target=37, operand0=0x71448db5565a, operand1=0) +37: RESUME_CHECK(2) + 42 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=37, operand0=0, operand1=0) + 43 ADD_TO_TRACE: _SET_IP (0, target=37, operand0=0x71448db5565a, operand1=0) + 44 ADD_TO_TRACE: _RESUME_CHECK (2, target=37, operand0=0, operand1=0) +37: TIER1_SET_IP(0) + 45 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=37, operand0=0, operand1=0) + 46 ADD_TO_TRACE: _SET_IP (0, target=37, operand0=0x71448db5565a, operand1=0) +38: JUMP_BACKWARD_NO_INTERRUPT(5) + 47 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=38, operand0=0, operand1=0) + 48 ADD_TO_TRACE: _SET_IP (0, target=38, operand0=0x71448db5565c, operand1=0) + 49 ADD_TO_TRACE: _CHECK_PERIODIC (0, target=38, operand0=0, operand1=0, error_target=0) +38: TIER1_GUARD_IP(0) + 50 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=38, operand0=0, operand1=0) + 51 ADD_TO_TRACE: _GUARD_IP (0, target=34, operand0=0x71448db55654, operand1=0) +34: SEND_GEN(3) + 52 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=34, operand0=0, operand1=0) + 53 ADD_TO_TRACE: _SET_IP (0, target=34, operand0=0x71448db55654, operand1=0) + 54 ADD_TO_TRACE: _CHECK_PEP_523 (3, target=34, operand0=0, operand1=0) + 55 ADD_TO_TRACE: _SEND_GEN_FRAME (3, target=34, operand0=0, operand1=0) + 56 ADD_TO_TRACE: _PUSH_FRAME (3, target=34, operand0=0, operand1=0) +34: TIER1_GUARD_IP(0) + 57 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=34, operand0=0, operand1=0) + 58 ADD_TO_TRACE: _GUARD_IP (0, target=37, operand0=0x71448db5565a, operand1=0) +37: RESUME_CHECK(2) + 59 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=37, operand0=0, operand1=0) + 60 ADD_TO_TRACE: _SET_IP (0, target=37, operand0=0x71448db5565a, operand1=0) + 61 ADD_TO_TRACE: _RESUME_CHECK (2, target=37, operand0=0, operand1=0) +37: TIER1_SET_IP(0) + 62 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=37, operand0=0, operand1=0) + 63 ADD_TO_TRACE: _SET_IP (0, target=37, operand0=0x71448db5565a, operand1=0) +38: JUMP_BACKWARD_NO_INTERRUPT(5) + 64 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=38, operand0=0, operand1=0) + 65 ADD_TO_TRACE: _SET_IP (0, target=38, operand0=0x71448db5565c, operand1=0) + 66 ADD_TO_TRACE: _CHECK_PERIODIC (0, target=38, operand0=0, operand1=0, error_target=0) +38: TIER1_GUARD_IP(0) + 67 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=38, operand0=0, operand1=0) + 68 ADD_TO_TRACE: _GUARD_IP (0, target=34, operand0=0x71448db55654, operand1=0) +34: SEND_GEN(3) + 69 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=34, operand0=0, operand1=0) + 70 ADD_TO_TRACE: _SET_IP (0, target=34, operand0=0x71448db55654, operand1=0) + 71 ADD_TO_TRACE: _CHECK_PEP_523 (3, target=34, operand0=0, operand1=0) + 72 ADD_TO_TRACE: _SEND_GEN_FRAME (3, target=34, operand0=0, operand1=0) + 73 ADD_TO_TRACE: _PUSH_FRAME (3, target=34, operand0=0, operand1=0) +34: TIER1_GUARD_IP(0) + 74 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=34, operand0=0, operand1=0) + 75 ADD_TO_TRACE: _GUARD_IP (0, target=89, operand0=0x71448db556c2, operand1=0) +89: RESUME_CHECK(2) + 76 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=89, operand0=0, operand1=0) + 77 ADD_TO_TRACE: _SET_IP (0, target=89, operand0=0x71448db556c2, operand1=0) + 78 ADD_TO_TRACE: _RESUME_CHECK (2, target=89, operand0=0, operand1=0) +89: TIER1_SET_IP(0) + 79 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=89, operand0=0, operand1=0) + 80 ADD_TO_TRACE: _SET_IP (0, target=89, operand0=0x71448db556c2, operand1=0) +90: JUMP_BACKWARD_NO_INTERRUPT(5) + 81 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=90, operand0=0, operand1=0) + 82 ADD_TO_TRACE: _SET_IP (0, target=90, operand0=0x71448db556c4, operand1=0) + 83 ADD_TO_TRACE: _CHECK_PERIODIC (0, target=90, operand0=0, operand1=0, error_target=0) +90: TIER1_GUARD_IP(0) + 84 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=90, operand0=0, operand1=0) + 85 ADD_TO_TRACE: _GUARD_IP (0, target=86, operand0=0x71448db556bc, operand1=0) +86: SEND_GEN(3) + 86 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=86, operand0=0, operand1=0) + 87 ADD_TO_TRACE: _SET_IP (0, target=86, operand0=0x71448db556bc, operand1=0) + 88 ADD_TO_TRACE: _CHECK_PEP_523 (3, target=86, operand0=0, operand1=0) + 89 ADD_TO_TRACE: _SEND_GEN_FRAME (3, target=86, operand0=0, operand1=0) + 90 ADD_TO_TRACE: _PUSH_FRAME (3, target=86, operand0=0, operand1=0) +86: TIER1_GUARD_IP(0) + 91 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=86, operand0=0, operand1=0) + 92 ADD_TO_TRACE: _GUARD_IP (0, target=37, operand0=0x71448db5565a, operand1=0) +37: RESUME_CHECK(2) + 93 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=37, operand0=0, operand1=0) + 94 ADD_TO_TRACE: _SET_IP (0, target=37, operand0=0x71448db5565a, operand1=0) + 95 ADD_TO_TRACE: _RESUME_CHECK (2, target=37, operand0=0, operand1=0) +37: TIER1_SET_IP(0) + 96 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=37, operand0=0, operand1=0) + 97 ADD_TO_TRACE: _SET_IP (0, target=37, operand0=0x71448db5565a, operand1=0) +38: JUMP_BACKWARD_NO_INTERRUPT(5) + 98 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=38, operand0=0, operand1=0) + 99 ADD_TO_TRACE: _SET_IP (0, target=38, operand0=0x71448db5565c, operand1=0) + 100 ADD_TO_TRACE: _CHECK_PERIODIC (0, target=38, operand0=0, operand1=0, error_target=0) +38: TIER1_GUARD_IP(0) + 101 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=38, operand0=0, operand1=0) + 102 ADD_TO_TRACE: _GUARD_IP (0, target=34, operand0=0x71448db55654, operand1=0) +34: SEND_GEN(3) + 103 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=34, operand0=0, operand1=0) + 104 ADD_TO_TRACE: _SET_IP (0, target=34, operand0=0x71448db55654, operand1=0) + 105 ADD_TO_TRACE: _CHECK_PEP_523 (3, target=34, operand0=0, operand1=0) + 106 ADD_TO_TRACE: _SEND_GEN_FRAME (3, target=34, operand0=0, operand1=0) + 107 ADD_TO_TRACE: _PUSH_FRAME (3, target=34, operand0=0, operand1=0) +34: TIER1_GUARD_IP(0) + 108 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=34, operand0=0, operand1=0) + 109 ADD_TO_TRACE: _GUARD_IP (0, target=89, operand0=0x71448db556c2, operand1=0) +89: RESUME_CHECK(2) + 110 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=89, operand0=0, operand1=0) + 111 ADD_TO_TRACE: _SET_IP (0, target=89, operand0=0x71448db556c2, operand1=0) + 112 ADD_TO_TRACE: _RESUME_CHECK (2, target=89, operand0=0, operand1=0) +89: TIER1_SET_IP(0) + 113 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=89, operand0=0, operand1=0) + 114 ADD_TO_TRACE: _SET_IP (0, target=89, operand0=0x71448db556c2, operand1=0) +90: JUMP_BACKWARD_NO_INTERRUPT(5) + 115 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=90, operand0=0, operand1=0) + 116 ADD_TO_TRACE: _SET_IP (0, target=90, operand0=0x71448db556c4, operand1=0) + 117 ADD_TO_TRACE: _CHECK_PERIODIC (0, target=90, operand0=0, operand1=0, error_target=0) +90: TIER1_GUARD_IP(0) + 118 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=90, operand0=0, operand1=0) + 119 ADD_TO_TRACE: _GUARD_IP (0, target=86, operand0=0x71448db556bc, operand1=0) +86: SEND_GEN(3) + 120 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=86, operand0=0, operand1=0) + 121 ADD_TO_TRACE: _SET_IP (0, target=86, operand0=0x71448db556bc, operand1=0) + 122 ADD_TO_TRACE: _CHECK_PEP_523 (3, target=86, operand0=0, operand1=0) + 123 ADD_TO_TRACE: _SEND_GEN_FRAME (3, target=86, operand0=0, operand1=0) + 124 ADD_TO_TRACE: _PUSH_FRAME (3, target=86, operand0=0, operand1=0) +86: TIER1_GUARD_IP(0) + 125 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=86, operand0=0, operand1=0) + 126 ADD_TO_TRACE: _GUARD_IP (0, target=37, operand0=0x71448db5565a, operand1=0) +37: RESUME_CHECK(2) + 127 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=37, operand0=0, operand1=0) + 128 ADD_TO_TRACE: _SET_IP (0, target=37, operand0=0x71448db5565a, operand1=0) + 129 ADD_TO_TRACE: _RESUME_CHECK (2, target=37, operand0=0, operand1=0) +37: TIER1_SET_IP(0) + 130 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=37, operand0=0, operand1=0) + 131 ADD_TO_TRACE: _SET_IP (0, target=37, operand0=0x71448db5565a, operand1=0) +38: JUMP_BACKWARD_NO_INTERRUPT(5) + 132 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=38, operand0=0, operand1=0) + 133 ADD_TO_TRACE: _SET_IP (0, target=38, operand0=0x71448db5565c, operand1=0) + 134 ADD_TO_TRACE: _CHECK_PERIODIC (0, target=38, operand0=0, operand1=0, error_target=0) +38: TIER1_GUARD_IP(0) + 135 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=38, operand0=0, operand1=0) + 136 ADD_TO_TRACE: _GUARD_IP (0, target=34, operand0=0x71448db55654, operand1=0) +34: SEND_GEN(3) + 137 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=34, operand0=0, operand1=0) + 138 ADD_TO_TRACE: _SET_IP (0, target=34, operand0=0x71448db55654, operand1=0) + 139 ADD_TO_TRACE: _CHECK_PEP_523 (3, target=34, operand0=0, operand1=0) + 140 ADD_TO_TRACE: _SEND_GEN_FRAME (3, target=34, operand0=0, operand1=0) + 141 ADD_TO_TRACE: _PUSH_FRAME (3, target=34, operand0=0, operand1=0) +34: TIER1_GUARD_IP(0) + 142 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=34, operand0=0, operand1=0) + 143 ADD_TO_TRACE: _GUARD_IP (0, target=37, operand0=0x71448db5565a, operand1=0) +37: RESUME_CHECK(2) + 144 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=37, operand0=0, operand1=0) + 145 ADD_TO_TRACE: _SET_IP (0, target=37, operand0=0x71448db5565a, operand1=0) + 146 ADD_TO_TRACE: _RESUME_CHECK (2, target=37, operand0=0, operand1=0) +37: TIER1_SET_IP(0) + 147 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=37, operand0=0, operand1=0) + 148 ADD_TO_TRACE: _SET_IP (0, target=37, operand0=0x71448db5565a, operand1=0) +38: JUMP_BACKWARD_NO_INTERRUPT(5) + 149 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=38, operand0=0, operand1=0) + 150 ADD_TO_TRACE: _SET_IP (0, target=38, operand0=0x71448db5565c, operand1=0) + 151 ADD_TO_TRACE: _CHECK_PERIODIC (0, target=38, operand0=0, operand1=0, error_target=0) +38: TIER1_GUARD_IP(0) + 152 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=38, operand0=0, operand1=0) + 153 ADD_TO_TRACE: _GUARD_IP (0, target=34, operand0=0x71448db55654, operand1=0) +34: SEND_GEN(3) + 154 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=34, operand0=0, operand1=0) + 155 ADD_TO_TRACE: _SET_IP (0, target=34, operand0=0x71448db55654, operand1=0) + 156 ADD_TO_TRACE: _CHECK_PEP_523 (3, target=34, operand0=0, operand1=0) + 157 ADD_TO_TRACE: _SEND_GEN_FRAME (3, target=34, operand0=0, operand1=0) + 158 ADD_TO_TRACE: _PUSH_FRAME (3, target=34, operand0=0, operand1=0) +34: TIER1_GUARD_IP(0) + 159 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=34, operand0=0, operand1=0) + 160 ADD_TO_TRACE: _GUARD_IP (0, target=89, operand0=0x71448db556c2, operand1=0) +89: RESUME_CHECK(2) + 161 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=89, operand0=0, operand1=0) + 162 ADD_TO_TRACE: _SET_IP (0, target=89, operand0=0x71448db556c2, operand1=0) + 163 ADD_TO_TRACE: _RESUME_CHECK (2, target=89, operand0=0, operand1=0) +89: TIER1_SET_IP(0) + 164 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=89, operand0=0, operand1=0) + 165 ADD_TO_TRACE: _SET_IP (0, target=89, operand0=0x71448db556c2, operand1=0) +90: JUMP_BACKWARD_NO_INTERRUPT(5) + 166 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=90, operand0=0, operand1=0) + 167 ADD_TO_TRACE: _SET_IP (0, target=90, operand0=0x71448db556c4, operand1=0) + 168 ADD_TO_TRACE: _CHECK_PERIODIC (0, target=90, operand0=0, operand1=0, error_target=0) +90: TIER1_GUARD_IP(0) + 169 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=90, operand0=0, operand1=0) + 170 ADD_TO_TRACE: _GUARD_IP (0, target=86, operand0=0x71448db556bc, operand1=0) +86: SEND_GEN(3) + 171 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=86, operand0=0, operand1=0) + 172 ADD_TO_TRACE: _SET_IP (0, target=86, operand0=0x71448db556bc, operand1=0) + 173 ADD_TO_TRACE: _CHECK_PEP_523 (3, target=86, operand0=0, operand1=0) + 174 ADD_TO_TRACE: _SEND_GEN_FRAME (3, target=86, operand0=0, operand1=0) + 175 ADD_TO_TRACE: _PUSH_FRAME (3, target=86, operand0=0, operand1=0) +86: TIER1_GUARD_IP(0) + 176 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=86, operand0=0, operand1=0) + 177 ADD_TO_TRACE: _GUARD_IP (0, target=89, operand0=0x71448db556c2, operand1=0) +89: RESUME_CHECK(2) + 178 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=89, operand0=0, operand1=0) + 179 ADD_TO_TRACE: _SET_IP (0, target=89, operand0=0x71448db556c2, operand1=0) + 180 ADD_TO_TRACE: _RESUME_CHECK (2, target=89, operand0=0, operand1=0) +89: TIER1_SET_IP(0) + 181 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=89, operand0=0, operand1=0) + 182 ADD_TO_TRACE: _SET_IP (0, target=89, operand0=0x71448db556c2, operand1=0) +90: JUMP_BACKWARD_NO_INTERRUPT(5) + 183 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=90, operand0=0, operand1=0) + 184 ADD_TO_TRACE: _SET_IP (0, target=90, operand0=0x71448db556c4, operand1=0) + 185 ADD_TO_TRACE: _CHECK_PERIODIC (0, target=90, operand0=0, operand1=0, error_target=0) +90: TIER1_GUARD_IP(0) + 186 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=90, operand0=0, operand1=0) + 187 ADD_TO_TRACE: _GUARD_IP (0, target=86, operand0=0x71448db556bc, operand1=0) +86: SEND_GEN(3) + 188 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=86, operand0=0, operand1=0) + 189 ADD_TO_TRACE: _SET_IP (0, target=86, operand0=0x71448db556bc, operand1=0) + 190 ADD_TO_TRACE: _CHECK_PEP_523 (3, target=86, operand0=0, operand1=0) + 191 ADD_TO_TRACE: _SEND_GEN_FRAME (3, target=86, operand0=0, operand1=0) + 192 ADD_TO_TRACE: _PUSH_FRAME (3, target=86, operand0=0, operand1=0) +86: TIER1_GUARD_IP(0) + 193 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=86, operand0=0, operand1=0) + 194 ADD_TO_TRACE: _GUARD_IP (0, target=89, operand0=0x71448db556c2, operand1=0) +89: RESUME_CHECK(2) + 195 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=89, operand0=0, operand1=0) + 196 ADD_TO_TRACE: _SET_IP (0, target=89, operand0=0x71448db556c2, operand1=0) + 197 ADD_TO_TRACE: _RESUME_CHECK (2, target=89, operand0=0, operand1=0) +89: TIER1_SET_IP(0) + 198 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=89, operand0=0, operand1=0) + 199 ADD_TO_TRACE: _SET_IP (0, target=89, operand0=0x71448db556c2, operand1=0) +90: JUMP_BACKWARD_NO_INTERRUPT(5) + 200 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=90, operand0=0, operand1=0) + 201 ADD_TO_TRACE: _SET_IP (0, target=90, operand0=0x71448db556c4, operand1=0) + 202 ADD_TO_TRACE: _CHECK_PERIODIC (0, target=90, operand0=0, operand1=0, error_target=0) +90: TIER1_GUARD_IP(0) + 203 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=90, operand0=0, operand1=0) + 204 ADD_TO_TRACE: _GUARD_IP (0, target=86, operand0=0x71448db556bc, operand1=0) +86: SEND_GEN(3) + 205 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=86, operand0=0, operand1=0) + 206 ADD_TO_TRACE: _SET_IP (0, target=86, operand0=0x71448db556bc, operand1=0) + 207 ADD_TO_TRACE: _CHECK_PEP_523 (3, target=86, operand0=0, operand1=0) + 208 ADD_TO_TRACE: _SEND_GEN_FRAME (3, target=86, operand0=0, operand1=0) + 209 ADD_TO_TRACE: _PUSH_FRAME (3, target=86, operand0=0, operand1=0) +86: TIER1_GUARD_IP(0) + 210 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=86, operand0=0, operand1=0) + 211 ADD_TO_TRACE: _GUARD_IP (0, target=89, operand0=0x71448db556c2, operand1=0) +89: RESUME_CHECK(2) + 212 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=89, operand0=0, operand1=0) + 213 ADD_TO_TRACE: _SET_IP (0, target=89, operand0=0x71448db556c2, operand1=0) + 214 ADD_TO_TRACE: _RESUME_CHECK (2, target=89, operand0=0, operand1=0) +89: TIER1_SET_IP(0) + 215 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=89, operand0=0, operand1=0) + 216 ADD_TO_TRACE: _SET_IP (0, target=89, operand0=0x71448db556c2, operand1=0) +90: JUMP_BACKWARD_NO_INTERRUPT(5) + 217 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=90, operand0=0, operand1=0) + 218 ADD_TO_TRACE: _SET_IP (0, target=90, operand0=0x71448db556c4, operand1=0) + 219 ADD_TO_TRACE: _CHECK_PERIODIC (0, target=90, operand0=0, operand1=0, error_target=0) +90: TIER1_GUARD_IP(0) + 220 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=90, operand0=0, operand1=0) + 221 ADD_TO_TRACE: _GUARD_IP (0, target=86, operand0=0x71448db556bc, operand1=0) +86: SEND_GEN(3) + 222 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=86, operand0=0, operand1=0) + 223 ADD_TO_TRACE: _SET_IP (0, target=86, operand0=0x71448db556bc, operand1=0) + 224 ADD_TO_TRACE: _CHECK_PEP_523 (3, target=86, operand0=0, operand1=0) + 225 ADD_TO_TRACE: _SEND_GEN_FRAME (3, target=86, operand0=0, operand1=0) + 226 ADD_TO_TRACE: _PUSH_FRAME (3, target=86, operand0=0, operand1=0) +86: TIER1_GUARD_IP(0) + 227 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=86, operand0=0, operand1=0) + 228 ADD_TO_TRACE: _GUARD_IP (0, target=37, operand0=0x71448db5565a, operand1=0) +37: RESUME_CHECK(2) + 229 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=37, operand0=0, operand1=0) + 230 ADD_TO_TRACE: _SET_IP (0, target=37, operand0=0x71448db5565a, operand1=0) + 231 ADD_TO_TRACE: _RESUME_CHECK (2, target=37, operand0=0, operand1=0) +37: TIER1_SET_IP(0) + 232 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=37, operand0=0, operand1=0) + 233 ADD_TO_TRACE: _SET_IP (0, target=37, operand0=0x71448db5565a, operand1=0) +38: JUMP_BACKWARD_NO_INTERRUPT(5) + 234 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=38, operand0=0, operand1=0) + 235 ADD_TO_TRACE: _SET_IP (0, target=38, operand0=0x71448db5565c, operand1=0) + 236 ADD_TO_TRACE: _CHECK_PERIODIC (0, target=38, operand0=0, operand1=0, error_target=0) +38: TIER1_GUARD_IP(0) + 237 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=38, operand0=0, operand1=0) + 238 ADD_TO_TRACE: _GUARD_IP (0, target=34, operand0=0x71448db55654, operand1=0) +34: SEND_GEN(3) + 239 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=34, operand0=0, operand1=0) + 240 ADD_TO_TRACE: _SET_IP (0, target=34, operand0=0x71448db55654, operand1=0) + 241 ADD_TO_TRACE: _CHECK_PEP_523 (3, target=34, operand0=0, operand1=0) + 242 ADD_TO_TRACE: _SEND_GEN_FRAME (3, target=34, operand0=0, operand1=0) + 243 ADD_TO_TRACE: _PUSH_FRAME (3, target=34, operand0=0, operand1=0) +34: TIER1_GUARD_IP(0) + 244 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=34, operand0=0, operand1=0) + 245 ADD_TO_TRACE: _GUARD_IP (0, target=89, operand0=0x71448db556c2, operand1=0) +89: RESUME_CHECK(2) + 246 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=89, operand0=0, operand1=0) + 247 ADD_TO_TRACE: _SET_IP (0, target=89, operand0=0x71448db556c2, operand1=0) + 248 ADD_TO_TRACE: _RESUME_CHECK (2, target=89, operand0=0, operand1=0) +89: TIER1_SET_IP(0) + 249 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=89, operand0=0, operand1=0) + 250 ADD_TO_TRACE: _SET_IP (0, target=89, operand0=0x71448db556c2, operand1=0) +90: JUMP_BACKWARD_NO_INTERRUPT(5) + 251 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=90, operand0=0, operand1=0) + 252 ADD_TO_TRACE: _SET_IP (0, target=90, operand0=0x71448db556c4, operand1=0) + 253 ADD_TO_TRACE: _CHECK_PERIODIC (0, target=90, operand0=0, operand1=0, error_target=0) +90: TIER1_GUARD_IP(0) + 254 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=90, operand0=0, operand1=0) + 255 ADD_TO_TRACE: _GUARD_IP (0, target=86, operand0=0x71448db556bc, operand1=0) +86: SEND_GEN(3) + 256 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=86, operand0=0, operand1=0) + 257 ADD_TO_TRACE: _SET_IP (0, target=86, operand0=0x71448db556bc, operand1=0) + 258 ADD_TO_TRACE: _CHECK_PEP_523 (3, target=86, operand0=0, operand1=0) + 259 ADD_TO_TRACE: _SEND_GEN_FRAME (3, target=86, operand0=0, operand1=0) + 260 ADD_TO_TRACE: _PUSH_FRAME (3, target=86, operand0=0, operand1=0) +86: TIER1_GUARD_IP(0) + 261 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=86, operand0=0, operand1=0) + 262 ADD_TO_TRACE: _GUARD_IP (0, target=37, operand0=0x71448db5565a, operand1=0) +37: RESUME_CHECK(2) + 263 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=37, operand0=0, operand1=0) + 264 ADD_TO_TRACE: _SET_IP (0, target=37, operand0=0x71448db5565a, operand1=0) + 265 ADD_TO_TRACE: _RESUME_CHECK (2, target=37, operand0=0, operand1=0) +37: TIER1_SET_IP(0) + 266 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=37, operand0=0, operand1=0) + 267 ADD_TO_TRACE: _SET_IP (0, target=37, operand0=0x71448db5565a, operand1=0) +38: JUMP_BACKWARD_NO_INTERRUPT(5) + 268 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=38, operand0=0, operand1=0) + 269 ADD_TO_TRACE: _SET_IP (0, target=38, operand0=0x71448db5565c, operand1=0) + 270 ADD_TO_TRACE: _CHECK_PERIODIC (0, target=38, operand0=0, operand1=0, error_target=0) +38: TIER1_GUARD_IP(0) + 271 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=38, operand0=0, operand1=0) + 272 ADD_TO_TRACE: _GUARD_IP (0, target=34, operand0=0x71448db55654, operand1=0) +34: SEND_GEN(3) + 273 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=34, operand0=0, operand1=0) + 274 ADD_TO_TRACE: _SET_IP (0, target=34, operand0=0x71448db55654, operand1=0) + 275 ADD_TO_TRACE: _CHECK_PEP_523 (3, target=34, operand0=0, operand1=0) + 276 ADD_TO_TRACE: _SEND_GEN_FRAME (3, target=34, operand0=0, operand1=0) + 277 ADD_TO_TRACE: _PUSH_FRAME (3, target=34, operand0=0, operand1=0) +34: TIER1_GUARD_IP(0) + 278 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=34, operand0=0, operand1=0) + 279 ADD_TO_TRACE: _GUARD_IP (0, target=53, operand0=0x71448db5567a, operand1=0) +53: RESUME_CHECK(5) + 280 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=53, operand0=0, operand1=0) + 281 ADD_TO_TRACE: _SET_IP (0, target=53, operand0=0x71448db5567a, operand1=0) + 282 ADD_TO_TRACE: _RESUME_CHECK (5, target=53, operand0=0, operand1=0) +53: TIER1_SET_IP(0) + 283 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=53, operand0=0, operand1=0) + 284 ADD_TO_TRACE: _SET_IP (0, target=53, operand0=0x71448db5567a, operand1=0) +54: POP_TOP(0) + 285 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=54, operand0=0, operand1=0) + 286 ADD_TO_TRACE: _SET_IP (0, target=54, operand0=0x71448db5567c, operand1=0) + 287 ADD_TO_TRACE: _POP_TOP (0, target=54, operand0=0, operand1=0) +54: TIER1_SET_IP(0) + 288 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=54, operand0=0, operand1=0) + 289 ADD_TO_TRACE: _SET_IP (0, target=54, operand0=0x71448db5567c, operand1=0) +55: LOAD_FAST_BORROW(0) + 290 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=55, operand0=0, operand1=0) + 291 ADD_TO_TRACE: _SET_IP (0, target=55, operand0=0x71448db5567e, operand1=0) + 292 ADD_TO_TRACE: _LOAD_FAST_BORROW (0, target=55, operand0=0, operand1=0) +55: TIER1_SET_IP(0) + 293 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=55, operand0=0, operand1=0) + 294 ADD_TO_TRACE: _SET_IP (0, target=55, operand0=0x71448db5567e, operand1=0) +56: LOAD_ATTR_INSTANCE_VALUE(4) + 295 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=56, operand0=0, operand1=0) + 296 ADD_TO_TRACE: _SET_IP (0, target=56, operand0=0x71448db55680, operand1=0) + 297 ADD_TO_TRACE: _GUARD_TYPE_VERSION (4, target=56, operand0=0x2004a, operand1=0) + 298 ADD_TO_TRACE: _CHECK_MANAGED_OBJECT_HAS_VALUES (4, target=56, operand0=0, operand1=0) + 299 ADD_TO_TRACE: _LOAD_ATTR_INSTANCE_VALUE (4, target=56, operand0=0x20, operand1=0) + 300 ADD_TO_TRACE: _PUSH_NULL_CONDITIONAL (4, target=56, operand0=0, operand1=0) +56: TIER1_SET_IP(0) + 301 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=56, operand0=0, operand1=0) + 302 ADD_TO_TRACE: _SET_IP (0, target=56, operand0=0x71448db55680, operand1=0) +66: TO_BOOL_NONE(0) + 303 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=66, operand0=0, operand1=0) + 304 ADD_TO_TRACE: _SET_IP (0, target=66, operand0=0x71448db55694, operand1=0) + 305 ADD_TO_TRACE: _TO_BOOL_NONE (0, target=66, operand0=0, operand1=0) +66: TIER1_SET_IP(0) + 306 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=66, operand0=0, operand1=0) + 307 ADD_TO_TRACE: _SET_IP (0, target=66, operand0=0x71448db55694, operand1=0) +70: POP_JUMP_IF_FALSE(23) + 308 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=70, operand0=0, operand1=0) + 309 ADD_TO_TRACE: _SET_IP (0, target=70, operand0=0x71448db5569c, operand1=0) + 310 ADD_TO_TRACE: _POP_JUMP_IF_FALSE (23, target=70, operand0=0, operand1=0) +70: TIER1_GUARD_IP(0) + 311 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=70, operand0=0, operand1=0) + 312 ADD_TO_TRACE: _GUARD_IP (0, target=95, operand0=0x71448db556ce, operand1=0) +95: LOAD_CONST(0) + 313 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=95, operand0=0, operand1=0) + 314 ADD_TO_TRACE: _SET_IP (0, target=95, operand0=0x71448db556ce, operand1=0) + 315 ADD_TO_TRACE: _LOAD_CONST (0, target=95, operand0=0, operand1=0) +95: TIER1_SET_IP(0) + 316 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=95, operand0=0, operand1=0) + 317 ADD_TO_TRACE: _SET_IP (0, target=95, operand0=0x71448db556ce, operand1=0) +96: RETURN_VALUE(0) + 318 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=96, operand0=0, operand1=0) + 319 ADD_TO_TRACE: _SET_IP (0, target=96, operand0=0x71448db556d0, operand1=0) + 320 ADD_TO_TRACE: _RETURN_VALUE (0, target=96, operand0=0, operand1=0) +96: TIER1_GUARD_IP(0) + 321 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=96, operand0=0, operand1=0) + 322 ADD_TO_TRACE: _GUARD_IP (0, target=39, operand0=0x71448db5565e, operand1=0) +39: END_SEND(0) + 323 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=39, operand0=0, operand1=0) + 324 ADD_TO_TRACE: _SET_IP (0, target=39, operand0=0x71448db5565e, operand1=0) + 325 ADD_TO_TRACE: _END_SEND (0, target=39, operand0=0, operand1=0) +39: TIER1_SET_IP(0) + 326 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=39, operand0=0, operand1=0) + 327 ADD_TO_TRACE: _SET_IP (0, target=39, operand0=0x71448db5565e, operand1=0) +40: POP_TOP(0) + 328 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=40, operand0=0, operand1=0) + 329 ADD_TO_TRACE: _SET_IP (0, target=40, operand0=0x71448db55660, operand1=0) + 330 ADD_TO_TRACE: _POP_TOP (0, target=40, operand0=0, operand1=0) +40: TIER1_SET_IP(0) + 331 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=40, operand0=0, operand1=0) + 332 ADD_TO_TRACE: _SET_IP (0, target=40, operand0=0x71448db55660, operand1=0) +41: LOAD_FAST_BORROW(0) + 333 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=41, operand0=0, operand1=0) + 334 ADD_TO_TRACE: _SET_IP (0, target=41, operand0=0x71448db55662, operand1=0) + 335 ADD_TO_TRACE: _LOAD_FAST_BORROW (0, target=41, operand0=0, operand1=0) +41: TIER1_SET_IP(0) + 336 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=41, operand0=0, operand1=0) + 337 ADD_TO_TRACE: _SET_IP (0, target=41, operand0=0x71448db55662, operand1=0) +42: LOAD_ATTR_INSTANCE_VALUE(2) + 338 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=42, operand0=0, operand1=0) + 339 ADD_TO_TRACE: _SET_IP (0, target=42, operand0=0x71448db55664, operand1=0) + 340 ADD_TO_TRACE: _GUARD_TYPE_VERSION (2, target=42, operand0=0x2004a, operand1=0) + 341 ADD_TO_TRACE: _CHECK_MANAGED_OBJECT_HAS_VALUES (2, target=42, operand0=0, operand1=0) + 342 ADD_TO_TRACE: _LOAD_ATTR_INSTANCE_VALUE (2, target=42, operand0=0x28, operand1=0) + 343 ADD_TO_TRACE: _PUSH_NULL_CONDITIONAL (2, target=42, operand0=0, operand1=0) +42: TIER1_SET_IP(0) + 344 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=42, operand0=0, operand1=0) + 345 ADD_TO_TRACE: _SET_IP (0, target=42, operand0=0x71448db55664, operand1=0) +52: YIELD_VALUE(0) + 346 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=52, operand0=0, operand1=0) + 347 ADD_TO_TRACE: _SET_IP (0, target=52, operand0=0x71448db55678, operand1=0) + 348 ADD_TO_TRACE: _YIELD_VALUE (0, target=52, operand0=0, operand1=0) +52: TIER1_GUARD_IP(0) + 349 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=52, operand0=0, operand1=0) + 350 ADD_TO_TRACE: _GUARD_IP (0, target=88, operand0=0x71448db556c0, operand1=0) +88: YIELD_VALUE(1) + 351 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=88, operand0=0, operand1=0) + 352 ADD_TO_TRACE: _SET_IP (0, target=88, operand0=0x71448db556c0, operand1=0) + 353 ADD_TO_TRACE: _YIELD_VALUE (1, target=88, operand0=0, operand1=0) +88: TIER1_GUARD_IP(0) + 354 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=88, operand0=0, operand1=0) + 355 ADD_TO_TRACE: _GUARD_IP (0, target=36, operand0=0x71448db55658, operand1=0) +36: YIELD_VALUE(1) + 356 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=36, operand0=0, operand1=0) + 357 ADD_TO_TRACE: _SET_IP (0, target=36, operand0=0x71448db55658, operand1=0) + 358 ADD_TO_TRACE: _YIELD_VALUE (1, target=36, operand0=0, operand1=0) +36: TIER1_GUARD_IP(0) + 359 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=36, operand0=0, operand1=0) + 360 ADD_TO_TRACE: _GUARD_IP (0, target=88, operand0=0x71448db556c0, operand1=0) +88: YIELD_VALUE(1) + 361 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=88, operand0=0, operand1=0) + 362 ADD_TO_TRACE: _SET_IP (0, target=88, operand0=0x71448db556c0, operand1=0) + 363 ADD_TO_TRACE: _YIELD_VALUE (1, target=88, operand0=0, operand1=0) +88: TIER1_GUARD_IP(0) + 364 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=88, operand0=0, operand1=0) + 365 ADD_TO_TRACE: _GUARD_IP (0, target=88, operand0=0x71448db556c0, operand1=0) +88: YIELD_VALUE(1) + 366 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=88, operand0=0, operand1=0) + 367 ADD_TO_TRACE: _SET_IP (0, target=88, operand0=0x71448db556c0, operand1=0) + 368 ADD_TO_TRACE: _YIELD_VALUE (1, target=88, operand0=0, operand1=0) +88: TIER1_GUARD_IP(0) + 369 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=88, operand0=0, operand1=0) + 370 ADD_TO_TRACE: _GUARD_IP (0, target=88, operand0=0x71448db556c0, operand1=0) +88: YIELD_VALUE(1) + 371 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=88, operand0=0, operand1=0) + 372 ADD_TO_TRACE: _SET_IP (0, target=88, operand0=0x71448db556c0, operand1=0) + 373 ADD_TO_TRACE: _YIELD_VALUE (1, target=88, operand0=0, operand1=0) +88: TIER1_GUARD_IP(0) + 374 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=88, operand0=0, operand1=0) + 375 ADD_TO_TRACE: _GUARD_IP (0, target=88, operand0=0x71448db556c0, operand1=0) +88: YIELD_VALUE(1) + 376 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=88, operand0=0, operand1=0) + 377 ADD_TO_TRACE: _SET_IP (0, target=88, operand0=0x71448db556c0, operand1=0) + 378 ADD_TO_TRACE: _YIELD_VALUE (1, target=88, operand0=0, operand1=0) +88: TIER1_GUARD_IP(0) + 379 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=88, operand0=0, operand1=0) + 380 ADD_TO_TRACE: _GUARD_IP (0, target=36, operand0=0x71448db55658, operand1=0) +36: YIELD_VALUE(1) + 381 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=36, operand0=0, operand1=0) + 382 ADD_TO_TRACE: _SET_IP (0, target=36, operand0=0x71448db55658, operand1=0) + 383 ADD_TO_TRACE: _YIELD_VALUE (1, target=36, operand0=0, operand1=0) +36: TIER1_GUARD_IP(0) + 384 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=36, operand0=0, operand1=0) + 385 ADD_TO_TRACE: _GUARD_IP (0, target=36, operand0=0x71448db55658, operand1=0) +36: YIELD_VALUE(1) + 386 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=36, operand0=0, operand1=0) + 387 ADD_TO_TRACE: _SET_IP (0, target=36, operand0=0x71448db55658, operand1=0) + 388 ADD_TO_TRACE: _YIELD_VALUE (1, target=36, operand0=0, operand1=0) +36: TIER1_GUARD_IP(0) + 389 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=36, operand0=0, operand1=0) + 390 ADD_TO_TRACE: _GUARD_IP (0, target=88, operand0=0x71448db556c0, operand1=0) +88: YIELD_VALUE(1) + 391 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=88, operand0=0, operand1=0) + 392 ADD_TO_TRACE: _SET_IP (0, target=88, operand0=0x71448db556c0, operand1=0) + 393 ADD_TO_TRACE: _YIELD_VALUE (1, target=88, operand0=0, operand1=0) +88: TIER1_GUARD_IP(0) + 394 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=88, operand0=0, operand1=0) + 395 ADD_TO_TRACE: _GUARD_IP (0, target=36, operand0=0x71448db55658, operand1=0) +36: YIELD_VALUE(1) + 396 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=36, operand0=0, operand1=0) + 397 ADD_TO_TRACE: _SET_IP (0, target=36, operand0=0x71448db55658, operand1=0) + 398 ADD_TO_TRACE: _YIELD_VALUE (1, target=36, operand0=0, operand1=0) +36: TIER1_GUARD_IP(0) + 399 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=36, operand0=0, operand1=0) + 400 ADD_TO_TRACE: _GUARD_IP (0, target=88, operand0=0x71448db556c0, operand1=0) +88: YIELD_VALUE(1) + 401 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=88, operand0=0, operand1=0) + 402 ADD_TO_TRACE: _SET_IP (0, target=88, operand0=0x71448db556c0, operand1=0) + 403 ADD_TO_TRACE: _YIELD_VALUE (1, target=88, operand0=0, operand1=0) +88: TIER1_GUARD_IP(0) + 404 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=88, operand0=0, operand1=0) + 405 ADD_TO_TRACE: _GUARD_IP (0, target=36, operand0=0x71448db55658, operand1=0) +36: YIELD_VALUE(1) + 406 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=36, operand0=0, operand1=0) + 407 ADD_TO_TRACE: _SET_IP (0, target=36, operand0=0x71448db55658, operand1=0) + 408 ADD_TO_TRACE: _YIELD_VALUE (1, target=36, operand0=0, operand1=0) +36: TIER1_GUARD_IP(0) + 409 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=36, operand0=0, operand1=0) + 410 ADD_TO_TRACE: _GUARD_IP (0, target=36, operand0=0x71448db55658, operand1=0) +36: YIELD_VALUE(1) + 411 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=36, operand0=0, operand1=0) + 412 ADD_TO_TRACE: _SET_IP (0, target=36, operand0=0x71448db55658, operand1=0) + 413 ADD_TO_TRACE: _YIELD_VALUE (1, target=36, operand0=0, operand1=0) +36: TIER1_GUARD_IP(0) + 414 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=36, operand0=0, operand1=0) + 415 ADD_TO_TRACE: _GUARD_IP (0, target=36, operand0=0x71448db55658, operand1=0) +36: YIELD_VALUE(1) + 416 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=36, operand0=0, operand1=0) + 417 ADD_TO_TRACE: _SET_IP (0, target=36, operand0=0x71448db55658, operand1=0) + 418 ADD_TO_TRACE: _YIELD_VALUE (1, target=36, operand0=0, operand1=0) +36: TIER1_GUARD_IP(0) + 419 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=36, operand0=0, operand1=0) + 420 ADD_TO_TRACE: _GUARD_IP (0, target=36, operand0=0x71448db55658, operand1=0) +36: YIELD_VALUE(1) + 421 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=36, operand0=0, operand1=0) + 422 ADD_TO_TRACE: _SET_IP (0, target=36, operand0=0x71448db55658, operand1=0) + 423 ADD_TO_TRACE: _YIELD_VALUE (1, target=36, operand0=0, operand1=0) +36: TIER1_GUARD_IP(0) + 424 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=36, operand0=0, operand1=0) + 425 ADD_TO_TRACE: _GUARD_IP (0, target=95, operand0=0x71448db5630e, operand1=0) +95: STORE_FAST(3) + 426 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=95, operand0=0, operand1=0) + 427 ADD_TO_TRACE: _SET_IP (0, target=95, operand0=0x71448db5630e, operand1=0) + 428 ADD_TO_TRACE: _STORE_FAST (3, target=95, operand0=0, operand1=0) +95: TIER1_SET_IP(0) + 429 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=95, operand0=0, operand1=0) + 430 ADD_TO_TRACE: _SET_IP (0, target=95, operand0=0x71448db5630e, operand1=0) +96: JUMP_BACKWARD_JIT(5) + 431 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=96, operand0=0, operand1=0) + 432 ADD_TO_TRACE: _SET_IP (0, target=96, operand0=0x71448db56310, operand1=0) + 433 ADD_TO_TRACE: _CHECK_PERIODIC (0, target=96, operand0=0, operand1=0, error_target=0) +96: TIER1_GUARD_IP(0) + 434 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=96, operand0=0, operand1=0) + 435 ADD_TO_TRACE: _GUARD_IP (0, target=93, operand0=0x71448db5630a, operand1=0) + 436 ADD_TO_TRACE: _JUMP_TO_TOP (0, target=0, operand0=0, operand1=0) +Created a proto-trace for bench_generators (/home/ken/Documents/GitHub/cpython/../bm_generators.py:36) at byte offset 186 -- length 436 + locals=[, , , ] + stack=[, , , ] + 0 uop: _START_EXECUTOR (0, jump_target=238, operand0=0x64d708ad28e0, operand1=0) + locals=[, , , ] + stack=[, , , ] + 1 uop: _MAKE_WARM (0, target=0, operand0=0, operand1=0) + locals=[, , , ] + stack=[, , , ] + 2 uop: _SET_IP (0, target=93, operand0=0x71448db5630a, operand1=0) + locals=[, , , ] + stack=[, , , ] + 3 uop: _FOR_ITER (3, jump_target=0, operand0=0, operand1=0, error_target=239) + locals=[, , , ] + stack=[, , , , ] + 4 uop: _CHECK_VALIDITY (0, jump_target=238, operand0=0, operand1=0) + locals=[, , , ] + stack=[, , , , ] + 5 uop: _GUARD_IP (0, target=37, operand0=0x71448db5565a, operand1=0) +0 ADD_TO_BYTECODE_TRACE: STORE_FAST 3 +1 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db56310 +6 ADD_TO_BYTECODE_TRACE: JUMP_BACKWARD_JIT 5 +8 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 +13 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5630a +STORE_FAST(3) +TIER1_SET_IP(3) +JUMP_BACKWARD_JIT(5) +TIER1_GUARD_IP(86) +TIER1_SET_IP(0) +Optimizing bench_generators (/home/ken/Documents/GitHub/cpython/../bm_generators.py:36) at byte offset 186 + 1 ADD_TO_TRACE: _START_EXECUTOR (0, target=93, operand0=0x71448db5630a, operand1=0) + 2 ADD_TO_TRACE: _MAKE_WARM (0, target=0, operand0=0, operand1=0) +93: STORE_FAST(3) + 3 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=93, operand0=0, operand1=0) + 4 ADD_TO_TRACE: _SET_IP (0, target=93, operand0=0x71448db5630a, operand1=0) + 5 ADD_TO_TRACE: _STORE_FAST (3, target=93, operand0=0, operand1=0) +93: TIER1_SET_IP(3) + 6 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=93, operand0=0, operand1=0) + 7 ADD_TO_TRACE: _SET_IP (0, target=93, operand0=0x71448db5630a, operand1=0) +96: JUMP_BACKWARD_JIT(5) + 8 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=96, operand0=0, operand1=0) + 9 ADD_TO_TRACE: _SET_IP (0, target=96, operand0=0x71448db56310, operand1=0) + 10 ADD_TO_TRACE: _CHECK_PERIODIC (0, target=96, operand0=0, operand1=0, error_target=0) +96: TIER1_GUARD_IP(86) + 11 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=96, operand0=0, operand1=0) + 12 ADD_TO_TRACE: _GUARD_IP (0, target=93, operand0=0x71448db5630a, operand1=0) + 13 ADD_TO_TRACE: _JUMP_TO_TOP (0, target=0, operand0=0, operand1=0) +Created a proto-trace for bench_generators (/home/ken/Documents/GitHub/cpython/../bm_generators.py:36) at byte offset 186 -- length 13 + locals=[, , , ] + stack=[, , , ] + 0 uop: _START_EXECUTOR (0, jump_target=10, operand0=0x71448db814f0, operand1=0) + locals=[, , , ] + stack=[, , , ] + 1 uop: _MAKE_WARM (0, target=0, operand0=0, operand1=0) + locals=[, , , ] + stack=[, , , ] + 2 uop: _SET_IP (0, target=93, operand0=0x71448db5630a, operand1=0) + locals=[, , , ] + stack=[, , , ] + 3 uop: _STORE_FAST_3 (3, target=93, operand0=0, operand1=0) + locals=[, , , ] + stack=[, , ] + 4 uop: _CHECK_VALIDITY (0, jump_target=10, operand0=0, operand1=0) + locals=[, , , ] + stack=[, , ] + 5 uop: _SET_IP (0, target=96, operand0=0x71448db56310, operand1=0) + locals=[, , , ] + stack=[, , ] + 6 uop: _CHECK_PERIODIC (0, jump_target=0, operand0=0, operand1=0, error_target=11) + locals=[, , , ] + stack=[, , ] + 7 uop: _CHECK_VALIDITY (0, jump_target=12, operand0=0, operand1=0) + locals=[, , , ] + stack=[, , ] From 13188a9423137527cd9479d701d82a0fd3f17a26 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 19 Sep 2025 22:31:24 +0100 Subject: [PATCH 003/190] refactor to translate on the go --- Include/internal/pycore_interp_structs.h | 4 +- Include/internal/pycore_magic_number.h | 3 +- Include/internal/pycore_opcode_metadata.h | 20 +- Include/internal/pycore_optimizer.h | 19 +- Include/internal/pycore_uop.h | 11 +- Include/internal/pycore_uop_metadata.h | 2 +- Include/opcode_ids.h | 168 ++++--- Lib/_opcode_metadata.py | 168 ++++--- Python/bytecodes.c | 33 +- Python/ceval.c | 95 +--- Python/ceval_macros.h | 2 +- Python/executor_cases.c.h | 11 +- Python/generated_cases.c.h | 104 +---- Python/opcode_targets.h | 20 +- Python/optimizer.c | 510 ++++++++++------------ Python/pystate.c | 2 +- 16 files changed, 450 insertions(+), 722 deletions(-) diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 2cd61c215ce01d..5834a6f8edda3a 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -944,8 +944,10 @@ struct _is { struct callable_cache callable_cache; PyObject *common_consts[NUM_COMMON_CONSTANTS]; // JIT tracing state + int jit_tracer_code_max_size; int jit_tracer_code_curr_size; - _Py_CODEUNIT *jit_tracer_code_buffer; + _PyBloomFilter jit_tracer_dependencies; + _PyUOpInstruction *jit_tracer_code_buffer; _Py_CODEUNIT *jit_tracer_initial_instr; int jit_tracer_initial_stack_depth; int jit_tracer_initial_chain_depth; diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index e1f5d29d2b8cc1..7ec7bd1c695516 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -286,7 +286,6 @@ Known values: Python 3.15a1 3653 (Fix handling of opcodes that may leave operands on the stack when optimizing LOAD_FAST) Python 3.15a1 3654 (Fix missing exception handlers in logical expression) Python 3.15a1 3655 (Fix miscompilation of some module-level annotations) - Python 3.15a1 3656 (Add GUARD_IP instruction) Python 3.16 will start with 3700 @@ -300,7 +299,7 @@ PC/launcher.c must also be updated. */ -#define PYC_MAGIC_NUMBER 3656 +#define PYC_MAGIC_NUMBER 3655 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 97642c2ff4678b..0c3e33df11913d 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -474,10 +474,6 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 3; case SWAP: return 2 + (oparg-2); - case TIER1_GUARD_IP: - return 0; - case TIER1_SET_IP: - return 0; case TO_BOOL: return 1; case TO_BOOL_ALWAYS_TRUE: @@ -961,10 +957,6 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 0; case SWAP: return 2 + (oparg-2); - case TIER1_GUARD_IP: - return 0; - case TIER1_SET_IP: - return 0; case TO_BOOL: return 1; case TO_BOOL_ALWAYS_TRUE: @@ -1284,8 +1276,6 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [STORE_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [SWAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, - [TIER1_GUARD_IP] = { true, INSTR_FMT_IXC000, HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG }, - [TIER1_SET_IP] = { true, INSTR_FMT_IXC000, HAS_ESCAPES_FLAG }, [TO_BOOL] = { true, INSTR_FMT_IXC00, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [TO_BOOL_ALWAYS_TRUE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG }, @@ -1736,8 +1726,6 @@ const char *_PyOpcode_OpName[267] = { [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", [SWAP] = "SWAP", - [TIER1_GUARD_IP] = "TIER1_GUARD_IP", - [TIER1_SET_IP] = "TIER1_SET_IP", [TO_BOOL] = "TO_BOOL", [TO_BOOL_ALWAYS_TRUE] = "TO_BOOL_ALWAYS_TRUE", [TO_BOOL_BOOL] = "TO_BOOL_BOOL", @@ -1761,8 +1749,6 @@ const char *_PyOpcode_OpName[267] = { extern const uint8_t _PyOpcode_Caches[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Caches[256] = { - [TIER1_GUARD_IP] = 4, - [TIER1_SET_IP] = 4, [TO_BOOL] = 3, [STORE_SUBSCR] = 1, [SEND] = 1, @@ -1828,6 +1814,8 @@ const uint8_t _PyOpcode_NeedsGuardIp[256] = { extern const uint8_t _PyOpcode_Deopt[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Deopt[256] = { + [121] = 121, + [122] = 122, [123] = 123, [124] = 124, [125] = 125, @@ -2065,8 +2053,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [STORE_SUBSCR_DICT] = STORE_SUBSCR, [STORE_SUBSCR_LIST_INT] = STORE_SUBSCR, [SWAP] = SWAP, - [TIER1_GUARD_IP] = TIER1_GUARD_IP, - [TIER1_SET_IP] = TIER1_SET_IP, [TO_BOOL] = TO_BOOL, [TO_BOOL_ALWAYS_TRUE] = TO_BOOL, [TO_BOOL_BOOL] = TO_BOOL, @@ -2089,6 +2075,8 @@ const uint8_t _PyOpcode_Deopt[256] = { #endif // NEED_OPCODE_METADATA #define EXTRA_CASES \ + case 121: \ + case 122: \ case 123: \ case 124: \ case 125: \ diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 9bfd301efecc9c..56224f8a73e4b4 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -21,14 +21,6 @@ typedef struct _PyExecutorLinkListNode { } _PyExecutorLinkListNode; -/* Bloom filter with m = 256 - * https://en.wikipedia.org/wiki/Bloom_filter */ -#define _Py_BLOOM_FILTER_WORDS 8 - -typedef struct { - uint32_t bits[_Py_BLOOM_FILTER_WORDS]; -} _PyBloomFilter; - typedef struct { uint8_t opcode; uint8_t oparg; @@ -364,6 +356,17 @@ PyAPI_FUNC(int) _PyDumpExecutors(FILE *out); extern void _Py_ClearExecutorDeletionList(PyInterpreterState *interp); #endif +int +_PyJIT_translate_single_bytecode_to_trace( + PyThreadState *tstate, + _Py_CODEUNIT *this_instr, + _Py_CODEUNIT *next_instr, + PyCodeObject *code, + PyFunctionObject *func, + int oparg); + +void +_PyJIT_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, int curr_stackdepth, int chain_depth); #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_uop.h b/Include/internal/pycore_uop.h index 301779905cf4c5..6effe12915423a 100644 --- a/Include/internal/pycore_uop.h +++ b/Include/internal/pycore_uop.h @@ -39,10 +39,13 @@ typedef struct _PyUOpInstruction{ #define UOP_MAX_TRACE_LENGTH 1200 #define UOP_BUFFER_SIZE (UOP_MAX_TRACE_LENGTH * sizeof(_PyUOpInstruction)) -// This is the length of the trace we record. -// This includes the inline caches. -#define TRACE_MAX_TRACE_LENGTH 2000 -#define TRACER_BUFFER_SIZE ((int)(TRACE_MAX_TRACE_LENGTH * sizeof(_Py_CODEUNIT))) +/* Bloom filter with m = 256 + * https://en.wikipedia.org/wiki/Bloom_filter */ +#define _Py_BLOOM_FILTER_WORDS 8 + +typedef struct { + uint32_t bits[_Py_BLOOM_FILTER_WORDS]; +} _PyBloomFilter; #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index f8099301b6b4b4..973093acf8aaa3 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -341,7 +341,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_ERROR_POP_N] = HAS_ARG_FLAG, [_TIER2_RESUME_CHECK] = HAS_PERIODIC_FLAG, [_COLD_EXIT] = HAS_ESCAPES_FLAG, - [_GUARD_IP] = HAS_ESCAPES_FLAG, + [_GUARD_IP] = 0, [_DYNAMIC_EXIT] = 0, }; diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 4021dc0ad72e14..1d5c74adefcd35 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -49,90 +49,88 @@ extern "C" { #define SETUP_ANNOTATIONS 36 #define STORE_SLICE 37 #define STORE_SUBSCR 38 -#define TIER1_GUARD_IP 39 -#define TIER1_SET_IP 40 -#define TO_BOOL 41 -#define UNARY_INVERT 42 -#define UNARY_NEGATIVE 43 -#define UNARY_NOT 44 -#define WITH_EXCEPT_START 45 -#define BINARY_OP 46 -#define BUILD_INTERPOLATION 47 -#define BUILD_LIST 48 -#define BUILD_MAP 49 -#define BUILD_SET 50 -#define BUILD_SLICE 51 -#define BUILD_STRING 52 -#define BUILD_TUPLE 53 -#define CALL 54 -#define CALL_INTRINSIC_1 55 -#define CALL_INTRINSIC_2 56 -#define CALL_KW 57 -#define COMPARE_OP 58 -#define CONTAINS_OP 59 -#define CONVERT_VALUE 60 -#define COPY 61 -#define COPY_FREE_VARS 62 -#define DELETE_ATTR 63 -#define DELETE_DEREF 64 -#define DELETE_FAST 65 -#define DELETE_GLOBAL 66 -#define DELETE_NAME 67 -#define DICT_MERGE 68 -#define DICT_UPDATE 69 -#define END_ASYNC_FOR 70 -#define EXTENDED_ARG 71 -#define FOR_ITER 72 -#define GET_AWAITABLE 73 -#define IMPORT_FROM 74 -#define IMPORT_NAME 75 -#define IS_OP 76 -#define JUMP_BACKWARD 77 -#define JUMP_BACKWARD_NO_INTERRUPT 78 -#define JUMP_FORWARD 79 -#define LIST_APPEND 80 -#define LIST_EXTEND 81 -#define LOAD_ATTR 82 -#define LOAD_COMMON_CONSTANT 83 -#define LOAD_CONST 84 -#define LOAD_DEREF 85 -#define LOAD_FAST 86 -#define LOAD_FAST_AND_CLEAR 87 -#define LOAD_FAST_BORROW 88 -#define LOAD_FAST_BORROW_LOAD_FAST_BORROW 89 -#define LOAD_FAST_CHECK 90 -#define LOAD_FAST_LOAD_FAST 91 -#define LOAD_FROM_DICT_OR_DEREF 92 -#define LOAD_FROM_DICT_OR_GLOBALS 93 -#define LOAD_GLOBAL 94 -#define LOAD_NAME 95 -#define LOAD_SMALL_INT 96 -#define LOAD_SPECIAL 97 -#define LOAD_SUPER_ATTR 98 -#define MAKE_CELL 99 -#define MAP_ADD 100 -#define MATCH_CLASS 101 -#define POP_JUMP_IF_FALSE 102 -#define POP_JUMP_IF_NONE 103 -#define POP_JUMP_IF_NOT_NONE 104 -#define POP_JUMP_IF_TRUE 105 -#define RAISE_VARARGS 106 -#define RERAISE 107 -#define SEND 108 -#define SET_ADD 109 -#define SET_FUNCTION_ATTRIBUTE 110 -#define SET_UPDATE 111 -#define STORE_ATTR 112 -#define STORE_DEREF 113 -#define STORE_FAST 114 -#define STORE_FAST_LOAD_FAST 115 -#define STORE_FAST_STORE_FAST 116 -#define STORE_GLOBAL 117 -#define STORE_NAME 118 -#define SWAP 119 -#define UNPACK_EX 120 -#define UNPACK_SEQUENCE 121 -#define YIELD_VALUE 122 +#define TO_BOOL 39 +#define UNARY_INVERT 40 +#define UNARY_NEGATIVE 41 +#define UNARY_NOT 42 +#define WITH_EXCEPT_START 43 +#define BINARY_OP 44 +#define BUILD_INTERPOLATION 45 +#define BUILD_LIST 46 +#define BUILD_MAP 47 +#define BUILD_SET 48 +#define BUILD_SLICE 49 +#define BUILD_STRING 50 +#define BUILD_TUPLE 51 +#define CALL 52 +#define CALL_INTRINSIC_1 53 +#define CALL_INTRINSIC_2 54 +#define CALL_KW 55 +#define COMPARE_OP 56 +#define CONTAINS_OP 57 +#define CONVERT_VALUE 58 +#define COPY 59 +#define COPY_FREE_VARS 60 +#define DELETE_ATTR 61 +#define DELETE_DEREF 62 +#define DELETE_FAST 63 +#define DELETE_GLOBAL 64 +#define DELETE_NAME 65 +#define DICT_MERGE 66 +#define DICT_UPDATE 67 +#define END_ASYNC_FOR 68 +#define EXTENDED_ARG 69 +#define FOR_ITER 70 +#define GET_AWAITABLE 71 +#define IMPORT_FROM 72 +#define IMPORT_NAME 73 +#define IS_OP 74 +#define JUMP_BACKWARD 75 +#define JUMP_BACKWARD_NO_INTERRUPT 76 +#define JUMP_FORWARD 77 +#define LIST_APPEND 78 +#define LIST_EXTEND 79 +#define LOAD_ATTR 80 +#define LOAD_COMMON_CONSTANT 81 +#define LOAD_CONST 82 +#define LOAD_DEREF 83 +#define LOAD_FAST 84 +#define LOAD_FAST_AND_CLEAR 85 +#define LOAD_FAST_BORROW 86 +#define LOAD_FAST_BORROW_LOAD_FAST_BORROW 87 +#define LOAD_FAST_CHECK 88 +#define LOAD_FAST_LOAD_FAST 89 +#define LOAD_FROM_DICT_OR_DEREF 90 +#define LOAD_FROM_DICT_OR_GLOBALS 91 +#define LOAD_GLOBAL 92 +#define LOAD_NAME 93 +#define LOAD_SMALL_INT 94 +#define LOAD_SPECIAL 95 +#define LOAD_SUPER_ATTR 96 +#define MAKE_CELL 97 +#define MAP_ADD 98 +#define MATCH_CLASS 99 +#define POP_JUMP_IF_FALSE 100 +#define POP_JUMP_IF_NONE 101 +#define POP_JUMP_IF_NOT_NONE 102 +#define POP_JUMP_IF_TRUE 103 +#define RAISE_VARARGS 104 +#define RERAISE 105 +#define SEND 106 +#define SET_ADD 107 +#define SET_FUNCTION_ATTRIBUTE 108 +#define SET_UPDATE 109 +#define STORE_ATTR 110 +#define STORE_DEREF 111 +#define STORE_FAST 112 +#define STORE_FAST_LOAD_FAST 113 +#define STORE_FAST_STORE_FAST 114 +#define STORE_GLOBAL 115 +#define STORE_NAME 116 +#define SWAP 117 +#define UNPACK_EX 118 +#define UNPACK_SEQUENCE 119 +#define YIELD_VALUE 120 #define RESUME 128 #define BINARY_OP_ADD_FLOAT 129 #define BINARY_OP_ADD_INT 130 @@ -249,7 +247,7 @@ extern "C" { #define SETUP_WITH 265 #define STORE_FAST_MAYBE_NULL 266 -#define HAVE_ARGUMENT 45 +#define HAVE_ARGUMENT 43 #define MIN_SPECIALIZED_OPCODE 129 #define MIN_INSTRUMENTED_OPCODE 234 diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index ca7fc579b3aadd..f168d169a32948 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -246,90 +246,88 @@ 'SETUP_ANNOTATIONS': 36, 'STORE_SLICE': 37, 'STORE_SUBSCR': 38, - 'TIER1_GUARD_IP': 39, - 'TIER1_SET_IP': 40, - 'TO_BOOL': 41, - 'UNARY_INVERT': 42, - 'UNARY_NEGATIVE': 43, - 'UNARY_NOT': 44, - 'WITH_EXCEPT_START': 45, - 'BINARY_OP': 46, - 'BUILD_INTERPOLATION': 47, - 'BUILD_LIST': 48, - 'BUILD_MAP': 49, - 'BUILD_SET': 50, - 'BUILD_SLICE': 51, - 'BUILD_STRING': 52, - 'BUILD_TUPLE': 53, - 'CALL': 54, - 'CALL_INTRINSIC_1': 55, - 'CALL_INTRINSIC_2': 56, - 'CALL_KW': 57, - 'COMPARE_OP': 58, - 'CONTAINS_OP': 59, - 'CONVERT_VALUE': 60, - 'COPY': 61, - 'COPY_FREE_VARS': 62, - 'DELETE_ATTR': 63, - 'DELETE_DEREF': 64, - 'DELETE_FAST': 65, - 'DELETE_GLOBAL': 66, - 'DELETE_NAME': 67, - 'DICT_MERGE': 68, - 'DICT_UPDATE': 69, - 'END_ASYNC_FOR': 70, - 'EXTENDED_ARG': 71, - 'FOR_ITER': 72, - 'GET_AWAITABLE': 73, - 'IMPORT_FROM': 74, - 'IMPORT_NAME': 75, - 'IS_OP': 76, - 'JUMP_BACKWARD': 77, - 'JUMP_BACKWARD_NO_INTERRUPT': 78, - 'JUMP_FORWARD': 79, - 'LIST_APPEND': 80, - 'LIST_EXTEND': 81, - 'LOAD_ATTR': 82, - 'LOAD_COMMON_CONSTANT': 83, - 'LOAD_CONST': 84, - 'LOAD_DEREF': 85, - 'LOAD_FAST': 86, - 'LOAD_FAST_AND_CLEAR': 87, - 'LOAD_FAST_BORROW': 88, - 'LOAD_FAST_BORROW_LOAD_FAST_BORROW': 89, - 'LOAD_FAST_CHECK': 90, - 'LOAD_FAST_LOAD_FAST': 91, - 'LOAD_FROM_DICT_OR_DEREF': 92, - 'LOAD_FROM_DICT_OR_GLOBALS': 93, - 'LOAD_GLOBAL': 94, - 'LOAD_NAME': 95, - 'LOAD_SMALL_INT': 96, - 'LOAD_SPECIAL': 97, - 'LOAD_SUPER_ATTR': 98, - 'MAKE_CELL': 99, - 'MAP_ADD': 100, - 'MATCH_CLASS': 101, - 'POP_JUMP_IF_FALSE': 102, - 'POP_JUMP_IF_NONE': 103, - 'POP_JUMP_IF_NOT_NONE': 104, - 'POP_JUMP_IF_TRUE': 105, - 'RAISE_VARARGS': 106, - 'RERAISE': 107, - 'SEND': 108, - 'SET_ADD': 109, - 'SET_FUNCTION_ATTRIBUTE': 110, - 'SET_UPDATE': 111, - 'STORE_ATTR': 112, - 'STORE_DEREF': 113, - 'STORE_FAST': 114, - 'STORE_FAST_LOAD_FAST': 115, - 'STORE_FAST_STORE_FAST': 116, - 'STORE_GLOBAL': 117, - 'STORE_NAME': 118, - 'SWAP': 119, - 'UNPACK_EX': 120, - 'UNPACK_SEQUENCE': 121, - 'YIELD_VALUE': 122, + 'TO_BOOL': 39, + 'UNARY_INVERT': 40, + 'UNARY_NEGATIVE': 41, + 'UNARY_NOT': 42, + 'WITH_EXCEPT_START': 43, + 'BINARY_OP': 44, + 'BUILD_INTERPOLATION': 45, + 'BUILD_LIST': 46, + 'BUILD_MAP': 47, + 'BUILD_SET': 48, + 'BUILD_SLICE': 49, + 'BUILD_STRING': 50, + 'BUILD_TUPLE': 51, + 'CALL': 52, + 'CALL_INTRINSIC_1': 53, + 'CALL_INTRINSIC_2': 54, + 'CALL_KW': 55, + 'COMPARE_OP': 56, + 'CONTAINS_OP': 57, + 'CONVERT_VALUE': 58, + 'COPY': 59, + 'COPY_FREE_VARS': 60, + 'DELETE_ATTR': 61, + 'DELETE_DEREF': 62, + 'DELETE_FAST': 63, + 'DELETE_GLOBAL': 64, + 'DELETE_NAME': 65, + 'DICT_MERGE': 66, + 'DICT_UPDATE': 67, + 'END_ASYNC_FOR': 68, + 'EXTENDED_ARG': 69, + 'FOR_ITER': 70, + 'GET_AWAITABLE': 71, + 'IMPORT_FROM': 72, + 'IMPORT_NAME': 73, + 'IS_OP': 74, + 'JUMP_BACKWARD': 75, + 'JUMP_BACKWARD_NO_INTERRUPT': 76, + 'JUMP_FORWARD': 77, + 'LIST_APPEND': 78, + 'LIST_EXTEND': 79, + 'LOAD_ATTR': 80, + 'LOAD_COMMON_CONSTANT': 81, + 'LOAD_CONST': 82, + 'LOAD_DEREF': 83, + 'LOAD_FAST': 84, + 'LOAD_FAST_AND_CLEAR': 85, + 'LOAD_FAST_BORROW': 86, + 'LOAD_FAST_BORROW_LOAD_FAST_BORROW': 87, + 'LOAD_FAST_CHECK': 88, + 'LOAD_FAST_LOAD_FAST': 89, + 'LOAD_FROM_DICT_OR_DEREF': 90, + 'LOAD_FROM_DICT_OR_GLOBALS': 91, + 'LOAD_GLOBAL': 92, + 'LOAD_NAME': 93, + 'LOAD_SMALL_INT': 94, + 'LOAD_SPECIAL': 95, + 'LOAD_SUPER_ATTR': 96, + 'MAKE_CELL': 97, + 'MAP_ADD': 98, + 'MATCH_CLASS': 99, + 'POP_JUMP_IF_FALSE': 100, + 'POP_JUMP_IF_NONE': 101, + 'POP_JUMP_IF_NOT_NONE': 102, + 'POP_JUMP_IF_TRUE': 103, + 'RAISE_VARARGS': 104, + 'RERAISE': 105, + 'SEND': 106, + 'SET_ADD': 107, + 'SET_FUNCTION_ATTRIBUTE': 108, + 'SET_UPDATE': 109, + 'STORE_ATTR': 110, + 'STORE_DEREF': 111, + 'STORE_FAST': 112, + 'STORE_FAST_LOAD_FAST': 113, + 'STORE_FAST_STORE_FAST': 114, + 'STORE_GLOBAL': 115, + 'STORE_NAME': 116, + 'SWAP': 117, + 'UNPACK_EX': 118, + 'UNPACK_SEQUENCE': 119, + 'YIELD_VALUE': 120, 'INSTRUMENTED_END_FOR': 234, 'INSTRUMENTED_POP_ITER': 235, 'INSTRUMENTED_END_SEND': 236, @@ -363,5 +361,5 @@ 'STORE_FAST_MAYBE_NULL': 266, } -HAVE_ARGUMENT = 45 +HAVE_ARGUMENT = 43 MIN_INSTRUMENTED_OPCODE = 234 diff --git a/Python/bytecodes.c b/Python/bytecodes.c index a495e332d97886..66553c8c501f3c 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2970,24 +2970,14 @@ dummy_func( start--; } if (tstate->interp->jit_tracer_code_buffer == NULL) { - tstate->interp->jit_tracer_code_buffer = (_Py_CODEUNIT *)_PyObject_VirtualAlloc(TRACER_BUFFER_SIZE); + tstate->interp->jit_tracer_code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); if (tstate->interp->jit_tracer_code_buffer == NULL) { // Don't error, just go to next instruction. DISPATCH(); } - tstate->interp->jit_tracer_code_curr_size = 0; } + _PyJIT_InitializeTracing(tstate, frame, next_instr, STACK_LEVEL(), 0); ENTER_TRACING(); - // Nothing in the buffer, begin tracing! - if (tstate->interp->jit_tracer_code_curr_size == 0) { - tstate->interp->jit_tracer_initial_instr = next_instr; - tstate->interp->jit_tracer_initial_code = _PyFrame_GetCode(frame); - tstate->interp->jit_tracer_initial_func = _PyFrame_GetFunction(frame); - tstate->interp->jit_tracer_seen_initial_before = 0; - tstate->interp->jit_completed_loop = false; - tstate->interp->jit_tracer_initial_stack_depth = (int)STACK_LEVEL(); - tstate->interp->jit_tracer_initial_chain_depth = 0; - } // Don't add the JUMP_BACKWARD_JIT instruction to the trace. DISPATCH(); } @@ -5443,13 +5433,7 @@ dummy_func( _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); assert(tstate->current_executor == (PyObject *)previous_executor); int chain_depth = previous_executor->vm_data.chain_depth + 1; - tstate->interp->jit_tracer_initial_chain_depth = chain_depth; - tstate->interp->jit_tracer_initial_instr = target; - tstate->interp->jit_tracer_initial_code = _PyFrame_GetCode(frame); - tstate->interp->jit_tracer_initial_func = _PyFrame_GetFunction(frame); - tstate->interp->jit_tracer_seen_initial_before = 0; - tstate->interp->jit_completed_loop = false; - exit->temperature = restart_backoff_counter(temperature); + _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), chain_depth); GOTO_TIER_ONE(target, 1); } assert(tstate->jit_exit == exit); @@ -5457,14 +5441,8 @@ dummy_func( TIER2_TO_TIER2(exit->executor); } - - no_save_ip tier1 inst(TIER1_GUARD_IP, (func_ptr/4 --)) { - (void)(func_ptr); - } - tier2 op(_GUARD_IP, (ip/4 --)) { if (frame->instr_ptr != (_Py_CODEUNIT *)ip) { - fprintf(stdout, "d:%p:%p\n", frame->instr_ptr, ip); GOTO_TIER_ONE(frame->instr_ptr, 1); } } @@ -5473,11 +5451,6 @@ dummy_func( GOTO_TIER_ONE(frame->instr_ptr, 1); } - tier1 inst(TIER1_SET_IP, (ip/4 --)) { - (void)(ip); - } - - label(pop_2_error) { stack_pointer -= 2; assert(WITHIN_STACK_BOUNDS()); diff --git a/Python/ceval.c b/Python/ceval.c index 4378afebc6ab2e..4029575853d6b6 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -975,99 +975,12 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch) // 1 for trace full, 0 for successful write. static int -add_to_code_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *this_instr, int oparg) +add_to_code_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *this_instr, _Py_CODEUNIT *next_instr, int oparg) { assert(frame != NULL); - assert(tstate->interp->jit_tracer_code_curr_size < TRACE_MAX_TRACE_LENGTH); - int curr_size = tstate->interp->jit_tracer_code_curr_size; - int opcode = this_instr->op.code; - if (opcode == ENTER_EXECUTOR) { - return 1; - // PyCodeObject *code = _PyFrame_GetCode(frame); - // assert(code != NULL); - // _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; - // assert(executor->vm_data.code == code); - // assert(executor->vm_data.valid); - // assert(tstate->current_executor == NULL); - // opcode = executor->vm_data.opcode; - // oparg = (oparg & ~255) | executor->vm_data.oparg; - } - else { - oparg = this_instr->op.arg; - } - assert(opcode != 0); - // Check if we looped back to the start. - if (this_instr == tstate->interp->jit_tracer_initial_instr) { - if (tstate->interp->jit_tracer_seen_initial_before >= 1) { - tstate->interp->jit_completed_loop = true; - return 1; - } - tstate->interp->jit_tracer_seen_initial_before++; - } -#ifdef Py_DEBUG - char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); - int lltrace = 3; - if (python_lltrace != NULL && *python_lltrace >= '0') { - lltrace = *python_lltrace - '0'; - } - if (lltrace >= 3) { - printf("%d ADD_TO_BYTECODE_TRACE: %s %d\n", curr_size, _PyOpcode_OpName[opcode], oparg); - } -#endif - // JUMP_BACKWARD_NO_INTERRUPT is not registered with cache due to a bug in the compiler. - int caches = opcode == JUMP_BACKWARD_NO_INTERRUPT ? 1 : _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; - int nsize = caches + 1; - int needs_guard_ip = _PyOpcode_NeedsGuardIp[opcode]; - int total_size = nsize + (needs_guard_ip ? 5 : 0) + 5; - if (curr_size + total_size >= TRACE_MAX_TRACE_LENGTH) { -#ifdef Py_DEBUG - if (lltrace >= 3) { - printf("END TRACE LENGTH: %d\n", curr_size); - } -#endif - return 1; - } - tstate->interp->jit_tracer_code_buffer[curr_size].op.code = opcode; - tstate->interp->jit_tracer_code_buffer[curr_size].op.arg = oparg; - for (int i = 1; i < nsize; i++) { - tstate->interp->jit_tracer_code_buffer[curr_size + i] = *(this_instr + i); - } - if (needs_guard_ip) { -#ifdef Py_DEBUG - if (lltrace >= 3) { - printf("%d ADD_TO_BYTECODE_TRACE: %s %d\n", curr_size + nsize, _PyOpcode_OpName[TIER1_GUARD_IP], 0); - } -#endif - tstate->interp->jit_tracer_code_buffer[curr_size + nsize].op.code = TIER1_GUARD_IP; - PyObject *func = PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - if (Py_IsNone(func)) { - func = NULL; // Trampoline frames don't set their func object field. - } - assert(func == NULL || PyFunction_Check(func)); - write_ptr(&tstate->interp->jit_tracer_code_buffer[curr_size + nsize + 1].cache, func); - -#ifdef Py_DEBUG - if (lltrace >= 3) { - printf("%d ADD_TO_BYTECODE_TRACE: %s %p\n", curr_size + nsize + 5, _PyOpcode_OpName[TIER1_SET_IP], frame->instr_ptr); - } -#endif - tstate->interp->jit_tracer_code_buffer[curr_size + nsize + 5].op.code = TIER1_SET_IP; - write_ptr(&tstate->interp->jit_tracer_code_buffer[curr_size + nsize + 6].cache, frame->instr_ptr); - - } - else { -#ifdef Py_DEBUG - if (lltrace >= 3) { - printf("%d ADD_TO_BYTECODE_TRACE: %s %p\n", curr_size + nsize, _PyOpcode_OpName[TIER1_SET_IP], frame->instr_ptr); - } -#endif - tstate->interp->jit_tracer_code_buffer[curr_size + nsize].op.code = TIER1_SET_IP; - write_ptr(&tstate->interp->jit_tracer_code_buffer[curr_size + nsize + 1].cache, frame->instr_ptr); - - } - - tstate->interp->jit_tracer_code_curr_size += total_size; - return 0; + assert(tstate->interp->jit_tracer_code_curr_size < UOP_MAX_TRACE_LENGTH); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + return !_PyJIT_translate_single_bytecode_to_trace(tstate, this_instr, next_instr, _PyFrame_GetCode(frame), func, oparg); } /* _PyEval_EvalFrameDefault is too large to optimize for speed with PGO on MSVC. diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index c0f33dd5b31f9c..a47b0e524c703a 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -144,7 +144,7 @@ DISPATCH(); # define RECORD_TRACE() do { \ frame->instr_ptr = next_instr; \ - if (add_to_code_trace(tstate, frame, this_instr, oparg)) { \ + if (add_to_code_trace(tstate, frame, this_instr, next_instr, oparg)) { \ BAIL_TRACING(); \ } \ } while (0); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 0b5aee504012cb..145f010db47962 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7560,15 +7560,9 @@ stack_pointer = _PyFrame_GetStackPointer(frame); assert(tstate->current_executor == (PyObject *)previous_executor); int chain_depth = previous_executor->vm_data.chain_depth + 1; - tstate->interp->jit_tracer_initial_chain_depth = chain_depth; - tstate->interp->jit_tracer_initial_instr = target; - tstate->interp->jit_tracer_initial_code = _PyFrame_GetCode(frame); _PyFrame_SetStackPointer(frame, stack_pointer); - tstate->interp->jit_tracer_initial_func = _PyFrame_GetFunction(frame); + _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), chain_depth); stack_pointer = _PyFrame_GetStackPointer(frame); - tstate->interp->jit_tracer_seen_initial_before = 0; - tstate->interp->jit_completed_loop = false; - exit->temperature = restart_backoff_counter(temperature); GOTO_TIER_ONE(target, 1); } assert(tstate->jit_exit == exit); @@ -7580,9 +7574,6 @@ case _GUARD_IP: { PyObject *ip = (PyObject *)CURRENT_OPERAND0(); if (frame->instr_ptr != (_Py_CODEUNIT *)ip) { - _PyFrame_SetStackPointer(frame, stack_pointer); - fprintf(stdout, "d:%p:%p\n", frame->instr_ptr, ip); - stack_pointer = _PyFrame_GetStackPointer(frame); GOTO_TIER_ONE(frame->instr_ptr, 1); } break; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index a9749abb5fad87..99fb53f26c5c48 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -7663,25 +7663,16 @@ } if (tstate->interp->jit_tracer_code_buffer == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); - tstate->interp->jit_tracer_code_buffer = (_Py_CODEUNIT *)_PyObject_VirtualAlloc(TRACER_BUFFER_SIZE); + tstate->interp->jit_tracer_code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); stack_pointer = _PyFrame_GetStackPointer(frame); if (tstate->interp->jit_tracer_code_buffer == NULL) { DISPATCH(); } - tstate->interp->jit_tracer_code_curr_size = 0; } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyJIT_InitializeTracing(tstate, frame, next_instr, STACK_LEVEL(), 0); + stack_pointer = _PyFrame_GetStackPointer(frame); ENTER_TRACING(); - if (tstate->interp->jit_tracer_code_curr_size == 0) { - tstate->interp->jit_tracer_initial_instr = next_instr; - tstate->interp->jit_tracer_initial_code = _PyFrame_GetCode(frame); - _PyFrame_SetStackPointer(frame, stack_pointer); - tstate->interp->jit_tracer_initial_func = _PyFrame_GetFunction(frame); - stack_pointer = _PyFrame_GetStackPointer(frame); - tstate->interp->jit_tracer_seen_initial_before = 0; - tstate->interp->jit_completed_loop = false; - tstate->interp->jit_tracer_initial_stack_depth = (int)STACK_LEVEL(); - tstate->interp->jit_tracer_initial_chain_depth = 0; - } DISPATCH(); } else { @@ -11428,41 +11419,6 @@ DISPATCH(); } - TARGET(TIER1_GUARD_IP) { - #if _Py_TAIL_CALL_INTERP - int opcode = TIER1_GUARD_IP; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - next_instr += 5; - INSTRUCTION_STATS(TIER1_GUARD_IP); - PyObject *func_ptr = read_obj(&this_instr[1].cache); - (void)func_ptr; - _PyFrame_SetStackPointer(frame, stack_pointer); - (void)(func_ptr); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH(); - } - - TARGET(TIER1_SET_IP) { - #if _Py_TAIL_CALL_INTERP - int opcode = TIER1_SET_IP; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(TIER1_SET_IP); - PyObject *ip = read_obj(&this_instr[1].cache); - (void)ip; - _PyFrame_SetStackPointer(frame, stack_pointer); - (void)(ip); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH(); - } - TARGET(TO_BOOL) { #if _Py_TAIL_CALL_INTERP int opcode = TO_BOOL; @@ -19877,25 +19833,16 @@ } if (tstate->interp->jit_tracer_code_buffer == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); - tstate->interp->jit_tracer_code_buffer = (_Py_CODEUNIT *)_PyObject_VirtualAlloc(TRACER_BUFFER_SIZE); + tstate->interp->jit_tracer_code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); stack_pointer = _PyFrame_GetStackPointer(frame); if (tstate->interp->jit_tracer_code_buffer == NULL) { TRACING_DISPATCH(); } - tstate->interp->jit_tracer_code_curr_size = 0; } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyJIT_InitializeTracing(tstate, frame, next_instr, STACK_LEVEL(), 0); + stack_pointer = _PyFrame_GetStackPointer(frame); ENTER_TRACING(); - if (tstate->interp->jit_tracer_code_curr_size == 0) { - tstate->interp->jit_tracer_initial_instr = next_instr; - tstate->interp->jit_tracer_initial_code = _PyFrame_GetCode(frame); - _PyFrame_SetStackPointer(frame, stack_pointer); - tstate->interp->jit_tracer_initial_func = _PyFrame_GetFunction(frame); - stack_pointer = _PyFrame_GetStackPointer(frame); - tstate->interp->jit_tracer_seen_initial_before = 0; - tstate->interp->jit_completed_loop = false; - tstate->interp->jit_tracer_initial_stack_depth = (int)STACK_LEVEL(); - tstate->interp->jit_tracer_initial_chain_depth = 0; - } TRACING_DISPATCH(); } else { @@ -23750,41 +23697,6 @@ TRACING_DISPATCH(); } - TRACING_TARGET(TIER1_GUARD_IP) { - #if _Py_TAIL_CALL_INTERP - int opcode = TIER1_GUARD_IP; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - next_instr += 5; - INSTRUCTION_STATS(TIER1_GUARD_IP); - PyObject *func_ptr = read_obj(&this_instr[1].cache); - (void)func_ptr; - _PyFrame_SetStackPointer(frame, stack_pointer); - (void)(func_ptr); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(TIER1_SET_IP) { - #if _Py_TAIL_CALL_INTERP - int opcode = TIER1_SET_IP; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(TIER1_SET_IP); - PyObject *ip = read_obj(&this_instr[1].cache); - (void)ip; - _PyFrame_SetStackPointer(frame, stack_pointer); - (void)(ip); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - TRACING_TARGET(TO_BOOL) { #if _Py_TAIL_CALL_INTERP int opcode = TO_BOOL; diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 47d6290b0dc1e8..299739764919bb 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -39,8 +39,6 @@ static void *opcode_targets_table[256] = { &&TARGET_SETUP_ANNOTATIONS, &&TARGET_STORE_SLICE, &&TARGET_STORE_SUBSCR, - &&TARGET_TIER1_GUARD_IP, - &&TARGET_TIER1_SET_IP, &&TARGET_TO_BOOL, &&TARGET_UNARY_INVERT, &&TARGET_UNARY_NEGATIVE, @@ -128,6 +126,8 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, &&TARGET_RESUME, &&TARGET_BINARY_OP_ADD_FLOAT, &&TARGET_BINARY_OP_ADD_INT, @@ -297,8 +297,6 @@ static void *opcode_tracing_targets_table[256] = { &&TARGET_TRACING_SETUP_ANNOTATIONS, &&TARGET_TRACING_STORE_SLICE, &&TARGET_TRACING_STORE_SUBSCR, - &&TARGET_TRACING_TIER1_GUARD_IP, - &&TARGET_TRACING_TIER1_SET_IP, &&TARGET_TRACING_TO_BOOL, &&TARGET_TRACING_UNARY_INVERT, &&TARGET_TRACING_UNARY_NEGATIVE, @@ -386,6 +384,8 @@ static void *opcode_tracing_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, + &&_unknown_opcode, + &&_unknown_opcode, &&TARGET_TRACING_RESUME, &&TARGET_TRACING_BINARY_OP_ADD_FLOAT, &&TARGET_TRACING_BINARY_OP_ADD_INT, @@ -943,10 +943,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SWAP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_SWAP(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TIER1_GUARD_IP(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_TIER1_GUARD_IP(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TIER1_SET_IP(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_TIER1_SET_IP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_TO_BOOL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS); @@ -1201,8 +1197,6 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [STORE_SUBSCR_DICT] = _TAIL_CALL_STORE_SUBSCR_DICT, [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_STORE_SUBSCR_LIST_INT, [SWAP] = _TAIL_CALL_SWAP, - [TIER1_GUARD_IP] = _TAIL_CALL_TIER1_GUARD_IP, - [TIER1_SET_IP] = _TAIL_CALL_TIER1_SET_IP, [TO_BOOL] = _TAIL_CALL_TO_BOOL, [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_TO_BOOL_ALWAYS_TRUE, [TO_BOOL_BOOL] = _TAIL_CALL_TO_BOOL_BOOL, @@ -1220,6 +1214,8 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE, [WITH_EXCEPT_START] = _TAIL_CALL_WITH_EXCEPT_START, [YIELD_VALUE] = _TAIL_CALL_YIELD_VALUE, + [121] = _TAIL_CALL_UNKNOWN_OPCODE, + [122] = _TAIL_CALL_UNKNOWN_OPCODE, [123] = _TAIL_CALL_UNKNOWN_OPCODE, [124] = _TAIL_CALL_UNKNOWN_OPCODE, [125] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -1459,8 +1455,6 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [STORE_SUBSCR_DICT] = _TAIL_CALL_TRACING_STORE_SUBSCR_DICT, [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_TRACING_STORE_SUBSCR_LIST_INT, [SWAP] = _TAIL_CALL_TRACING_SWAP, - [TIER1_GUARD_IP] = _TAIL_CALL_TRACING_TIER1_GUARD_IP, - [TIER1_SET_IP] = _TAIL_CALL_TRACING_TIER1_SET_IP, [TO_BOOL] = _TAIL_CALL_TRACING_TO_BOOL, [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_TRACING_TO_BOOL_ALWAYS_TRUE, [TO_BOOL_BOOL] = _TAIL_CALL_TRACING_TO_BOOL_BOOL, @@ -1478,6 +1472,8 @@ static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_TRACING_UNPACK_SEQUENCE_TWO_TUPLE, [WITH_EXCEPT_START] = _TAIL_CALL_TRACING_WITH_EXCEPT_START, [YIELD_VALUE] = _TAIL_CALL_TRACING_YIELD_VALUE, + [121] = _TAIL_CALL_UNKNOWN_OPCODE, + [122] = _TAIL_CALL_UNKNOWN_OPCODE, [123] = _TAIL_CALL_UNKNOWN_OPCODE, [124] = _TAIL_CALL_UNKNOWN_OPCODE, [125] = _TAIL_CALL_UNKNOWN_OPCODE, diff --git a/Python/optimizer.c b/Python/optimizer.c index e4ccca924302ca..32aa4481c7e504 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -513,35 +513,34 @@ add_to_trace( DPRINTF(2, "No room for %s (need %d, got %d)\n", \ (opname), (n), max_length - trace_length); \ OPT_STAT_INC(trace_too_long); \ - goto done; \ + goto full; \ } // Reserve space for N uops, plus 3 for _SET_IP, _CHECK_VALIDITY and _EXIT_TRACE #define RESERVE(needed) RESERVE_RAW((needed) + 3, _PyUOpName(opcode)) -/* Returns the length of the trace on success, - * 0 if it failed to produce a worthwhile trace, - * and -1 on an error. +/* Returns 1 on success (added to trace), 0 on trace end. */ -static int -translate_bytecode_to_trace( - _PyInterpreterFrame *frame, +int +_PyJIT_translate_single_bytecode_to_trace( PyThreadState *tstate, - _PyUOpInstruction *trace, - int buffer_size, - _PyBloomFilter *dependencies, bool progress_needed) + _Py_CODEUNIT *this_instr, + _Py_CODEUNIT *next_instr, + PyCodeObject *code, + PyFunctionObject *func, + int oparg) { - bool first = true; - PyCodeObject *code = tstate->interp->jit_tracer_initial_code;; - PyFunctionObject *func = tstate->interp->jit_tracer_initial_func; - PyCodeObject *initial_code = code; - _Py_BloomFilter_Add(dependencies, initial_code); - _Py_CODEUNIT *target_instr = tstate->interp->jit_tracer_initial_instr; - _Py_CODEUNIT *initial_instr = tstate->interp->jit_tracer_code_buffer; - int trace_length = 0; - // Leave space for possible trailing _EXIT_TRACE - int max_length = buffer_size-2; + if (Py_IsNone((PyObject *)func)) { + func = NULL; + } + bool progress_needed = (tstate->interp->jit_tracer_initial_chain_depth % MAX_CHAIN_DEPTH) == 0;; + _PyBloomFilter *dependencies = &tstate->interp->jit_tracer_dependencies; + _Py_BloomFilter_Add(dependencies, code); + _Py_CODEUNIT *target_instr = this_instr; + int trace_length = tstate->interp->jit_tracer_code_curr_size; + _PyUOpInstruction *trace = tstate->interp->jit_tracer_code_buffer; + int max_length = tstate->interp->jit_tracer_code_max_size; #ifdef Py_DEBUG char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); @@ -551,269 +550,233 @@ translate_bytecode_to_trace( } #endif - DPRINTF(2, - "Optimizing %s (%s:%d) at byte offset %d\n", - PyUnicode_AsUTF8(code->co_qualname), - PyUnicode_AsUTF8(code->co_filename), - code->co_firstlineno, - 2 * INSTR_IP(target_instr, code)); - ADD_TO_TRACE(_START_EXECUTOR, 0, (uintptr_t)target_instr, INSTR_IP(target_instr, code)); - ADD_TO_TRACE(_MAKE_WARM, 0, 0, 0); uint32_t target = 0; - bool preceded_by_for_iter = false; - - for (int x = 0; x < tstate->interp->jit_tracer_code_curr_size;) { - target = INSTR_IP(target_instr, code); - // One for possible _DEOPT, one because _CHECK_VALIDITY itself might _DEOPT - max_length-=2; - uint32_t opcode = initial_instr[x].op.code; - uint32_t oparg = initial_instr[x].op.arg; - - DPRINTF(2, "%d: %s(%d)\n", target, _PyOpcode_OpName[opcode], oparg); - - if (opcode == EXTENDED_ARG) { - x++; - opcode = initial_instr[x].op.code; - oparg = (oparg << 8) | initial_instr[x].op.arg; - if (opcode == EXTENDED_ARG) { - x--; - goto done; - } - } - assert(opcode != ENTER_EXECUTOR && opcode != EXTENDED_ARG); - if (opcode != NOP) { - if (opcode != TIER1_GUARD_IP) { - RESERVE_RAW(1, "_CHECK_VALIDITY"); - ADD_TO_TRACE(_CHECK_VALIDITY, 0, 0, target); - } - if (!OPCODE_HAS_NO_SAVE_IP(opcode)) { - RESERVE_RAW(2, "_SET_IP"); - ADD_TO_TRACE(_SET_IP, 0, (uintptr_t)target_instr, target); - } - } - /* Special case the first instruction, - * so that we can guarantee forward progress */ - if (first && progress_needed) { - assert(first); - if (OPCODE_HAS_EXIT(opcode) || OPCODE_HAS_DEOPT(opcode)) { - opcode = _PyOpcode_Deopt[opcode]; - } - assert(!OPCODE_HAS_EXIT(opcode)); - assert(!OPCODE_HAS_DEOPT(opcode)); - } + target = INSTR_IP(target_instr, code); + // One for possible _DEOPT, one because _CHECK_VALIDITY itself might _DEOPT + max_length-=2; + uint32_t opcode = this_instr->op.code; + if ((uint16_t)oparg != (uint64_t)oparg) { + goto full; + } - if (OPCODE_HAS_EXIT(opcode)) { - // Make space for side exit and final _EXIT_TRACE: - RESERVE_RAW(2, "_EXIT_TRACE"); - max_length--; - } - if (OPCODE_HAS_ERROR(opcode)) { - // Make space for error stub and final _EXIT_TRACE: - RESERVE_RAW(2, "_ERROR_POP_N"); - max_length--; - } - if (opcode == FOR_ITER_RANGE || opcode == FOR_ITER_LIST || opcode == FOR_ITER_TUPLE) { - preceded_by_for_iter = true; - // assert((initial_instr + x + 1 + _PyOpcode_Caches[FOR_ITER])->op.code == TIER1_GUARD_IP); - // assert((initial_instr + x + 6 + _PyOpcode_Caches[FOR_ITER])->op.code == TIER1_SET_IP); - // PyObject *temp = read_obj(&(initial_instr + x + 2 + _PyOpcode_Caches[FOR_ITER])->cache); - // if (temp != NULL) { - // code = ((PyFunctionObject*)temp)->func_code; - // } - // target_instr = (_Py_CODEUNIT *)(read_obj(&(initial_instr + x + 7 + _PyOpcode_Caches[FOR_ITER])->cache)); - // target = INSTR_IP(target_instr, code); - // - // (initial_instr + x + 1 + _PyOpcode_Caches[FOR_ITER])->op.code = NOP; - // (initial_instr + x + 2 + _PyOpcode_Caches[FOR_ITER])->op.code = NOP; - // (initial_instr + x + 3 + _PyOpcode_Caches[FOR_ITER])->op.code = NOP; - // (initial_instr + x + 4 + _PyOpcode_Caches[FOR_ITER])->op.code = NOP; - // (initial_instr + x + 5 + _PyOpcode_Caches[FOR_ITER])->op.code = NOP; - // - // (initial_instr + x + 6 + _PyOpcode_Caches[FOR_ITER])->op.code = NOP; - // (initial_instr + x + 7 + _PyOpcode_Caches[FOR_ITER])->op.code = NOP; - // (initial_instr + x + 8 + _PyOpcode_Caches[FOR_ITER])->op.code = NOP; - // (initial_instr + x + 9 + _PyOpcode_Caches[FOR_ITER])->op.code = NOP; - // (initial_instr + x + 10 + _PyOpcode_Caches[FOR_ITER])->op.code = NOP; + DPRINTF(2, "%d: %s(%d)\n", target, _PyOpcode_OpName[opcode], oparg); + + if (opcode == EXTENDED_ARG) { + return 1; + } + if (opcode == ENTER_EXECUTOR) { + goto full; + } + assert(opcode != ENTER_EXECUTOR && opcode != EXTENDED_ARG); + if (opcode == NOP) { + return 1; + } + + RESERVE_RAW(1, "_CHECK_VALIDITY"); + ADD_TO_TRACE(_CHECK_VALIDITY, 0, 0, target); + + if (!OPCODE_HAS_NO_SAVE_IP(opcode)) { + RESERVE_RAW(2, "_SET_IP"); + ADD_TO_TRACE(_SET_IP, 0, (uintptr_t)target_instr, target); + } + + bool needs_guard_ip = _PyOpcode_NeedsGuardIp[opcode] && + !(opcode == FOR_ITER_RANGE || opcode == FOR_ITER_LIST || opcode == FOR_ITER_TUPLE) && + !(opcode == JUMP_BACKWARD_NO_INTERRUPT || opcode == JUMP_BACKWARD || opcode == JUMP_BACKWARD_JIT); + + if (needs_guard_ip) { + RESERVE_RAW(1, "_GUARD_IP"); + } + + /* Special case the first instruction, + * so that we can guarantee forward progress */ + if (progress_needed && tstate->interp->jit_tracer_initial_instr == this_instr && tstate->interp->jit_tracer_code_curr_size == 0) { + if (OPCODE_HAS_EXIT(opcode) || OPCODE_HAS_DEOPT(opcode)) { + opcode = _PyOpcode_Deopt[opcode]; } - switch (opcode) { - case JUMP_BACKWARD_JIT: - case JUMP_BACKWARD: - ADD_TO_TRACE(_CHECK_PERIODIC, 0, 0, target); - _Py_FALLTHROUGH; - case JUMP_BACKWARD_NO_INTERRUPT: - ADD_TO_TRACE(_JUMP_BACKWARD_NO_INTERRUPT, oparg, 0, target); - break; - case TIER1_GUARD_IP: - func = (PyFunctionObject *)read_obj(&(initial_instr + x + 1)->cache); - assert(func == NULL || PyFunction_Check(func)); - if (func != NULL) { - code = (PyCodeObject *)func->func_code; - } - assert((initial_instr + x + 5)->op.code == TIER1_SET_IP); - target_instr = (_Py_CODEUNIT*)read_obj(&(initial_instr + x + 6)->cache); - if (preceded_by_for_iter) {; - preceded_by_for_iter = false; - } - else { - ADD_TO_TRACE(_GUARD_IP, 0, (uintptr_t)target_instr, 0) - } - break; + assert(!OPCODE_HAS_EXIT(opcode)); + assert(!OPCODE_HAS_DEOPT(opcode)); + } - case TIER1_SET_IP: - target_instr = (_Py_CODEUNIT*)read_obj(&(initial_instr + x + 1)->cache); - break; + // Loop back to the start + if (tstate->interp->jit_tracer_initial_instr == this_instr && tstate->interp->jit_tracer_code_curr_size > 2) { + ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0, 0); + goto done; + } - case RESUME: - /* Use a special tier 2 version of RESUME_CHECK to allow traces to - * start with RESUME_CHECK */ - ADD_TO_TRACE(_TIER2_RESUME_CHECK, 0, 0, target); - break; + if (OPCODE_HAS_EXIT(opcode)) { + // Make space for side exit and final _EXIT_TRACE: + RESERVE_RAW(2, "_EXIT_TRACE"); + max_length--; + } + if (OPCODE_HAS_ERROR(opcode)) { + // Make space for error stub and final _EXIT_TRACE: + RESERVE_RAW(2, "_ERROR_POP_N"); + max_length--; + } - default: - { - const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; - if (expansion->nuops > 0) { - // Reserve space for nuops (+ _SET_IP + _EXIT_TRACE) - int nuops = expansion->nuops; - RESERVE(nuops + 1); /* One extra for exit */ - uint32_t orig_oparg = oparg; // For OPARG_TOP/BOTTOM - for (int i = 0; i < nuops; i++) { - oparg = orig_oparg; - uint32_t uop = expansion->uops[i].uop; - uint64_t operand = 0; - // Add one to account for the actual opcode/oparg pair: - int offset = expansion->uops[i].offset + 1; - switch (expansion->uops[i].size) { - case OPARG_SIMPLE: - assert(opcode != _JUMP_BACKWARD_NO_INTERRUPT && opcode != JUMP_BACKWARD); - break; - case OPARG_CACHE_1: - operand = read_u16(&(initial_instr + x)[offset].cache); - break; - case OPARG_CACHE_2: - operand = read_u32(&(initial_instr + x)[offset].cache); - break; - case OPARG_CACHE_4: - operand = read_u64(&(initial_instr + x)[offset].cache); - break; - case OPARG_TOP: // First half of super-instr - oparg = orig_oparg >> 4; - break; - case OPARG_BOTTOM: // Second half of super-instr - oparg = orig_oparg & 0xF; - break; - case OPARG_SAVE_RETURN_OFFSET: // op=_SAVE_RETURN_OFFSET; oparg=return_offset - oparg = offset; - assert(uop == _SAVE_RETURN_OFFSET); - break; - case OPARG_REPLACED: - uop = _PyUOp_Replacements[uop]; - assert(uop != 0); - - uint32_t next_inst = target + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + (oparg > 255); - if (uop == _TIER2_RESUME_CHECK) { - target = next_inst; - } + switch (opcode) { + case JUMP_BACKWARD_JIT: + case JUMP_BACKWARD: + ADD_TO_TRACE(_CHECK_PERIODIC, 0, 0, target); + _Py_FALLTHROUGH; + case JUMP_BACKWARD_NO_INTERRUPT: + break; + + case RESUME: + /* Use a special tier 2 version of RESUME_CHECK to allow traces to + * start with RESUME_CHECK */ + ADD_TO_TRACE(_TIER2_RESUME_CHECK, 0, 0, target); + break; + + default: + { + const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; + if (expansion->nuops > 0) { + // Reserve space for nuops (+ _SET_IP + _EXIT_TRACE) + int nuops = expansion->nuops; + RESERVE(nuops + 1); /* One extra for exit */ + uint32_t orig_oparg = oparg; // For OPARG_TOP/BOTTOM + for (int i = 0; i < nuops; i++) { + oparg = orig_oparg; + uint32_t uop = expansion->uops[i].uop; + uint64_t operand = 0; + // Add one to account for the actual opcode/oparg pair: + int offset = expansion->uops[i].offset + 1; + switch (expansion->uops[i].size) { + case OPARG_SIMPLE: + assert(opcode != _JUMP_BACKWARD_NO_INTERRUPT && opcode != JUMP_BACKWARD); + break; + case OPARG_CACHE_1: + operand = read_u16(&this_instr[offset].cache); + break; + case OPARG_CACHE_2: + operand = read_u32(&this_instr[offset].cache); + break; + case OPARG_CACHE_4: + operand = read_u64(&this_instr[offset].cache); + break; + case OPARG_TOP: // First half of super-instr + oparg = orig_oparg >> 4; + break; + case OPARG_BOTTOM: // Second half of super-instr + oparg = orig_oparg & 0xF; + break; + case OPARG_SAVE_RETURN_OFFSET: // op=_SAVE_RETURN_OFFSET; oparg=return_offset + oparg = offset; + assert(uop == _SAVE_RETURN_OFFSET); + break; + case OPARG_REPLACED: + uop = _PyUOp_Replacements[uop]; + assert(uop != 0); + + uint32_t next_inst = target + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + (oparg > 255); + if (uop == _TIER2_RESUME_CHECK) { + target = next_inst; + } #ifdef Py_DEBUG - else { - uint32_t jump_target = next_inst + oparg; - assert(_Py_GetBaseCodeUnit(code, jump_target).op.code == END_FOR); - assert(_Py_GetBaseCodeUnit(code, jump_target+1).op.code == POP_ITER); - } + else { + uint32_t jump_target = next_inst + oparg; + assert(_Py_GetBaseCodeUnit(code, jump_target).op.code == END_FOR); + assert(_Py_GetBaseCodeUnit(code, jump_target+1).op.code == POP_ITER); + } #endif - break; - case OPERAND1_1: - assert(trace[trace_length-1].opcode == uop); - operand = read_u16(&(initial_instr + x)[offset].cache); - trace[trace_length-1].operand1 = operand; - continue; - case OPERAND1_2: - assert(trace[trace_length-1].opcode == uop); - operand = read_u32(&(initial_instr + x)[offset].cache); - trace[trace_length-1].operand1 = operand; - continue; - case OPERAND1_4: - assert(trace[trace_length-1].opcode == uop); - operand = read_u64(&(initial_instr + x)[offset].cache); - trace[trace_length-1].operand1 = operand; - continue; - default: - fprintf(stderr, - "opcode=%d, oparg=%d; nuops=%d, i=%d; size=%d, offset=%d\n", - opcode, oparg, nuops, i, - expansion->uops[i].size, - expansion->uops[i].offset); - Py_FatalError("garbled expansion"); + break; + case OPERAND1_1: + assert(trace[trace_length-1].opcode == uop); + operand = read_u16(&this_instr[offset].cache); + trace[trace_length-1].operand1 = operand; + continue; + case OPERAND1_2: + assert(trace[trace_length-1].opcode == uop); + operand = read_u32(&this_instr[offset].cache); + trace[trace_length-1].operand1 = operand; + continue; + case OPERAND1_4: + assert(trace[trace_length-1].opcode == uop); + operand = read_u64(&this_instr[offset].cache); + trace[trace_length-1].operand1 = operand; + continue; + default: + fprintf(stderr, + "opcode=%d, oparg=%d; nuops=%d, i=%d; size=%d, offset=%d\n", + opcode, oparg, nuops, i, + expansion->uops[i].size, + expansion->uops[i].offset); + Py_FatalError("garbled expansion"); + } + if (uop == _PUSH_FRAME || uop == _RETURN_VALUE || uop == _RETURN_GENERATOR || uop == _YIELD_VALUE) { + if (func != NULL) { + operand = (uintptr_t)func; } - - if (uop == _BINARY_OP_INPLACE_ADD_UNICODE) { - assert(i + 1 == nuops); - _Py_CODEUNIT *next_instr = target_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; - assert(next_instr->op.code == STORE_FAST); - operand = next_instr->op.arg; - // Skip the STORE_FAST: - x++; + else if (code != NULL) { + operand = (uintptr_t)code | 1; + } + else { + operand = 0; } - - // All other instructions - ADD_TO_TRACE(uop, oparg, operand, target); } - break; + // All other instructions + ADD_TO_TRACE(uop, oparg, operand, target); } - DPRINTF(2, "Unsupported opcode %s\n", _PyOpcode_OpName[opcode]); - OPT_UNSUPPORTED_OPCODE(opcode); - goto done; // Break out of loop - } // End default - - } // End switch (opcode) - - x++; - // Add cache size for opcode - x += _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; - - if (opcode == CALL_LIST_APPEND) { - assert(initial_instr[x].op.code == POP_TOP); - x++; - } + break; + } + DPRINTF(2, "Unsupported opcode %s\n", _PyOpcode_OpName[opcode]); + OPT_UNSUPPORTED_OPCODE(opcode); + } // End default - if (x > tstate->interp->jit_tracer_code_curr_size) { - goto done; - } - // Jump here after _PUSH_FRAME or likely branches. - first = false; - } // End for (;;) + } // End switch (opcode) -done: - // Looped back to top. - if (!first && tstate->interp->jit_completed_loop && target_instr == tstate->interp->jit_tracer_initial_instr) { - ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0, 0); + if (needs_guard_ip) { + ADD_TO_TRACE(_GUARD_IP, 0, (uintptr_t)next_instr, 0); } - // Skip short traces where we can't even translate a single instruction: - if (first) { - OPT_STAT_INC(trace_too_short); - DPRINTF(2, - "No trace for %s (%s:%d) at byte offset %d (no progress)\n", - PyUnicode_AsUTF8(code->co_qualname), - PyUnicode_AsUTF8(code->co_filename), - code->co_firstlineno, - 2 * INSTR_IP(initial_instr, code)); - return 0; - } - if (!is_terminator(&trace[trace_length-1])) { - /* Allow space for _EXIT_TRACE */ - max_length += 2; + tstate->interp->jit_tracer_code_curr_size = trace_length; + tstate->interp->jit_tracer_code_max_size = max_length; + return 1; +done: + tstate->interp->jit_tracer_code_curr_size = trace_length; + tstate->interp->jit_tracer_code_max_size = max_length; + return 0; +full: + if (!is_terminator(&tstate->interp->jit_tracer_code_buffer[trace_length-1])) { + // Undo the last few instructions. + trace_length = tstate->interp->jit_tracer_code_curr_size; ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); } - DPRINTF(1, - "Created a proto-trace for %s (%s:%d) at byte offset %d -- length %d\n", - PyUnicode_AsUTF8(code->co_qualname), - PyUnicode_AsUTF8(code->co_filename), - code->co_firstlineno, - 2 * INSTR_IP(tstate->interp->jit_tracer_initial_instr, (PyCodeObject *)tstate->interp->jit_tracer_initial_func->func_code), - trace_length); - OPT_HIST(trace_length, trace_length_hist); - return trace_length; + tstate->interp->jit_tracer_code_curr_size = trace_length; + tstate->interp->jit_tracer_code_max_size = max_length; + return 0; +} + +void +_PyJIT_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, int curr_stackdepth, int chain_depth) +{ + PyCodeObject *code = _PyFrame_GetCode(frame); +#ifdef Py_DEBUG + char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); + int lltrace = 3; + if (python_lltrace != NULL && *python_lltrace >= '0') { + lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that + } + DPRINTF(2, + "Optimizing %s (%s:%d) at byte offset %d\n", + PyUnicode_AsUTF8(code->co_qualname), + PyUnicode_AsUTF8(code->co_filename), + code->co_firstlineno, + 2 * INSTR_IP(next_instr, code)); +#endif + add_to_trace(tstate->interp->jit_tracer_code_buffer, 0, _START_EXECUTOR, 0, (uintptr_t)next_instr, INSTR_IP(next_instr, code)); + add_to_trace(tstate->interp->jit_tracer_code_buffer, 1, _MAKE_WARM, 0, 0, 0); + tstate->interp->jit_tracer_code_curr_size = 2; + tstate->interp->jit_tracer_code_max_size = UOP_MAX_TRACE_LENGTH; + tstate->interp->jit_tracer_initial_instr = next_instr; + tstate->interp->jit_tracer_initial_code = code; + tstate->interp->jit_tracer_initial_func = _PyFrame_GetFunction(frame); + tstate->interp->jit_tracer_seen_initial_before = 0; + tstate->interp->jit_completed_loop = false; + tstate->interp->jit_tracer_initial_stack_depth = curr_stackdepth; + tstate->interp->jit_tracer_initial_chain_depth = chain_depth; } #undef RESERVE @@ -1123,7 +1086,7 @@ uop_optimize( return 0; } } - _PyUOpInstruction *buffer = interp->jit_uop_buffer; + _PyUOpInstruction *buffer = interp->jit_tracer_code_buffer; OPT_STAT_INC(attempts); char *env_var = Py_GETENV("PYTHON_UOPS_OPTIMIZE"); bool is_noopt = true; @@ -1131,19 +1094,8 @@ uop_optimize( is_noopt = false; } int curr_stackentries = tstate->interp->jit_tracer_initial_stack_depth; -// #ifdef Py_DEBUG -// for (int x = 0; x < tstate->interp->jit_tracer_code_curr_size;) { -// int opcode = tstate->interp->jit_tracer_code_buffer[x].op.code; -// int oparg = tstate->interp->jit_tracer_code_buffer[x].op.arg; -// fprintf(stdout, "%s(%d)\n", _PyOpcode_OpName[opcode], oparg); -// x += 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; -// } -// #endif - int length = translate_bytecode_to_trace(frame, tstate, buffer, UOP_MAX_TRACE_LENGTH, &dependencies, progress_needed); - if (length <= 0) { - // Error or nothing translated - return length; - } + int length = interp->jit_tracer_code_curr_size; + assert(length > 0); assert(length < UOP_MAX_TRACE_LENGTH); OPT_STAT_INC(traces_created); if (!is_noopt) { diff --git a/Python/pystate.c b/Python/pystate.c index 33ffc034b4410c..671daabd1edb23 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -815,7 +815,7 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) interp->jit_uop_buffer = NULL; } if (interp->jit_tracer_code_buffer != NULL) { - _PyObject_VirtualFree(interp->jit_tracer_code_buffer, TRACER_BUFFER_SIZE); + _PyObject_VirtualFree(interp->jit_tracer_code_buffer, UOP_BUFFER_SIZE); interp->jit_tracer_code_buffer = NULL; } #endif From e63de3968b08d3d7ce93f1c57fadb6498aa65be1 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 19 Sep 2025 22:44:30 +0100 Subject: [PATCH 004/190] working python startup --- Python/ceval.c | 7 +- Python/ceval_macros.h | 2 +- Python/generated_cases.c.h | 900 +++++++++++++++++++++++ Python/optimizer.c | 4 +- Tools/cases_generator/tier1_generator.py | 2 + 5 files changed, 908 insertions(+), 7 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index 4029575853d6b6..3f333d0ae30fee 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -975,12 +975,12 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch) // 1 for trace full, 0 for successful write. static int -add_to_code_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *this_instr, _Py_CODEUNIT *next_instr, int oparg) +add_to_code_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, PyCodeObject *old_code, _Py_CODEUNIT *this_instr, _Py_CODEUNIT *next_instr, int oparg) { assert(frame != NULL); assert(tstate->interp->jit_tracer_code_curr_size < UOP_MAX_TRACE_LENGTH); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - return !_PyJIT_translate_single_bytecode_to_trace(tstate, this_instr, next_instr, _PyFrame_GetCode(frame), func, oparg); + return !_PyJIT_translate_single_bytecode_to_trace(tstate, this_instr, next_instr, old_code, func, oparg); } /* _PyEval_EvalFrameDefault is too large to optimize for speed with PGO on MSVC. @@ -1193,10 +1193,9 @@ _PyTier2Interpreter( assert(next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT); tier2_dispatch: for (;;) { - frame->lltrace = 4; uopcode = next_uop->opcode; #ifdef Py_DEBUG - if (1 && next_uop->opcode != _YIELD_VALUE) { + if (frame->lltrace >= 4 && next_uop->opcode != _YIELD_VALUE) { // dump_stack(frame, stack_pointer); if (next_uop->opcode == _START_EXECUTOR) { printf("%4d uop: ", 0); diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index a47b0e524c703a..df0f5d25719fda 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -144,7 +144,7 @@ DISPATCH(); # define RECORD_TRACE() do { \ frame->instr_ptr = next_instr; \ - if (add_to_code_trace(tstate, frame, this_instr, next_instr, oparg)) { \ + if (add_to_code_trace(tstate, frame, old_code, this_instr, next_instr, oparg)) { \ BAIL_TRACING(); \ } \ } while (0); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 99fb53f26c5c48..7d6809b4328c2c 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -23,6 +23,8 @@ int opcode = BINARY_OP; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP); @@ -86,6 +88,8 @@ int opcode = BINARY_OP_ADD_FLOAT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -144,6 +148,8 @@ int opcode = BINARY_OP_ADD_INT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -204,6 +210,8 @@ int opcode = BINARY_OP_ADD_UNICODE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -264,6 +272,8 @@ int opcode = BINARY_OP_EXTEND; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -328,6 +338,8 @@ int opcode = BINARY_OP_INPLACE_ADD_UNICODE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -411,6 +423,8 @@ int opcode = BINARY_OP_MULTIPLY_FLOAT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -469,6 +483,8 @@ int opcode = BINARY_OP_MULTIPLY_INT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -529,6 +545,8 @@ int opcode = BINARY_OP_SUBSCR_DICT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -595,6 +613,8 @@ int opcode = BINARY_OP_SUBSCR_GETITEM; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -679,6 +699,8 @@ int opcode = BINARY_OP_SUBSCR_LIST_INT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -769,6 +791,8 @@ int opcode = BINARY_OP_SUBSCR_LIST_SLICE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -841,6 +865,8 @@ int opcode = BINARY_OP_SUBSCR_STR_INT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -919,6 +945,8 @@ int opcode = BINARY_OP_SUBSCR_TUPLE_INT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -992,6 +1020,8 @@ int opcode = BINARY_OP_SUBTRACT_FLOAT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -1050,6 +1080,8 @@ int opcode = BINARY_OP_SUBTRACT_INT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -1110,6 +1142,8 @@ int opcode = BINARY_SLICE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BINARY_SLICE); @@ -1166,6 +1200,8 @@ int opcode = BUILD_INTERPOLATION; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_INTERPOLATION); @@ -1224,6 +1260,8 @@ int opcode = BUILD_LIST; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_LIST); @@ -1248,6 +1286,8 @@ int opcode = BUILD_MAP; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_MAP); @@ -1300,6 +1340,8 @@ int opcode = BUILD_SET; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_SET); @@ -1357,6 +1399,8 @@ int opcode = BUILD_SLICE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_SLICE); @@ -1392,6 +1436,8 @@ int opcode = BUILD_STRING; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_STRING); @@ -1439,6 +1485,8 @@ int opcode = BUILD_TEMPLATE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_TEMPLATE); @@ -1477,6 +1525,8 @@ int opcode = BUILD_TUPLE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_TUPLE); @@ -1499,6 +1549,8 @@ int opcode = CACHE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CACHE); @@ -1512,6 +1564,8 @@ int opcode = CALL; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL); @@ -1690,6 +1744,8 @@ int opcode = CALL_ALLOC_AND_ENTER_INIT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -1811,6 +1867,8 @@ int opcode = CALL_BOUND_METHOD_EXACT_ARGS; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -1954,6 +2012,8 @@ int opcode = CALL_BOUND_METHOD_GENERAL; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -2082,6 +2142,8 @@ int opcode = CALL_BUILTIN_CLASS; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -2187,6 +2249,8 @@ int opcode = CALL_BUILTIN_FAST; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -2296,6 +2360,8 @@ int opcode = CALL_BUILTIN_FAST_WITH_KEYWORDS; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -2405,6 +2471,8 @@ int opcode = CALL_BUILTIN_O; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -2489,6 +2557,8 @@ int opcode = CALL_FUNCTION_EX; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -2656,6 +2726,8 @@ int opcode = CALL_INTRINSIC_1; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CALL_INTRINSIC_1); @@ -2686,6 +2758,8 @@ int opcode = CALL_INTRINSIC_2; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CALL_INTRINSIC_2); @@ -2725,6 +2799,8 @@ int opcode = CALL_ISINSTANCE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -2801,6 +2877,8 @@ int opcode = CALL_KW; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW); @@ -2983,6 +3061,8 @@ int opcode = CALL_KW_BOUND_METHOD; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -3113,6 +3193,8 @@ int opcode = CALL_KW_NON_PY; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -3240,6 +3322,8 @@ int opcode = CALL_KW_PY; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -3350,6 +3434,8 @@ int opcode = CALL_LEN; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -3422,6 +3508,8 @@ int opcode = CALL_LIST_APPEND; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -3510,6 +3598,8 @@ int opcode = CALL_METHOD_DESCRIPTOR_FAST; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -3631,6 +3721,8 @@ int opcode = CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -3754,6 +3846,8 @@ int opcode = CALL_METHOD_DESCRIPTOR_NOARGS; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -3847,6 +3941,8 @@ int opcode = CALL_METHOD_DESCRIPTOR_O; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -3953,6 +4049,8 @@ int opcode = CALL_NON_PY_GENERAL; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -4068,6 +4166,8 @@ int opcode = CALL_PY_EXACT_ARGS; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -4181,6 +4281,8 @@ int opcode = CALL_PY_GENERAL; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -4281,6 +4383,8 @@ int opcode = CALL_STR_1; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -4353,6 +4457,8 @@ int opcode = CALL_TUPLE_1; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -4425,6 +4531,8 @@ int opcode = CALL_TYPE_1; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -4480,6 +4588,8 @@ int opcode = CHECK_EG_MATCH; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CHECK_EG_MATCH); @@ -4551,6 +4661,8 @@ int opcode = CHECK_EXC_MATCH; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CHECK_EXC_MATCH); @@ -4588,6 +4700,8 @@ int opcode = CLEANUP_THROW; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -4647,6 +4761,8 @@ int opcode = COMPARE_OP; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP); @@ -4720,6 +4836,8 @@ int opcode = COMPARE_OP_FLOAT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -4775,6 +4893,8 @@ int opcode = COMPARE_OP_INT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -4834,6 +4954,8 @@ int opcode = COMPARE_OP_STR; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -4893,6 +5015,8 @@ int opcode = CONTAINS_OP; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP); @@ -4953,6 +5077,8 @@ int opcode = CONTAINS_OP_DICT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -5011,6 +5137,8 @@ int opcode = CONTAINS_OP_SET; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -5069,6 +5197,8 @@ int opcode = CONVERT_VALUE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CONVERT_VALUE); @@ -5101,6 +5231,8 @@ int opcode = COPY; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(COPY); @@ -5119,6 +5251,8 @@ int opcode = COPY_FREE_VARS; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(COPY_FREE_VARS); @@ -5140,6 +5274,8 @@ int opcode = DELETE_ATTR; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_ATTR); @@ -5165,6 +5301,8 @@ int opcode = DELETE_DEREF; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_DEREF); @@ -5187,6 +5325,8 @@ int opcode = DELETE_FAST; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_FAST); @@ -5213,6 +5353,8 @@ int opcode = DELETE_GLOBAL; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_GLOBAL); @@ -5238,6 +5380,8 @@ int opcode = DELETE_NAME; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_NAME); @@ -5270,6 +5414,8 @@ int opcode = DELETE_SUBSCR; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_SUBSCR); @@ -5302,6 +5448,8 @@ int opcode = DICT_MERGE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DICT_MERGE); @@ -5341,6 +5489,8 @@ int opcode = DICT_UPDATE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DICT_UPDATE); @@ -5384,6 +5534,8 @@ int opcode = END_ASYNC_FOR; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -5429,6 +5581,8 @@ int opcode = END_FOR; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; next_instr += 1; INSTRUCTION_STATS(END_FOR); _PyStackRef value; @@ -5446,6 +5600,8 @@ int opcode = END_SEND; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(END_SEND); @@ -5469,6 +5625,8 @@ int opcode = ENTER_EXECUTOR; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -5505,6 +5663,8 @@ int opcode = EXIT_INIT_CHECK; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(EXIT_INIT_CHECK); @@ -5528,6 +5688,8 @@ int opcode = EXTENDED_ARG; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(EXTENDED_ARG); @@ -5544,6 +5706,8 @@ int opcode = FORMAT_SIMPLE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(FORMAT_SIMPLE); @@ -5580,6 +5744,8 @@ int opcode = FORMAT_WITH_SPEC; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(FORMAT_WITH_SPEC); @@ -5616,6 +5782,8 @@ int opcode = FOR_ITER; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER); @@ -5670,6 +5838,8 @@ int opcode = FOR_ITER_GEN; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -5741,6 +5911,8 @@ int opcode = FOR_ITER_LIST; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -5825,6 +5997,8 @@ int opcode = FOR_ITER_RANGE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -5892,6 +6066,8 @@ int opcode = FOR_ITER_TUPLE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -5948,6 +6124,8 @@ int opcode = GET_AITER; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_AITER); @@ -6009,6 +6187,8 @@ int opcode = GET_ANEXT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_ANEXT); @@ -6033,6 +6213,8 @@ int opcode = GET_AWAITABLE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_AWAITABLE); @@ -6062,6 +6244,8 @@ int opcode = GET_ITER; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_ITER); @@ -6108,6 +6292,8 @@ int opcode = GET_LEN; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_LEN); @@ -6136,6 +6322,8 @@ int opcode = GET_YIELD_FROM_ITER; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_YIELD_FROM_ITER); @@ -6181,6 +6369,8 @@ int opcode = IMPORT_FROM; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IMPORT_FROM); @@ -6206,6 +6396,8 @@ int opcode = IMPORT_NAME; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IMPORT_NAME); @@ -6245,6 +6437,8 @@ int opcode = INSTRUMENTED_CALL; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -6433,6 +6627,8 @@ int opcode = INSTRUMENTED_CALL_FUNCTION_EX; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -6600,6 +6796,8 @@ int opcode = INSTRUMENTED_CALL_KW; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -6786,6 +6984,8 @@ int opcode = INSTRUMENTED_END_ASYNC_FOR; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -6839,6 +7039,8 @@ int opcode = INSTRUMENTED_END_FOR; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; next_instr += 1; @@ -6868,6 +7070,8 @@ int opcode = INSTRUMENTED_END_SEND; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -6902,6 +7106,8 @@ int opcode = INSTRUMENTED_FOR_ITER; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -6938,6 +7144,8 @@ int opcode = INSTRUMENTED_INSTRUCTION; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -6965,6 +7173,8 @@ int opcode = INSTRUMENTED_JUMP_BACKWARD; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -6992,6 +7202,8 @@ int opcode = INSTRUMENTED_JUMP_FORWARD; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -7006,6 +7218,8 @@ int opcode = INSTRUMENTED_LINE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -7046,6 +7260,8 @@ int opcode = INSTRUMENTED_LOAD_SUPER_ATTR; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -7166,6 +7382,8 @@ int opcode = INSTRUMENTED_NOT_TAKEN; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -7182,6 +7400,8 @@ int opcode = INSTRUMENTED_POP_ITER; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -7207,6 +7427,8 @@ int opcode = INSTRUMENTED_POP_JUMP_IF_FALSE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -7231,6 +7453,8 @@ int opcode = INSTRUMENTED_POP_JUMP_IF_NONE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -7262,6 +7486,8 @@ int opcode = INSTRUMENTED_POP_JUMP_IF_NOT_NONE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -7291,6 +7517,8 @@ int opcode = INSTRUMENTED_POP_JUMP_IF_TRUE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -7315,6 +7543,8 @@ int opcode = INSTRUMENTED_RESUME; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -7395,6 +7625,8 @@ int opcode = INSTRUMENTED_RETURN_VALUE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -7447,6 +7679,8 @@ int opcode = INSTRUMENTED_YIELD_VALUE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -7518,6 +7752,8 @@ int opcode = INTERPRETER_EXIT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INTERPRETER_EXIT); @@ -7553,6 +7789,8 @@ int opcode = IS_OP; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IS_OP); @@ -7586,6 +7824,8 @@ int opcode = JUMP_BACKWARD; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(JUMP_BACKWARD); @@ -7627,6 +7867,8 @@ int opcode = JUMP_BACKWARD_JIT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -7688,6 +7930,8 @@ int opcode = JUMP_BACKWARD_NO_INTERRUPT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); @@ -7703,6 +7947,8 @@ int opcode = JUMP_BACKWARD_NO_JIT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(JUMP_BACKWARD_NO_JIT); @@ -7732,6 +7978,8 @@ int opcode = JUMP_FORWARD; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(JUMP_FORWARD); @@ -7744,6 +7992,8 @@ int opcode = LIST_APPEND; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LIST_APPEND); @@ -7766,6 +8016,8 @@ int opcode = LIST_EXTEND; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LIST_EXTEND); @@ -7813,6 +8065,8 @@ int opcode = LOAD_ATTR; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR); @@ -7894,6 +8148,8 @@ int opcode = LOAD_ATTR_CLASS; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -7952,6 +8208,8 @@ int opcode = LOAD_ATTR_CLASS_WITH_METACLASS_CHECK; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -8020,6 +8278,8 @@ int opcode = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -8078,6 +8338,8 @@ int opcode = LOAD_ATTR_INSTANCE_VALUE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -8158,6 +8420,8 @@ int opcode = LOAD_ATTR_METHOD_LAZY_DICT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -8214,6 +8478,8 @@ int opcode = LOAD_ATTR_METHOD_NO_DICT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -8260,6 +8526,8 @@ int opcode = LOAD_ATTR_METHOD_WITH_VALUES; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -8327,6 +8595,8 @@ int opcode = LOAD_ATTR_MODULE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -8401,6 +8671,8 @@ int opcode = LOAD_ATTR_NONDESCRIPTOR_NO_DICT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -8448,6 +8720,8 @@ int opcode = LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -8516,6 +8790,8 @@ int opcode = LOAD_ATTR_PROPERTY; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -8610,6 +8886,8 @@ int opcode = LOAD_ATTR_SLOT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -8679,6 +8957,8 @@ int opcode = LOAD_ATTR_WITH_HINT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -8789,6 +9069,8 @@ int opcode = LOAD_BUILD_CLASS; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_BUILD_CLASS); @@ -8819,6 +9101,8 @@ int opcode = LOAD_COMMON_CONSTANT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); @@ -8836,6 +9120,8 @@ int opcode = LOAD_CONST; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_CONST); @@ -8853,6 +9139,8 @@ int opcode = LOAD_DEREF; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_DEREF); @@ -8881,6 +9169,8 @@ int opcode = LOAD_FAST; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST); @@ -8898,6 +9188,8 @@ int opcode = LOAD_FAST_AND_CLEAR; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); @@ -8915,6 +9207,8 @@ int opcode = LOAD_FAST_BORROW; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_BORROW); @@ -8932,6 +9226,8 @@ int opcode = LOAD_FAST_BORROW_LOAD_FAST_BORROW; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_BORROW_LOAD_FAST_BORROW); @@ -8953,6 +9249,8 @@ int opcode = LOAD_FAST_CHECK; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_CHECK); @@ -8979,6 +9277,8 @@ int opcode = LOAD_FAST_LOAD_FAST; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); @@ -9000,6 +9300,8 @@ int opcode = LOAD_FROM_DICT_OR_DEREF; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); @@ -9045,6 +9347,8 @@ int opcode = LOAD_FROM_DICT_OR_GLOBALS; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); @@ -9120,6 +9424,8 @@ int opcode = LOAD_GLOBAL; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL); @@ -9176,6 +9482,8 @@ int opcode = LOAD_GLOBAL_BUILTIN; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -9256,6 +9564,8 @@ int opcode = LOAD_GLOBAL_MODULE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -9323,6 +9633,8 @@ int opcode = LOAD_LOCALS; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_LOCALS); @@ -9347,6 +9659,8 @@ int opcode = LOAD_NAME; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_NAME); @@ -9370,6 +9684,8 @@ int opcode = LOAD_SMALL_INT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_SMALL_INT); @@ -9388,6 +9704,8 @@ int opcode = LOAD_SPECIAL; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_SPECIAL); @@ -9434,6 +9752,8 @@ int opcode = LOAD_SUPER_ATTR; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR); @@ -9571,6 +9891,8 @@ int opcode = LOAD_SUPER_ATTR_ATTR; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -9633,6 +9955,8 @@ int opcode = LOAD_SUPER_ATTR_METHOD; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -9711,6 +10035,8 @@ int opcode = MAKE_CELL; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAKE_CELL); @@ -9732,6 +10058,8 @@ int opcode = MAKE_FUNCTION; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAKE_FUNCTION); @@ -9765,6 +10093,8 @@ int opcode = MAP_ADD; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAP_ADD); @@ -9796,6 +10126,8 @@ int opcode = MATCH_CLASS; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_CLASS); @@ -9848,6 +10180,8 @@ int opcode = MATCH_KEYS; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_KEYS); @@ -9875,6 +10209,8 @@ int opcode = MATCH_MAPPING; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_MAPPING); @@ -9894,6 +10230,8 @@ int opcode = MATCH_SEQUENCE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_SEQUENCE); @@ -9913,6 +10251,8 @@ int opcode = NOP; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(NOP); @@ -9924,6 +10264,8 @@ int opcode = NOT_TAKEN; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(NOT_TAKEN); @@ -9935,6 +10277,8 @@ int opcode = POP_EXCEPT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_EXCEPT); @@ -9956,6 +10300,8 @@ int opcode = POP_ITER; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_ITER); @@ -9977,6 +10323,8 @@ int opcode = POP_JUMP_IF_FALSE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_FALSE); @@ -9996,6 +10344,8 @@ int opcode = POP_JUMP_IF_NONE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_NONE); @@ -10036,6 +10386,8 @@ int opcode = POP_JUMP_IF_NOT_NONE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); @@ -10076,6 +10428,8 @@ int opcode = POP_JUMP_IF_TRUE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_TRUE); @@ -10095,6 +10449,8 @@ int opcode = POP_TOP; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_TOP); @@ -10113,6 +10469,8 @@ int opcode = PUSH_EXC_INFO; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(PUSH_EXC_INFO); @@ -10142,6 +10500,8 @@ int opcode = PUSH_NULL; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(PUSH_NULL); @@ -10158,6 +10518,8 @@ int opcode = RAISE_VARARGS; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -10187,6 +10549,8 @@ int opcode = RERAISE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -10215,6 +10579,8 @@ int opcode = RESERVED; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESERVED); @@ -10228,6 +10594,8 @@ int opcode = RESUME; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESUME); @@ -10304,6 +10672,8 @@ int opcode = RESUME_CHECK; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -10342,6 +10712,8 @@ int opcode = RETURN_GENERATOR; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RETURN_GENERATOR); @@ -10384,6 +10756,8 @@ int opcode = RETURN_VALUE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RETURN_VALUE); @@ -10418,6 +10792,8 @@ int opcode = SEND; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(SEND); @@ -10523,6 +10899,8 @@ int opcode = SEND_GEN; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -10592,6 +10970,8 @@ int opcode = SETUP_ANNOTATIONS; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SETUP_ANNOTATIONS); @@ -10638,6 +11018,8 @@ int opcode = SET_ADD; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_ADD); @@ -10662,6 +11044,8 @@ int opcode = SET_FUNCTION_ATTRIBUTE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); @@ -10690,6 +11074,8 @@ int opcode = SET_UPDATE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_UPDATE); @@ -10717,6 +11103,8 @@ int opcode = STORE_ATTR; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR); @@ -10774,6 +11162,8 @@ int opcode = STORE_ATTR_INSTANCE_VALUE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -10850,6 +11240,8 @@ int opcode = STORE_ATTR_SLOT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -10901,6 +11293,8 @@ int opcode = STORE_ATTR_WITH_HINT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -10989,6 +11383,8 @@ int opcode = STORE_DEREF; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_DEREF); @@ -11008,6 +11404,8 @@ int opcode = STORE_FAST; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_FAST); @@ -11028,6 +11426,8 @@ int opcode = STORE_FAST_LOAD_FAST; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); @@ -11051,6 +11451,8 @@ int opcode = STORE_FAST_STORE_FAST; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_FAST_STORE_FAST); @@ -11082,6 +11484,8 @@ int opcode = STORE_GLOBAL; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_GLOBAL); @@ -11107,6 +11511,8 @@ int opcode = STORE_NAME; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_NAME); @@ -11153,6 +11559,8 @@ int opcode = STORE_SLICE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_SLICE); @@ -11213,6 +11621,8 @@ int opcode = STORE_SUBSCR; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR); @@ -11272,6 +11682,8 @@ int opcode = STORE_SUBSCR_DICT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -11323,6 +11735,8 @@ int opcode = STORE_SUBSCR_LIST_INT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -11404,6 +11818,8 @@ int opcode = SWAP; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SWAP); @@ -11424,6 +11840,8 @@ int opcode = TO_BOOL; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL); @@ -11476,6 +11894,8 @@ int opcode = TO_BOOL_ALWAYS_TRUE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -11519,6 +11939,8 @@ int opcode = TO_BOOL_BOOL; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -11543,6 +11965,8 @@ int opcode = TO_BOOL_INT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -11583,6 +12007,8 @@ int opcode = TO_BOOL_LIST; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -11626,6 +12052,8 @@ int opcode = TO_BOOL_NONE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -11653,6 +12081,8 @@ int opcode = TO_BOOL_STR; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -11701,6 +12131,8 @@ int opcode = UNARY_INVERT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNARY_INVERT); @@ -11730,6 +12162,8 @@ int opcode = UNARY_NEGATIVE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNARY_NEGATIVE); @@ -11759,6 +12193,8 @@ int opcode = UNARY_NOT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNARY_NOT); @@ -11777,6 +12213,8 @@ int opcode = UNPACK_EX; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNPACK_EX); @@ -11804,6 +12242,8 @@ int opcode = UNPACK_SEQUENCE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE); @@ -11855,6 +12295,8 @@ int opcode = UNPACK_SEQUENCE_LIST; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -11914,6 +12356,8 @@ int opcode = UNPACK_SEQUENCE_TUPLE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -11964,6 +12408,8 @@ int opcode = UNPACK_SEQUENCE_TWO_TUPLE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -12015,6 +12461,8 @@ int opcode = WITH_EXCEPT_START; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(WITH_EXCEPT_START); @@ -12060,6 +12508,8 @@ int opcode = YIELD_VALUE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(YIELD_VALUE); @@ -12111,6 +12561,8 @@ int opcode = BINARY_OP; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP); @@ -12174,6 +12626,8 @@ int opcode = BINARY_OP_ADD_FLOAT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -12232,6 +12686,8 @@ int opcode = BINARY_OP_ADD_INT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -12292,6 +12748,8 @@ int opcode = BINARY_OP_ADD_UNICODE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -12352,6 +12810,8 @@ int opcode = BINARY_OP_EXTEND; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -12416,6 +12876,8 @@ int opcode = BINARY_OP_INPLACE_ADD_UNICODE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -12499,6 +12961,8 @@ int opcode = BINARY_OP_MULTIPLY_FLOAT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -12557,6 +13021,8 @@ int opcode = BINARY_OP_MULTIPLY_INT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -12617,6 +13083,8 @@ int opcode = BINARY_OP_SUBSCR_DICT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -12683,6 +13151,8 @@ int opcode = BINARY_OP_SUBSCR_GETITEM; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -12767,6 +13237,8 @@ int opcode = BINARY_OP_SUBSCR_LIST_INT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -12857,6 +13329,8 @@ int opcode = BINARY_OP_SUBSCR_LIST_SLICE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -12929,6 +13403,8 @@ int opcode = BINARY_OP_SUBSCR_STR_INT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -13007,6 +13483,8 @@ int opcode = BINARY_OP_SUBSCR_TUPLE_INT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -13080,6 +13558,8 @@ int opcode = BINARY_OP_SUBTRACT_FLOAT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -13138,6 +13618,8 @@ int opcode = BINARY_OP_SUBTRACT_INT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -13198,6 +13680,8 @@ int opcode = BINARY_SLICE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -13256,6 +13740,8 @@ int opcode = BUILD_INTERPOLATION; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -13316,6 +13802,8 @@ int opcode = BUILD_LIST; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -13342,6 +13830,8 @@ int opcode = BUILD_MAP; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -13396,6 +13886,8 @@ int opcode = BUILD_SET; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -13455,6 +13947,8 @@ int opcode = BUILD_SLICE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -13492,6 +13986,8 @@ int opcode = BUILD_STRING; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -13541,6 +14037,8 @@ int opcode = BUILD_TEMPLATE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -13581,6 +14079,8 @@ int opcode = BUILD_TUPLE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -13605,6 +14105,8 @@ int opcode = CACHE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -13620,6 +14122,8 @@ int opcode = CALL; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL); @@ -13798,6 +14302,8 @@ int opcode = CALL_ALLOC_AND_ENTER_INIT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -13919,6 +14425,8 @@ int opcode = CALL_BOUND_METHOD_EXACT_ARGS; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -14062,6 +14570,8 @@ int opcode = CALL_BOUND_METHOD_GENERAL; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -14190,6 +14700,8 @@ int opcode = CALL_BUILTIN_CLASS; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -14295,6 +14807,8 @@ int opcode = CALL_BUILTIN_FAST; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -14404,6 +14918,8 @@ int opcode = CALL_BUILTIN_FAST_WITH_KEYWORDS; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -14513,6 +15029,8 @@ int opcode = CALL_BUILTIN_O; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -14597,6 +15115,8 @@ int opcode = CALL_FUNCTION_EX; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -14764,6 +15284,8 @@ int opcode = CALL_INTRINSIC_1; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -14796,6 +15318,8 @@ int opcode = CALL_INTRINSIC_2; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -14837,6 +15361,8 @@ int opcode = CALL_ISINSTANCE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -14913,6 +15439,8 @@ int opcode = CALL_KW; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW); @@ -15095,6 +15623,8 @@ int opcode = CALL_KW_BOUND_METHOD; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -15225,6 +15755,8 @@ int opcode = CALL_KW_NON_PY; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -15352,6 +15884,8 @@ int opcode = CALL_KW_PY; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -15462,6 +15996,8 @@ int opcode = CALL_LEN; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -15534,6 +16070,8 @@ int opcode = CALL_LIST_APPEND; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -15622,6 +16160,8 @@ int opcode = CALL_METHOD_DESCRIPTOR_FAST; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -15743,6 +16283,8 @@ int opcode = CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -15866,6 +16408,8 @@ int opcode = CALL_METHOD_DESCRIPTOR_NOARGS; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -15959,6 +16503,8 @@ int opcode = CALL_METHOD_DESCRIPTOR_O; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -16065,6 +16611,8 @@ int opcode = CALL_NON_PY_GENERAL; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -16180,6 +16728,8 @@ int opcode = CALL_PY_EXACT_ARGS; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -16293,6 +16843,8 @@ int opcode = CALL_PY_GENERAL; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -16393,6 +16945,8 @@ int opcode = CALL_STR_1; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -16465,6 +17019,8 @@ int opcode = CALL_TUPLE_1; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -16537,6 +17093,8 @@ int opcode = CALL_TYPE_1; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -16592,6 +17150,8 @@ int opcode = CHECK_EG_MATCH; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -16665,6 +17225,8 @@ int opcode = CHECK_EXC_MATCH; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -16704,6 +17266,8 @@ int opcode = CLEANUP_THROW; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -16763,6 +17327,8 @@ int opcode = COMPARE_OP; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP); @@ -16836,6 +17402,8 @@ int opcode = COMPARE_OP_FLOAT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -16891,6 +17459,8 @@ int opcode = COMPARE_OP_INT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -16950,6 +17520,8 @@ int opcode = COMPARE_OP_STR; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -17009,6 +17581,8 @@ int opcode = CONTAINS_OP; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP); @@ -17069,6 +17643,8 @@ int opcode = CONTAINS_OP_DICT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -17127,6 +17703,8 @@ int opcode = CONTAINS_OP_SET; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -17185,6 +17763,8 @@ int opcode = CONVERT_VALUE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -17219,6 +17799,8 @@ int opcode = COPY; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -17239,6 +17821,8 @@ int opcode = COPY_FREE_VARS; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -17262,6 +17846,8 @@ int opcode = DELETE_ATTR; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -17289,6 +17875,8 @@ int opcode = DELETE_DEREF; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -17313,6 +17901,8 @@ int opcode = DELETE_FAST; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -17341,6 +17931,8 @@ int opcode = DELETE_GLOBAL; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -17368,6 +17960,8 @@ int opcode = DELETE_NAME; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -17402,6 +17996,8 @@ int opcode = DELETE_SUBSCR; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -17436,6 +18032,8 @@ int opcode = DICT_MERGE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -17477,6 +18075,8 @@ int opcode = DICT_UPDATE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -17522,6 +18122,8 @@ int opcode = END_ASYNC_FOR; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -17567,6 +18169,8 @@ int opcode = END_FOR; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; next_instr += 1; @@ -17586,6 +18190,8 @@ int opcode = END_SEND; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -17611,6 +18217,8 @@ int opcode = ENTER_EXECUTOR; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -17647,6 +18255,8 @@ int opcode = EXIT_INIT_CHECK; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -17672,6 +18282,8 @@ int opcode = EXTENDED_ARG; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -17690,6 +18302,8 @@ int opcode = FORMAT_SIMPLE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -17728,6 +18342,8 @@ int opcode = FORMAT_WITH_SPEC; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -17766,6 +18382,8 @@ int opcode = FOR_ITER; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER); @@ -17820,6 +18438,8 @@ int opcode = FOR_ITER_GEN; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -17891,6 +18511,8 @@ int opcode = FOR_ITER_LIST; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -17975,6 +18597,8 @@ int opcode = FOR_ITER_RANGE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -18042,6 +18666,8 @@ int opcode = FOR_ITER_TUPLE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -18098,6 +18724,8 @@ int opcode = GET_AITER; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -18161,6 +18789,8 @@ int opcode = GET_ANEXT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -18187,6 +18817,8 @@ int opcode = GET_AWAITABLE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -18218,6 +18850,8 @@ int opcode = GET_ITER; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -18266,6 +18900,8 @@ int opcode = GET_LEN; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -18296,6 +18932,8 @@ int opcode = GET_YIELD_FROM_ITER; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -18343,6 +18981,8 @@ int opcode = IMPORT_FROM; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -18370,6 +19010,8 @@ int opcode = IMPORT_NAME; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -18411,6 +19053,8 @@ int opcode = INSTRUMENTED_CALL; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -18599,6 +19243,8 @@ int opcode = INSTRUMENTED_CALL_FUNCTION_EX; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -18766,6 +19412,8 @@ int opcode = INSTRUMENTED_CALL_KW; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -18952,6 +19600,8 @@ int opcode = INSTRUMENTED_END_ASYNC_FOR; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -19005,6 +19655,8 @@ int opcode = INSTRUMENTED_END_FOR; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; next_instr += 1; @@ -19034,6 +19686,8 @@ int opcode = INSTRUMENTED_END_SEND; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -19068,6 +19722,8 @@ int opcode = INSTRUMENTED_FOR_ITER; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -19104,6 +19760,8 @@ int opcode = INSTRUMENTED_INSTRUCTION; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -19131,6 +19789,8 @@ int opcode = INSTRUMENTED_JUMP_BACKWARD; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -19158,6 +19818,8 @@ int opcode = INSTRUMENTED_JUMP_FORWARD; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -19172,6 +19834,8 @@ int opcode = INSTRUMENTED_LINE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -19212,6 +19876,8 @@ int opcode = INSTRUMENTED_LOAD_SUPER_ATTR; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -19332,6 +19998,8 @@ int opcode = INSTRUMENTED_NOT_TAKEN; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -19348,6 +20016,8 @@ int opcode = INSTRUMENTED_POP_ITER; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -19373,6 +20043,8 @@ int opcode = INSTRUMENTED_POP_JUMP_IF_FALSE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -19397,6 +20069,8 @@ int opcode = INSTRUMENTED_POP_JUMP_IF_NONE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -19428,6 +20102,8 @@ int opcode = INSTRUMENTED_POP_JUMP_IF_NOT_NONE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -19457,6 +20133,8 @@ int opcode = INSTRUMENTED_POP_JUMP_IF_TRUE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -19481,6 +20159,8 @@ int opcode = INSTRUMENTED_RESUME; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -19561,6 +20241,8 @@ int opcode = INSTRUMENTED_RETURN_VALUE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -19613,6 +20295,8 @@ int opcode = INSTRUMENTED_YIELD_VALUE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -19684,6 +20368,8 @@ int opcode = INTERPRETER_EXIT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -19721,6 +20407,8 @@ int opcode = IS_OP; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -19756,6 +20444,8 @@ int opcode = JUMP_BACKWARD; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(JUMP_BACKWARD); @@ -19797,6 +20487,8 @@ int opcode = JUMP_BACKWARD_JIT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -19858,6 +20550,8 @@ int opcode = JUMP_BACKWARD_NO_INTERRUPT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -19875,6 +20569,8 @@ int opcode = JUMP_BACKWARD_NO_JIT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -19906,6 +20602,8 @@ int opcode = JUMP_FORWARD; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -19920,6 +20618,8 @@ int opcode = LIST_APPEND; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -19944,6 +20644,8 @@ int opcode = LIST_EXTEND; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -19993,6 +20695,8 @@ int opcode = LOAD_ATTR; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR); @@ -20074,6 +20778,8 @@ int opcode = LOAD_ATTR_CLASS; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -20132,6 +20838,8 @@ int opcode = LOAD_ATTR_CLASS_WITH_METACLASS_CHECK; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -20200,6 +20908,8 @@ int opcode = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -20258,6 +20968,8 @@ int opcode = LOAD_ATTR_INSTANCE_VALUE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -20338,6 +21050,8 @@ int opcode = LOAD_ATTR_METHOD_LAZY_DICT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -20394,6 +21108,8 @@ int opcode = LOAD_ATTR_METHOD_NO_DICT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -20440,6 +21156,8 @@ int opcode = LOAD_ATTR_METHOD_WITH_VALUES; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -20507,6 +21225,8 @@ int opcode = LOAD_ATTR_MODULE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -20581,6 +21301,8 @@ int opcode = LOAD_ATTR_NONDESCRIPTOR_NO_DICT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -20628,6 +21350,8 @@ int opcode = LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -20696,6 +21420,8 @@ int opcode = LOAD_ATTR_PROPERTY; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -20790,6 +21516,8 @@ int opcode = LOAD_ATTR_SLOT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -20859,6 +21587,8 @@ int opcode = LOAD_ATTR_WITH_HINT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -20969,6 +21699,8 @@ int opcode = LOAD_BUILD_CLASS; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -21001,6 +21733,8 @@ int opcode = LOAD_COMMON_CONSTANT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -21020,6 +21754,8 @@ int opcode = LOAD_CONST; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -21039,6 +21775,8 @@ int opcode = LOAD_DEREF; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -21069,6 +21807,8 @@ int opcode = LOAD_FAST; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -21088,6 +21828,8 @@ int opcode = LOAD_FAST_AND_CLEAR; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -21107,6 +21849,8 @@ int opcode = LOAD_FAST_BORROW; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -21126,6 +21870,8 @@ int opcode = LOAD_FAST_BORROW_LOAD_FAST_BORROW; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -21149,6 +21895,8 @@ int opcode = LOAD_FAST_CHECK; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -21177,6 +21925,8 @@ int opcode = LOAD_FAST_LOAD_FAST; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -21200,6 +21950,8 @@ int opcode = LOAD_FROM_DICT_OR_DEREF; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -21247,6 +21999,8 @@ int opcode = LOAD_FROM_DICT_OR_GLOBALS; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -21324,6 +22078,8 @@ int opcode = LOAD_GLOBAL; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL); @@ -21380,6 +22136,8 @@ int opcode = LOAD_GLOBAL_BUILTIN; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -21460,6 +22218,8 @@ int opcode = LOAD_GLOBAL_MODULE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -21527,6 +22287,8 @@ int opcode = LOAD_LOCALS; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -21553,6 +22315,8 @@ int opcode = LOAD_NAME; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -21578,6 +22342,8 @@ int opcode = LOAD_SMALL_INT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -21598,6 +22364,8 @@ int opcode = LOAD_SPECIAL; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -21646,6 +22414,8 @@ int opcode = LOAD_SUPER_ATTR; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR); @@ -21783,6 +22553,8 @@ int opcode = LOAD_SUPER_ATTR_ATTR; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -21845,6 +22617,8 @@ int opcode = LOAD_SUPER_ATTR_METHOD; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -21923,6 +22697,8 @@ int opcode = MAKE_CELL; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -21946,6 +22722,8 @@ int opcode = MAKE_FUNCTION; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -21981,6 +22759,8 @@ int opcode = MAP_ADD; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22014,6 +22794,8 @@ int opcode = MATCH_CLASS; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22068,6 +22850,8 @@ int opcode = MATCH_KEYS; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22097,6 +22881,8 @@ int opcode = MATCH_MAPPING; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22118,6 +22904,8 @@ int opcode = MATCH_SEQUENCE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22139,6 +22927,8 @@ int opcode = NOP; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22152,6 +22942,8 @@ int opcode = NOT_TAKEN; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22165,6 +22957,8 @@ int opcode = POP_EXCEPT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22188,6 +22982,8 @@ int opcode = POP_ITER; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22211,6 +23007,8 @@ int opcode = POP_JUMP_IF_FALSE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22232,6 +23030,8 @@ int opcode = POP_JUMP_IF_NONE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22274,6 +23074,8 @@ int opcode = POP_JUMP_IF_NOT_NONE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22316,6 +23118,8 @@ int opcode = POP_JUMP_IF_TRUE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22337,6 +23141,8 @@ int opcode = POP_TOP; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22357,6 +23163,8 @@ int opcode = PUSH_EXC_INFO; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22388,6 +23196,8 @@ int opcode = PUSH_NULL; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22406,6 +23216,8 @@ int opcode = RAISE_VARARGS; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22435,6 +23247,8 @@ int opcode = RERAISE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22463,6 +23277,8 @@ int opcode = RESERVED; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22478,6 +23294,8 @@ int opcode = RESUME; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESUME); @@ -22554,6 +23372,8 @@ int opcode = RESUME_CHECK; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22592,6 +23412,8 @@ int opcode = RETURN_GENERATOR; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22636,6 +23458,8 @@ int opcode = RETURN_VALUE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22672,6 +23496,8 @@ int opcode = SEND; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(SEND); @@ -22777,6 +23603,8 @@ int opcode = SEND_GEN; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22846,6 +23674,8 @@ int opcode = SETUP_ANNOTATIONS; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22894,6 +23724,8 @@ int opcode = SET_ADD; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22920,6 +23752,8 @@ int opcode = SET_FUNCTION_ATTRIBUTE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22950,6 +23784,8 @@ int opcode = SET_UPDATE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22979,6 +23815,8 @@ int opcode = STORE_ATTR; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR); @@ -23036,6 +23874,8 @@ int opcode = STORE_ATTR_INSTANCE_VALUE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23112,6 +23952,8 @@ int opcode = STORE_ATTR_SLOT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23163,6 +24005,8 @@ int opcode = STORE_ATTR_WITH_HINT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23251,6 +24095,8 @@ int opcode = STORE_DEREF; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23272,6 +24118,8 @@ int opcode = STORE_FAST; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23294,6 +24142,8 @@ int opcode = STORE_FAST_LOAD_FAST; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23319,6 +24169,8 @@ int opcode = STORE_FAST_STORE_FAST; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23352,6 +24204,8 @@ int opcode = STORE_GLOBAL; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23379,6 +24233,8 @@ int opcode = STORE_NAME; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23427,6 +24283,8 @@ int opcode = STORE_SLICE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23489,6 +24347,8 @@ int opcode = STORE_SUBSCR; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR); @@ -23548,6 +24408,8 @@ int opcode = STORE_SUBSCR_DICT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23599,6 +24461,8 @@ int opcode = STORE_SUBSCR_LIST_INT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23680,6 +24544,8 @@ int opcode = SWAP; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23702,6 +24568,8 @@ int opcode = TO_BOOL; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL); @@ -23754,6 +24622,8 @@ int opcode = TO_BOOL_ALWAYS_TRUE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23797,6 +24667,8 @@ int opcode = TO_BOOL_BOOL; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23821,6 +24693,8 @@ int opcode = TO_BOOL_INT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23861,6 +24735,8 @@ int opcode = TO_BOOL_LIST; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23904,6 +24780,8 @@ int opcode = TO_BOOL_NONE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23931,6 +24809,8 @@ int opcode = TO_BOOL_STR; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23979,6 +24859,8 @@ int opcode = UNARY_INVERT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -24010,6 +24892,8 @@ int opcode = UNARY_NEGATIVE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -24041,6 +24925,8 @@ int opcode = UNARY_NOT; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -24061,6 +24947,8 @@ int opcode = UNPACK_EX; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -24090,6 +24978,8 @@ int opcode = UNPACK_SEQUENCE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE); @@ -24141,6 +25031,8 @@ int opcode = UNPACK_SEQUENCE_LIST; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -24200,6 +25092,8 @@ int opcode = UNPACK_SEQUENCE_TUPLE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -24250,6 +25144,8 @@ int opcode = UNPACK_SEQUENCE_TWO_TUPLE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -24301,6 +25197,8 @@ int opcode = WITH_EXCEPT_START; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -24348,6 +25246,8 @@ int opcode = YIELD_VALUE; (void)(opcode); #endif + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; diff --git a/Python/optimizer.c b/Python/optimizer.c index 32aa4481c7e504..ae1745a71de48b 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -544,7 +544,7 @@ _PyJIT_translate_single_bytecode_to_trace( #ifdef Py_DEBUG char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); - int lltrace = 3; + int lltrace = 0; if (python_lltrace != NULL && *python_lltrace >= '0') { lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that } @@ -755,7 +755,7 @@ _PyJIT_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_ PyCodeObject *code = _PyFrame_GetCode(frame); #ifdef Py_DEBUG char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); - int lltrace = 3; + int lltrace = 0; if (python_lltrace != NULL && *python_lltrace >= '0') { lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that } diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 349deeacd1b2e6..c133a5e7b9fbd2 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -233,6 +233,8 @@ def generate_tier1_cases( out.emit(f"int opcode = {name};\n") out.emit(f"(void)(opcode);\n") out.emit(f"#endif\n") + out.emit(f"PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable);\n") + out.emit(f"(void)old_code;\n") needs_this = is_tracing or uses_this(inst) unused_guard = "(void)this_instr;\n" if inst.properties.needs_prev: From fba9d2d989f9fcda4824a65d9f8d669059b1a194 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 19 Sep 2025 23:54:27 +0100 Subject: [PATCH 005/190] fix a bug with specializzation --- Include/internal/pycore_opcode_metadata.h | 8 +- Include/internal/pycore_optimizer.h | 2 + Include/internal/pycore_uop_metadata.h | 8 - Python/bytecodes.c | 10 +- Python/ceval.c | 4 +- Python/ceval_macros.h | 3 +- Python/executor_cases.c.h | 26 +- Python/generated_cases.c.h | 737 ++++++++++++++-------- Python/optimizer.c | 56 +- Python/optimizer_cases.c.h | 12 +- Tools/cases_generator/tier1_generator.py | 2 + Tools/cases_generator/tracer_generator.py | 27 + 12 files changed, 581 insertions(+), 314 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 0c3e33df11913d..8f25a9b6336bfd 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1454,10 +1454,10 @@ _PyOpcode_macro_expansion[256] = { [NOT_TAKEN] = { .nuops = 1, .uops = { { _NOP, OPARG_SIMPLE, 0 } } }, [POP_EXCEPT] = { .nuops = 1, .uops = { { _POP_EXCEPT, OPARG_SIMPLE, 0 } } }, [POP_ITER] = { .nuops = 1, .uops = { { _POP_ITER, OPARG_SIMPLE, 0 } } }, - [POP_JUMP_IF_FALSE] = { .nuops = 1, .uops = { { _POP_JUMP_IF_FALSE, OPARG_SIMPLE, 1 } } }, - [POP_JUMP_IF_NONE] = { .nuops = 2, .uops = { { _IS_NONE, OPARG_SIMPLE, 1 }, { _POP_JUMP_IF_TRUE, OPARG_SIMPLE, 1 } } }, - [POP_JUMP_IF_NOT_NONE] = { .nuops = 2, .uops = { { _IS_NONE, OPARG_SIMPLE, 1 }, { _POP_JUMP_IF_FALSE, OPARG_SIMPLE, 1 } } }, - [POP_JUMP_IF_TRUE] = { .nuops = 1, .uops = { { _POP_JUMP_IF_TRUE, OPARG_SIMPLE, 1 } } }, + [POP_JUMP_IF_FALSE] = { .nuops = 1, .uops = { { _POP_JUMP_IF_FALSE, OPARG_REPLACED, 1 } } }, + [POP_JUMP_IF_NONE] = { .nuops = 2, .uops = { { _IS_NONE, OPARG_SIMPLE, 1 }, { _POP_JUMP_IF_TRUE, OPARG_REPLACED, 1 } } }, + [POP_JUMP_IF_NOT_NONE] = { .nuops = 2, .uops = { { _IS_NONE, OPARG_SIMPLE, 1 }, { _POP_JUMP_IF_FALSE, OPARG_REPLACED, 1 } } }, + [POP_JUMP_IF_TRUE] = { .nuops = 1, .uops = { { _POP_JUMP_IF_TRUE, OPARG_REPLACED, 1 } } }, [POP_TOP] = { .nuops = 1, .uops = { { _POP_TOP, OPARG_SIMPLE, 0 } } }, [PUSH_EXC_INFO] = { .nuops = 1, .uops = { { _PUSH_EXC_INFO, OPARG_SIMPLE, 0 } } }, [PUSH_NULL] = { .nuops = 1, .uops = { { _PUSH_NULL, OPARG_SIMPLE, 0 } } }, diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 56224f8a73e4b4..bea2838b8ba092 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -359,10 +359,12 @@ extern void _Py_ClearExecutorDeletionList(PyInterpreterState *interp); int _PyJIT_translate_single_bytecode_to_trace( PyThreadState *tstate, + _PyInterpreterFrame *frame, _Py_CODEUNIT *this_instr, _Py_CODEUNIT *next_instr, PyCodeObject *code, PyFunctionObject *func, + int opcode, int oparg); void diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 973093acf8aaa3..2b33e63ac6cf94 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -205,8 +205,6 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CHECK_EXC_MATCH] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_IMPORT_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_IMPORT_FROM] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_POP_JUMP_IF_FALSE] = HAS_ARG_FLAG | HAS_JUMP_FLAG, - [_POP_JUMP_IF_TRUE] = HAS_ARG_FLAG | HAS_JUMP_FLAG, [_IS_NONE] = HAS_ESCAPES_FLAG, [_JUMP_BACKWARD_NO_INTERRUPT] = HAS_ARG_FLAG | HAS_JUMP_FLAG, [_GET_LEN] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -608,8 +606,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW] = "_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW", [_POP_EXCEPT] = "_POP_EXCEPT", [_POP_ITER] = "_POP_ITER", - [_POP_JUMP_IF_FALSE] = "_POP_JUMP_IF_FALSE", - [_POP_JUMP_IF_TRUE] = "_POP_JUMP_IF_TRUE", [_POP_TOP] = "_POP_TOP", [_POP_TOP_FLOAT] = "_POP_TOP_FLOAT", [_POP_TOP_INT] = "_POP_TOP_INT", @@ -1053,10 +1049,6 @@ int _PyUop_num_popped(int opcode, int oparg) return 2; case _IMPORT_FROM: return 0; - case _POP_JUMP_IF_FALSE: - return 1; - case _POP_JUMP_IF_TRUE: - return 1; case _IS_NONE: return 1; case _JUMP_BACKWARD_NO_INTERRUPT: diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 66553c8c501f3c..e972e4732b48a2 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3050,18 +3050,20 @@ dummy_func( #endif /* _Py_TIER2 */ } - op(_POP_JUMP_IF_FALSE, (cond -- )) { + replaced op(_POP_JUMP_IF_FALSE, (cond -- )) { assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsFalse(cond); DEAD(cond); - JUMPBY(flag ? oparg : 0); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } - op(_POP_JUMP_IF_TRUE, (cond -- )) { + replaced op(_POP_JUMP_IF_TRUE, (cond -- )) { assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsTrue(cond); DEAD(cond); - JUMPBY(flag ? oparg : 0); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } op(_IS_NONE, (value -- b)) { diff --git a/Python/ceval.c b/Python/ceval.c index 3f333d0ae30fee..940f27bd56611d 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -975,12 +975,12 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch) // 1 for trace full, 0 for successful write. static int -add_to_code_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, PyCodeObject *old_code, _Py_CODEUNIT *this_instr, _Py_CODEUNIT *next_instr, int oparg) +add_to_code_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, PyCodeObject *old_code, _Py_CODEUNIT *this_instr, _Py_CODEUNIT *next_instr, int opcode, int oparg) { assert(frame != NULL); assert(tstate->interp->jit_tracer_code_curr_size < UOP_MAX_TRACE_LENGTH); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - return !_PyJIT_translate_single_bytecode_to_trace(tstate, this_instr, next_instr, old_code, func, oparg); + return !_PyJIT_translate_single_bytecode_to_trace(tstate, frame, this_instr, next_instr, old_code, func, opcode, oparg); } /* _PyEval_EvalFrameDefault is too large to optimize for speed with PGO on MSVC. diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index df0f5d25719fda..71e2e0baaeb57b 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -144,7 +144,7 @@ DISPATCH(); # define RECORD_TRACE() do { \ frame->instr_ptr = next_instr; \ - if (add_to_code_trace(tstate, frame, old_code, this_instr, next_instr, oparg)) { \ + if (add_to_code_trace(tstate, frame, old_code, this_instr, next_instr, opcode, oparg)) { \ BAIL_TRACING(); \ } \ } while (0); @@ -407,6 +407,7 @@ do { \ } \ if (keep_tracing_bit) { \ ENTER_TRACING(); \ + _PyJIT_InitializeTracing(tstate, frame, next_instr, STACK_LEVEL(), 0); \ } \ else { \ LEAVE_TRACING(); \ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 145f010db47962..1000d7dedab068 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4171,31 +4171,9 @@ break; } - case _POP_JUMP_IF_FALSE: { - TIER2_JUMPBY(2); - _PyStackRef cond; - oparg = CURRENT_OPARG(); - cond = stack_pointer[-1]; - assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsFalse(cond); - TIER2_JUMPBY(flag ? oparg : 0); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - break; - } + /* _POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 because it is replaced */ - case _POP_JUMP_IF_TRUE: { - TIER2_JUMPBY(2); - _PyStackRef cond; - oparg = CURRENT_OPARG(); - cond = stack_pointer[-1]; - assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsTrue(cond); - TIER2_JUMPBY(flag ? oparg : 0); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - break; - } + /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 because it is replaced */ case _IS_NONE: { _PyStackRef value; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 7d6809b4328c2c..007d97f8b11517 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -10325,6 +10325,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_FALSE); @@ -10333,7 +10335,8 @@ cond = stack_pointer[-1]; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsFalse(cond); - JUMPBY(flag ? oparg : 0); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -10346,6 +10349,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_NONE); @@ -10374,7 +10379,8 @@ cond = b; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsTrue(cond); - JUMPBY(flag ? oparg : 0); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -10388,6 +10394,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); @@ -10416,7 +10424,8 @@ cond = b; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsFalse(cond); - JUMPBY(flag ? oparg : 0); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -10430,6 +10439,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_TRUE); @@ -10438,7 +10449,8 @@ cond = stack_pointer[-1]; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsTrue(cond); - JUMPBY(flag ? oparg : 0); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); DISPATCH(); @@ -12557,6 +12569,7 @@ TRACING_TARGET(BINARY_OP) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP; (void)(opcode); @@ -12622,6 +12635,7 @@ } TRACING_TARGET(BINARY_OP_ADD_FLOAT) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_ADD_FLOAT; (void)(opcode); @@ -12645,7 +12659,7 @@ if (!PyFloat_CheckExact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); } } // _GUARD_NOS_FLOAT @@ -12655,7 +12669,7 @@ if (!PyFloat_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); } } /* Skip 5 cache entries */ @@ -12682,6 +12696,7 @@ } TRACING_TARGET(BINARY_OP_ADD_INT) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_ADD_INT; (void)(opcode); @@ -12705,7 +12720,7 @@ if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); } } // _GUARD_NOS_INT @@ -12715,7 +12730,7 @@ if (!_PyLong_CheckExactAndCompact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); } } /* Skip 5 cache entries */ @@ -12732,7 +12747,7 @@ if (PyStackRef_IsNull(res)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); } PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); @@ -12744,6 +12759,7 @@ } TRACING_TARGET(BINARY_OP_ADD_UNICODE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_ADD_UNICODE; (void)(opcode); @@ -12768,7 +12784,7 @@ if (!PyUnicode_CheckExact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); } } // _GUARD_NOS_UNICODE @@ -12778,7 +12794,7 @@ if (!PyUnicode_CheckExact(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); } } /* Skip 5 cache entries */ @@ -12806,6 +12822,7 @@ } TRACING_TARGET(BINARY_OP_EXTEND) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_EXTEND; (void)(opcode); @@ -12838,7 +12855,7 @@ if (!res) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); } } /* Skip -4 cache entry */ @@ -12872,6 +12889,7 @@ } TRACING_TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_INPLACE_ADD_UNICODE; (void)(opcode); @@ -12895,7 +12913,7 @@ if (!PyUnicode_CheckExact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); } } // _GUARD_NOS_UNICODE @@ -12905,7 +12923,7 @@ if (!PyUnicode_CheckExact(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); } } /* Skip 5 cache entries */ @@ -12928,7 +12946,7 @@ if (PyStackRef_AsPyObjectBorrow(*target_local) != left_o) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); } STAT_INC(BINARY_OP, hit); assert(Py_REFCNT(left_o) >= 2 || !PyStackRef_IsHeapSafe(left)); @@ -12957,6 +12975,7 @@ } TRACING_TARGET(BINARY_OP_MULTIPLY_FLOAT) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_MULTIPLY_FLOAT; (void)(opcode); @@ -12980,7 +12999,7 @@ if (!PyFloat_CheckExact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); } } // _GUARD_NOS_FLOAT @@ -12990,7 +13009,7 @@ if (!PyFloat_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); } } /* Skip 5 cache entries */ @@ -13017,6 +13036,7 @@ } TRACING_TARGET(BINARY_OP_MULTIPLY_INT) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_MULTIPLY_INT; (void)(opcode); @@ -13040,7 +13060,7 @@ if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); } } // _GUARD_NOS_INT @@ -13050,7 +13070,7 @@ if (!_PyLong_CheckExactAndCompact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); } } /* Skip 5 cache entries */ @@ -13067,7 +13087,7 @@ if (PyStackRef_IsNull(res)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); } PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); @@ -13079,6 +13099,7 @@ } TRACING_TARGET(BINARY_OP_SUBSCR_DICT) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_SUBSCR_DICT; (void)(opcode); @@ -13102,7 +13123,7 @@ if (!PyDict_CheckExact(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); } } /* Skip 5 cache entries */ @@ -13147,6 +13168,7 @@ } TRACING_TARGET(BINARY_OP_SUBSCR_GETITEM) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_SUBSCR_GETITEM; (void)(opcode); @@ -13169,7 +13191,7 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); } } // _BINARY_OP_SUBSCR_CHECK_FUNC @@ -13179,28 +13201,28 @@ if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); } PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); if (getitem_o == NULL) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); } assert(PyFunction_Check(getitem_o)); uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); if (((PyFunctionObject *)getitem_o)->func_version != cached_version) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); } PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); assert(code->co_argcount == 2); if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); } getitem = PyStackRef_FromPyObjectNew(getitem_o); STAT_INC(BINARY_OP, hit); @@ -13233,6 +13255,7 @@ } TRACING_TARGET(BINARY_OP_SUBSCR_LIST_INT) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_SUBSCR_LIST_INT; (void)(opcode); @@ -13257,7 +13280,7 @@ if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); } } // _GUARD_NOS_LIST @@ -13267,7 +13290,7 @@ if (!PyList_CheckExact(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); } } /* Skip 5 cache entries */ @@ -13282,7 +13305,7 @@ if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; #ifdef Py_GIL_DISABLED @@ -13292,7 +13315,7 @@ if (res_o == NULL) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); } STAT_INC(BINARY_OP, hit); res = PyStackRef_FromPyObjectSteal(res_o); @@ -13300,7 +13323,7 @@ if (index >= PyList_GET_SIZE(list)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); } STAT_INC(BINARY_OP, hit); PyObject *res_o = PyList_GET_ITEM(list, index); @@ -13325,6 +13348,7 @@ } TRACING_TARGET(BINARY_OP_SUBSCR_LIST_SLICE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_SUBSCR_LIST_SLICE; (void)(opcode); @@ -13349,7 +13373,7 @@ if (!PySlice_Check(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); } } // _GUARD_NOS_LIST @@ -13359,7 +13383,7 @@ if (!PyList_CheckExact(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); } } /* Skip 5 cache entries */ @@ -13399,6 +13423,7 @@ } TRACING_TARGET(BINARY_OP_SUBSCR_STR_INT) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_SUBSCR_STR_INT; (void)(opcode); @@ -13423,7 +13448,7 @@ if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); } } // _GUARD_NOS_UNICODE @@ -13433,7 +13458,7 @@ if (!PyUnicode_CheckExact(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); } } /* Skip 5 cache entries */ @@ -13448,19 +13473,19 @@ if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; if (PyUnicode_GET_LENGTH(str) <= index) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); } Py_UCS4 c = PyUnicode_READ_CHAR(str, index); if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); } STAT_INC(BINARY_OP, hit); PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; @@ -13479,6 +13504,7 @@ } TRACING_TARGET(BINARY_OP_SUBSCR_TUPLE_INT) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_SUBSCR_TUPLE_INT; (void)(opcode); @@ -13503,7 +13529,7 @@ if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); } } // _GUARD_NOS_TUPLE @@ -13513,7 +13539,7 @@ if (!PyTuple_CheckExact(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); } } /* Skip 5 cache entries */ @@ -13528,13 +13554,13 @@ if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; if (index >= PyTuple_GET_SIZE(tuple)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); } STAT_INC(BINARY_OP, hit); PyObject *res_o = PyTuple_GET_ITEM(tuple, index); @@ -13554,6 +13580,7 @@ } TRACING_TARGET(BINARY_OP_SUBTRACT_FLOAT) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_SUBTRACT_FLOAT; (void)(opcode); @@ -13577,7 +13604,7 @@ if (!PyFloat_CheckExact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); } } // _GUARD_NOS_FLOAT @@ -13587,7 +13614,7 @@ if (!PyFloat_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); } } /* Skip 5 cache entries */ @@ -13614,6 +13641,7 @@ } TRACING_TARGET(BINARY_OP_SUBTRACT_INT) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = BINARY_OP_SUBTRACT_INT; (void)(opcode); @@ -13637,7 +13665,7 @@ if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); } } // _GUARD_NOS_INT @@ -13647,7 +13675,7 @@ if (!_PyLong_CheckExactAndCompact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); } } /* Skip 5 cache entries */ @@ -13664,7 +13692,7 @@ if (PyStackRef_IsNull(res)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(BINARY_OP); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); } PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); @@ -13676,6 +13704,7 @@ } TRACING_TARGET(BINARY_SLICE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = BINARY_SLICE; (void)(opcode); @@ -13736,6 +13765,7 @@ } TRACING_TARGET(BUILD_INTERPOLATION) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = BUILD_INTERPOLATION; (void)(opcode); @@ -13798,6 +13828,7 @@ } TRACING_TARGET(BUILD_LIST) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = BUILD_LIST; (void)(opcode); @@ -13826,6 +13857,7 @@ } TRACING_TARGET(BUILD_MAP) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = BUILD_MAP; (void)(opcode); @@ -13882,6 +13914,7 @@ } TRACING_TARGET(BUILD_SET) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = BUILD_SET; (void)(opcode); @@ -13943,6 +13976,7 @@ } TRACING_TARGET(BUILD_SLICE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = BUILD_SLICE; (void)(opcode); @@ -13982,6 +14016,7 @@ } TRACING_TARGET(BUILD_STRING) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = BUILD_STRING; (void)(opcode); @@ -14033,6 +14068,7 @@ } TRACING_TARGET(BUILD_TEMPLATE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = BUILD_TEMPLATE; (void)(opcode); @@ -14075,6 +14111,7 @@ } TRACING_TARGET(BUILD_TUPLE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = BUILD_TUPLE; (void)(opcode); @@ -14101,6 +14138,7 @@ } TRACING_TARGET(CACHE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = CACHE; (void)(opcode); @@ -14118,6 +14156,7 @@ } TRACING_TARGET(CALL) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = CALL; (void)(opcode); @@ -14298,6 +14337,7 @@ } TRACING_TARGET(CALL_ALLOC_AND_ENTER_INIT) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = CALL_ALLOC_AND_ENTER_INIT; (void)(opcode); @@ -14323,7 +14363,7 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } } // _CHECK_AND_ALLOCATE_OBJECT @@ -14335,18 +14375,18 @@ if (!PyStackRef_IsNull(self_or_null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } if (!PyType_Check(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } PyTypeObject *tp = (PyTypeObject *)callable_o; if (FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } assert(tp->tp_new == PyBaseObject_Type.tp_new); assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); @@ -14357,7 +14397,7 @@ if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -14421,6 +14461,7 @@ } TRACING_TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = CALL_BOUND_METHOD_EXACT_ARGS; (void)(opcode); @@ -14444,7 +14485,7 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } } // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS @@ -14454,12 +14495,12 @@ if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } } // _INIT_CALL_BOUND_METHOD_EXACT_ARGS @@ -14485,13 +14526,13 @@ if (!PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } } // _CHECK_FUNCTION_EXACT_ARGS @@ -14503,7 +14544,7 @@ if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } } // _CHECK_STACK_SPACE @@ -14514,7 +14555,7 @@ if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } } // _CHECK_RECURSION_REMAINING @@ -14522,7 +14563,7 @@ if (tstate->py_recursion_remaining <= 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } } // _INIT_CALL_PY_EXACT_ARGS @@ -14566,6 +14607,7 @@ } TRACING_TARGET(CALL_BOUND_METHOD_GENERAL) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = CALL_BOUND_METHOD_GENERAL; (void)(opcode); @@ -14589,7 +14631,7 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } } // _CHECK_METHOD_VERSION @@ -14601,23 +14643,23 @@ if (Py_TYPE(callable_o) != &PyMethod_Type) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } PyObject *func = ((PyMethodObject *)callable_o)->im_func; if (!PyFunction_Check(func)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } if (((PyFunctionObject *)func)->func_version != func_version) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } } // _EXPAND_METHOD @@ -14642,7 +14684,7 @@ if (tstate->py_recursion_remaining <= 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } } // _PY_FRAME_GENERAL @@ -14696,6 +14738,7 @@ } TRACING_TARGET(CALL_BUILTIN_CLASS) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = CALL_BUILTIN_CLASS; (void)(opcode); @@ -14723,7 +14766,7 @@ if (!PyType_Check(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } PyTypeObject *tp = (PyTypeObject *)callable_o; int total_args = oparg; @@ -14735,7 +14778,7 @@ if (tp->tp_vectorcall == NULL) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } STAT_INC(CALL, hit); STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); @@ -14803,6 +14846,7 @@ } TRACING_TARGET(CALL_BUILTIN_FAST) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = CALL_BUILTIN_FAST; (void)(opcode); @@ -14836,12 +14880,12 @@ if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } if (PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); @@ -14914,6 +14958,7 @@ } TRACING_TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = CALL_BUILTIN_FAST_WITH_KEYWORDS; (void)(opcode); @@ -14947,12 +14992,12 @@ if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } if (PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } STAT_INC(CALL, hit); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -15025,6 +15070,7 @@ } TRACING_TARGET(CALL_BUILTIN_O) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = CALL_BUILTIN_O; (void)(opcode); @@ -15057,22 +15103,22 @@ if (total_args != 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } if (_Py_ReachedRecursionLimit(tstate)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } STAT_INC(CALL, hit); PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); @@ -15111,6 +15157,7 @@ } TRACING_TARGET(CALL_FUNCTION_EX) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = CALL_FUNCTION_EX; (void)(opcode); @@ -15280,6 +15327,7 @@ } TRACING_TARGET(CALL_INTRINSIC_1) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = CALL_INTRINSIC_1; (void)(opcode); @@ -15314,6 +15362,7 @@ } TRACING_TARGET(CALL_INTRINSIC_2) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = CALL_INTRINSIC_2; (void)(opcode); @@ -15357,6 +15406,7 @@ } TRACING_TARGET(CALL_ISINSTANCE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = CALL_ISINSTANCE; (void)(opcode); @@ -15382,7 +15432,7 @@ if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } } // _GUARD_CALLABLE_ISINSTANCE @@ -15393,7 +15443,7 @@ if (callable_o != interp->callable_cache.isinstance) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } } // _CALL_ISINSTANCE @@ -15435,6 +15485,7 @@ } TRACING_TARGET(CALL_KW) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = CALL_KW; (void)(opcode); @@ -15619,6 +15670,7 @@ } TRACING_TARGET(CALL_KW_BOUND_METHOD) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = CALL_KW_BOUND_METHOD; (void)(opcode); @@ -15643,7 +15695,7 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(CALL_KW); + JUMP_TO_PREDICTED(TRACING_CALL_KW); } } // _CHECK_METHOD_VERSION_KW @@ -15655,23 +15707,23 @@ if (Py_TYPE(callable_o) != &PyMethod_Type) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(CALL_KW); + JUMP_TO_PREDICTED(TRACING_CALL_KW); } PyObject *func = ((PyMethodObject *)callable_o)->im_func; if (!PyFunction_Check(func)) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(CALL_KW); + JUMP_TO_PREDICTED(TRACING_CALL_KW); } if (((PyFunctionObject *)func)->func_version != func_version) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(CALL_KW); + JUMP_TO_PREDICTED(TRACING_CALL_KW); } if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(CALL_KW); + JUMP_TO_PREDICTED(TRACING_CALL_KW); } } // _EXPAND_METHOD_KW @@ -15751,6 +15803,7 @@ } TRACING_TARGET(CALL_KW_NON_PY) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = CALL_KW_NON_PY; (void)(opcode); @@ -15778,12 +15831,12 @@ if (PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(CALL_KW); + JUMP_TO_PREDICTED(TRACING_CALL_KW); } if (Py_TYPE(callable_o) == &PyMethod_Type) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(CALL_KW); + JUMP_TO_PREDICTED(TRACING_CALL_KW); } } // _CALL_KW_NON_PY @@ -15880,6 +15933,7 @@ } TRACING_TARGET(CALL_KW_PY) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = CALL_KW_PY; (void)(opcode); @@ -15903,7 +15957,7 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(CALL_KW); + JUMP_TO_PREDICTED(TRACING_CALL_KW); } } // _CHECK_FUNCTION_VERSION_KW @@ -15914,13 +15968,13 @@ if (!PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(CALL_KW); + JUMP_TO_PREDICTED(TRACING_CALL_KW); } PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(CALL_KW); + JUMP_TO_PREDICTED(TRACING_CALL_KW); } } // _CHECK_RECURSION_REMAINING @@ -15928,7 +15982,7 @@ if (tstate->py_recursion_remaining <= 1) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(CALL_KW); + JUMP_TO_PREDICTED(TRACING_CALL_KW); } } // _PY_FRAME_KW @@ -15992,6 +16046,7 @@ } TRACING_TARGET(CALL_LEN) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = CALL_LEN; (void)(opcode); @@ -16016,7 +16071,7 @@ if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } } // _GUARD_CALLABLE_LEN @@ -16027,7 +16082,7 @@ if (callable_o != interp->callable_cache.len) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } } // _CALL_LEN @@ -16066,6 +16121,7 @@ } TRACING_TARGET(CALL_LIST_APPEND) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = CALL_LIST_APPEND; (void)(opcode); @@ -16092,7 +16148,7 @@ if (callable_o != interp->callable_cache.list_append) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } } // _GUARD_NOS_NOT_NULL @@ -16102,7 +16158,7 @@ if (o == NULL) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } } // _GUARD_NOS_LIST @@ -16111,7 +16167,7 @@ if (!PyList_CheckExact(o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } } // _CALL_LIST_APPEND @@ -16123,12 +16179,12 @@ if (!PyList_CheckExact(self_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } if (!LOCK_OBJECT(self_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } STAT_INC(CALL, hit); int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); @@ -16156,6 +16212,7 @@ } TRACING_TARGET(CALL_METHOD_DESCRIPTOR_FAST) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = CALL_METHOD_DESCRIPTOR_FAST; (void)(opcode); @@ -16189,26 +16246,26 @@ if (total_args == 0) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } PyMethodDef *meth = method->d_method; if (meth->ml_flags != METH_FASTCALL) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); assert(self != NULL); if (!Py_IS_TYPE(self, method->d_common.d_type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } STAT_INC(CALL, hit); int nargs = total_args - 1; @@ -16279,6 +16336,7 @@ } TRACING_TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS; (void)(opcode); @@ -16312,19 +16370,19 @@ if (total_args == 0) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } PyMethodDef *meth = method->d_method; if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } PyTypeObject *d_type = method->d_common.d_type; PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); @@ -16332,7 +16390,7 @@ if (!Py_IS_TYPE(self, d_type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } STAT_INC(CALL, hit); int nargs = total_args - 1; @@ -16404,6 +16462,7 @@ } TRACING_TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = CALL_METHOD_DESCRIPTOR_NOARGS; (void)(opcode); @@ -16437,13 +16496,13 @@ if (total_args != 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } PyMethodDef *meth = method->d_method; _PyStackRef self_stackref = args[0]; @@ -16451,17 +16510,17 @@ if (!Py_IS_TYPE(self, method->d_common.d_type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } if (meth->ml_flags != METH_NOARGS) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } if (_Py_ReachedRecursionLimit(tstate)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; @@ -16499,6 +16558,7 @@ } TRACING_TARGET(CALL_METHOD_DESCRIPTOR_O) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = CALL_METHOD_DESCRIPTOR_O; (void)(opcode); @@ -16533,23 +16593,23 @@ if (total_args != 2) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } PyMethodDef *meth = method->d_method; if (meth->ml_flags != METH_O) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } if (_Py_ReachedRecursionLimit(tstate)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } _PyStackRef arg_stackref = arguments[1]; _PyStackRef self_stackref = arguments[0]; @@ -16557,7 +16617,7 @@ method->d_common.d_type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } STAT_INC(CALL, hit); PyCFunction cfunc = meth->ml_meth; @@ -16607,6 +16667,7 @@ } TRACING_TARGET(CALL_NON_PY_GENERAL) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = CALL_NON_PY_GENERAL; (void)(opcode); @@ -16633,12 +16694,12 @@ if (PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } if (Py_TYPE(callable_o) == &PyMethod_Type) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } } // _CALL_NON_PY_GENERAL @@ -16724,6 +16785,7 @@ } TRACING_TARGET(CALL_PY_EXACT_ARGS) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = CALL_PY_EXACT_ARGS; (void)(opcode); @@ -16746,7 +16808,7 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } } // _CHECK_FUNCTION_VERSION @@ -16757,13 +16819,13 @@ if (!PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } } // _CHECK_FUNCTION_EXACT_ARGS @@ -16776,7 +16838,7 @@ if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } } // _CHECK_STACK_SPACE @@ -16787,7 +16849,7 @@ if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } } // _CHECK_RECURSION_REMAINING @@ -16795,7 +16857,7 @@ if (tstate->py_recursion_remaining <= 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } } // _INIT_CALL_PY_EXACT_ARGS @@ -16839,6 +16901,7 @@ } TRACING_TARGET(CALL_PY_GENERAL) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = CALL_PY_GENERAL; (void)(opcode); @@ -16861,7 +16924,7 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } } // _CHECK_FUNCTION_VERSION @@ -16872,13 +16935,13 @@ if (!PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } } // _CHECK_RECURSION_REMAINING @@ -16886,7 +16949,7 @@ if (tstate->py_recursion_remaining <= 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } } // _PY_FRAME_GENERAL @@ -16941,6 +17004,7 @@ } TRACING_TARGET(CALL_STR_1) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = CALL_STR_1; (void)(opcode); @@ -16965,7 +17029,7 @@ if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } } // _GUARD_CALLABLE_STR_1 @@ -16975,7 +17039,7 @@ if (callable_o != (PyObject *)&PyUnicode_Type) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } } // _CALL_STR_1 @@ -17015,6 +17079,7 @@ } TRACING_TARGET(CALL_TUPLE_1) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = CALL_TUPLE_1; (void)(opcode); @@ -17039,7 +17104,7 @@ if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } } // _GUARD_CALLABLE_TUPLE_1 @@ -17049,7 +17114,7 @@ if (callable_o != (PyObject *)&PyTuple_Type) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } } // _CALL_TUPLE_1 @@ -17089,6 +17154,7 @@ } TRACING_TARGET(CALL_TYPE_1) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = CALL_TYPE_1; (void)(opcode); @@ -17113,7 +17179,7 @@ if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } } // _GUARD_CALLABLE_TYPE_1 @@ -17123,7 +17189,7 @@ if (callable_o != (PyObject *)&PyType_Type) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(CALL); + JUMP_TO_PREDICTED(TRACING_CALL); } } // _CALL_TYPE_1 @@ -17146,6 +17212,7 @@ } TRACING_TARGET(CHECK_EG_MATCH) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = CHECK_EG_MATCH; (void)(opcode); @@ -17221,6 +17288,7 @@ } TRACING_TARGET(CHECK_EXC_MATCH) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = CHECK_EXC_MATCH; (void)(opcode); @@ -17262,6 +17330,7 @@ } TRACING_TARGET(CLEANUP_THROW) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = CLEANUP_THROW; (void)(opcode); @@ -17323,6 +17392,7 @@ } TRACING_TARGET(COMPARE_OP) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = COMPARE_OP; (void)(opcode); @@ -17398,6 +17468,7 @@ } TRACING_TARGET(COMPARE_OP_FLOAT) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = COMPARE_OP_FLOAT; (void)(opcode); @@ -17421,7 +17492,7 @@ if (!PyFloat_CheckExact(value_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - JUMP_TO_PREDICTED(COMPARE_OP); + JUMP_TO_PREDICTED(TRACING_COMPARE_OP); } } // _GUARD_NOS_FLOAT @@ -17431,7 +17502,7 @@ if (!PyFloat_CheckExact(left_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - JUMP_TO_PREDICTED(COMPARE_OP); + JUMP_TO_PREDICTED(TRACING_COMPARE_OP); } } /* Skip 1 cache entry */ @@ -17455,6 +17526,7 @@ } TRACING_TARGET(COMPARE_OP_INT) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = COMPARE_OP_INT; (void)(opcode); @@ -17478,7 +17550,7 @@ if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - JUMP_TO_PREDICTED(COMPARE_OP); + JUMP_TO_PREDICTED(TRACING_COMPARE_OP); } } // _GUARD_NOS_INT @@ -17488,7 +17560,7 @@ if (!_PyLong_CheckExactAndCompact(left_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - JUMP_TO_PREDICTED(COMPARE_OP); + JUMP_TO_PREDICTED(TRACING_COMPARE_OP); } } /* Skip 1 cache entry */ @@ -17516,6 +17588,7 @@ } TRACING_TARGET(COMPARE_OP_STR) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = COMPARE_OP_STR; (void)(opcode); @@ -17540,7 +17613,7 @@ if (!PyUnicode_CheckExact(value_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - JUMP_TO_PREDICTED(COMPARE_OP); + JUMP_TO_PREDICTED(TRACING_COMPARE_OP); } } // _GUARD_NOS_UNICODE @@ -17550,7 +17623,7 @@ if (!PyUnicode_CheckExact(o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - JUMP_TO_PREDICTED(COMPARE_OP); + JUMP_TO_PREDICTED(TRACING_COMPARE_OP); } } /* Skip 1 cache entry */ @@ -17577,6 +17650,7 @@ } TRACING_TARGET(CONTAINS_OP) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = CONTAINS_OP; (void)(opcode); @@ -17639,6 +17713,7 @@ } TRACING_TARGET(CONTAINS_OP_DICT) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = CONTAINS_OP_DICT; (void)(opcode); @@ -17662,7 +17737,7 @@ if (!PyDict_CheckExact(o)) { UPDATE_MISS_STATS(CONTAINS_OP); assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); - JUMP_TO_PREDICTED(CONTAINS_OP); + JUMP_TO_PREDICTED(TRACING_CONTAINS_OP); } } /* Skip 1 cache entry */ @@ -17699,6 +17774,7 @@ } TRACING_TARGET(CONTAINS_OP_SET) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = CONTAINS_OP_SET; (void)(opcode); @@ -17722,7 +17798,7 @@ if (!PyAnySet_CheckExact(o)) { UPDATE_MISS_STATS(CONTAINS_OP); assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); - JUMP_TO_PREDICTED(CONTAINS_OP); + JUMP_TO_PREDICTED(TRACING_CONTAINS_OP); } } /* Skip 1 cache entry */ @@ -17759,6 +17835,7 @@ } TRACING_TARGET(CONVERT_VALUE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = CONVERT_VALUE; (void)(opcode); @@ -17795,6 +17872,7 @@ } TRACING_TARGET(COPY) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = COPY; (void)(opcode); @@ -17817,6 +17895,7 @@ } TRACING_TARGET(COPY_FREE_VARS) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = COPY_FREE_VARS; (void)(opcode); @@ -17842,6 +17921,7 @@ } TRACING_TARGET(DELETE_ATTR) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = DELETE_ATTR; (void)(opcode); @@ -17871,6 +17951,7 @@ } TRACING_TARGET(DELETE_DEREF) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = DELETE_DEREF; (void)(opcode); @@ -17897,6 +17978,7 @@ } TRACING_TARGET(DELETE_FAST) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = DELETE_FAST; (void)(opcode); @@ -17927,6 +18009,7 @@ } TRACING_TARGET(DELETE_GLOBAL) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = DELETE_GLOBAL; (void)(opcode); @@ -17956,6 +18039,7 @@ } TRACING_TARGET(DELETE_NAME) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = DELETE_NAME; (void)(opcode); @@ -17992,6 +18076,7 @@ } TRACING_TARGET(DELETE_SUBSCR) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = DELETE_SUBSCR; (void)(opcode); @@ -18028,6 +18113,7 @@ } TRACING_TARGET(DICT_MERGE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = DICT_MERGE; (void)(opcode); @@ -18071,6 +18157,7 @@ } TRACING_TARGET(DICT_UPDATE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = DICT_UPDATE; (void)(opcode); @@ -18118,6 +18205,7 @@ } TRACING_TARGET(END_ASYNC_FOR) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = END_ASYNC_FOR; (void)(opcode); @@ -18165,6 +18253,7 @@ } TRACING_TARGET(END_FOR) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = END_FOR; (void)(opcode); @@ -18186,6 +18275,7 @@ } TRACING_TARGET(END_SEND) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = END_SEND; (void)(opcode); @@ -18213,6 +18303,7 @@ } TRACING_TARGET(ENTER_EXECUTOR) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = ENTER_EXECUTOR; (void)(opcode); @@ -18251,6 +18342,7 @@ } TRACING_TARGET(EXIT_INIT_CHECK) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = EXIT_INIT_CHECK; (void)(opcode); @@ -18278,6 +18370,7 @@ } TRACING_TARGET(EXTENDED_ARG) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = EXTENDED_ARG; (void)(opcode); @@ -18298,6 +18391,7 @@ } TRACING_TARGET(FORMAT_SIMPLE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = FORMAT_SIMPLE; (void)(opcode); @@ -18338,6 +18432,7 @@ } TRACING_TARGET(FORMAT_WITH_SPEC) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = FORMAT_WITH_SPEC; (void)(opcode); @@ -18378,6 +18473,7 @@ } TRACING_TARGET(FOR_ITER) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = FOR_ITER; (void)(opcode); @@ -18434,6 +18530,7 @@ } TRACING_TARGET(FOR_ITER_GEN) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = FOR_ITER_GEN; (void)(opcode); @@ -18455,7 +18552,7 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(FOR_ITER); + JUMP_TO_PREDICTED(TRACING_FOR_ITER); } } // _FOR_ITER_GEN_FRAME @@ -18465,19 +18562,19 @@ if (Py_TYPE(gen) != &PyGen_Type) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(FOR_ITER); + JUMP_TO_PREDICTED(TRACING_FOR_ITER); } #ifdef Py_GIL_DISABLED if (!_PyObject_IsUniquelyReferenced((PyObject *)gen)) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(FOR_ITER); + JUMP_TO_PREDICTED(TRACING_FOR_ITER); } #endif if (gen->gi_frame_state >= FRAME_EXECUTING) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(FOR_ITER); + JUMP_TO_PREDICTED(TRACING_FOR_ITER); } STAT_INC(FOR_ITER, hit); _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; @@ -18507,6 +18604,7 @@ } TRACING_TARGET(FOR_ITER_LIST) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = FOR_ITER_LIST; (void)(opcode); @@ -18531,14 +18629,14 @@ if (Py_TYPE(iter_o) != &PyList_Type) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(FOR_ITER); + JUMP_TO_PREDICTED(TRACING_FOR_ITER); } assert(PyStackRef_IsTaggedInt(null_or_index)); #ifdef Py_GIL_DISABLED if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(FOR_ITER); + JUMP_TO_PREDICTED(TRACING_FOR_ITER); } #endif } @@ -18572,7 +18670,7 @@ if (result < 0) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(FOR_ITER); + JUMP_TO_PREDICTED(TRACING_FOR_ITER); } if (result == 0) { null_or_index = PyStackRef_TagInt(-1); @@ -18593,6 +18691,7 @@ } TRACING_TARGET(FOR_ITER_RANGE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = FOR_ITER_RANGE; (void)(opcode); @@ -18615,13 +18714,13 @@ if (Py_TYPE(r) != &PyRangeIter_Type) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(FOR_ITER); + JUMP_TO_PREDICTED(TRACING_FOR_ITER); } #ifdef Py_GIL_DISABLED if (!_PyObject_IsUniquelyReferenced((PyObject *)r)) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(FOR_ITER); + JUMP_TO_PREDICTED(TRACING_FOR_ITER); } #endif } @@ -18662,6 +18761,7 @@ } TRACING_TARGET(FOR_ITER_TUPLE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = FOR_ITER_TUPLE; (void)(opcode); @@ -18686,7 +18786,7 @@ if (Py_TYPE(iter_o) != &PyTuple_Type) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(FOR_ITER); + JUMP_TO_PREDICTED(TRACING_FOR_ITER); } assert(PyStackRef_IsTaggedInt(null_or_index)); } @@ -18720,6 +18820,7 @@ } TRACING_TARGET(GET_AITER) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = GET_AITER; (void)(opcode); @@ -18785,6 +18886,7 @@ } TRACING_TARGET(GET_ANEXT) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = GET_ANEXT; (void)(opcode); @@ -18813,6 +18915,7 @@ } TRACING_TARGET(GET_AWAITABLE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = GET_AWAITABLE; (void)(opcode); @@ -18846,6 +18949,7 @@ } TRACING_TARGET(GET_ITER) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = GET_ITER; (void)(opcode); @@ -18896,6 +19000,7 @@ } TRACING_TARGET(GET_LEN) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = GET_LEN; (void)(opcode); @@ -18928,6 +19033,7 @@ } TRACING_TARGET(GET_YIELD_FROM_ITER) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = GET_YIELD_FROM_ITER; (void)(opcode); @@ -18977,6 +19083,7 @@ } TRACING_TARGET(IMPORT_FROM) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = IMPORT_FROM; (void)(opcode); @@ -19006,6 +19113,7 @@ } TRACING_TARGET(IMPORT_NAME) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = IMPORT_NAME; (void)(opcode); @@ -19049,6 +19157,7 @@ } TRACING_TARGET(INSTRUMENTED_CALL) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_CALL; (void)(opcode); @@ -19239,6 +19348,7 @@ } TRACING_TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_CALL_FUNCTION_EX; (void)(opcode); @@ -19408,6 +19518,7 @@ } TRACING_TARGET(INSTRUMENTED_CALL_KW) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_CALL_KW; (void)(opcode); @@ -19596,6 +19707,7 @@ } TRACING_TARGET(INSTRUMENTED_END_ASYNC_FOR) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_END_ASYNC_FOR; (void)(opcode); @@ -19651,6 +19763,7 @@ } TRACING_TARGET(INSTRUMENTED_END_FOR) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_END_FOR; (void)(opcode); @@ -19682,6 +19795,7 @@ } TRACING_TARGET(INSTRUMENTED_END_SEND) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_END_SEND; (void)(opcode); @@ -19718,6 +19832,7 @@ } TRACING_TARGET(INSTRUMENTED_FOR_ITER) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_FOR_ITER; (void)(opcode); @@ -19756,6 +19871,7 @@ } TRACING_TARGET(INSTRUMENTED_INSTRUCTION) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_INSTRUCTION; (void)(opcode); @@ -19785,6 +19901,7 @@ } TRACING_TARGET(INSTRUMENTED_JUMP_BACKWARD) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_JUMP_BACKWARD; (void)(opcode); @@ -19814,6 +19931,7 @@ } TRACING_TARGET(INSTRUMENTED_JUMP_FORWARD) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_JUMP_FORWARD; (void)(opcode); @@ -19830,6 +19948,7 @@ } TRACING_TARGET(INSTRUMENTED_LINE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_LINE; (void)(opcode); @@ -19872,6 +19991,7 @@ } TRACING_TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_LOAD_SUPER_ATTR; (void)(opcode); @@ -19994,6 +20114,7 @@ } TRACING_TARGET(INSTRUMENTED_NOT_TAKEN) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_NOT_TAKEN; (void)(opcode); @@ -20012,6 +20133,7 @@ } TRACING_TARGET(INSTRUMENTED_POP_ITER) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_POP_ITER; (void)(opcode); @@ -20039,6 +20161,7 @@ } TRACING_TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_POP_JUMP_IF_FALSE; (void)(opcode); @@ -20065,6 +20188,7 @@ } TRACING_TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_POP_JUMP_IF_NONE; (void)(opcode); @@ -20098,6 +20222,7 @@ } TRACING_TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_POP_JUMP_IF_NOT_NONE; (void)(opcode); @@ -20129,6 +20254,7 @@ } TRACING_TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_POP_JUMP_IF_TRUE; (void)(opcode); @@ -20155,6 +20281,7 @@ } TRACING_TARGET(INSTRUMENTED_RESUME) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_RESUME; (void)(opcode); @@ -20237,6 +20364,7 @@ } TRACING_TARGET(INSTRUMENTED_RETURN_VALUE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_RETURN_VALUE; (void)(opcode); @@ -20291,6 +20419,7 @@ } TRACING_TARGET(INSTRUMENTED_YIELD_VALUE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = INSTRUMENTED_YIELD_VALUE; (void)(opcode); @@ -20364,6 +20493,7 @@ } TRACING_TARGET(INTERPRETER_EXIT) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = INTERPRETER_EXIT; (void)(opcode); @@ -20403,6 +20533,7 @@ } TRACING_TARGET(IS_OP) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = IS_OP; (void)(opcode); @@ -20440,6 +20571,7 @@ } TRACING_TARGET(JUMP_BACKWARD) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = JUMP_BACKWARD; (void)(opcode); @@ -20483,6 +20615,7 @@ } TRACING_TARGET(JUMP_BACKWARD_JIT) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = JUMP_BACKWARD_JIT; (void)(opcode); @@ -20546,6 +20679,7 @@ } TRACING_TARGET(JUMP_BACKWARD_NO_INTERRUPT) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = JUMP_BACKWARD_NO_INTERRUPT; (void)(opcode); @@ -20565,6 +20699,7 @@ } TRACING_TARGET(JUMP_BACKWARD_NO_JIT) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = JUMP_BACKWARD_NO_JIT; (void)(opcode); @@ -20598,6 +20733,7 @@ } TRACING_TARGET(JUMP_FORWARD) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = JUMP_FORWARD; (void)(opcode); @@ -20614,6 +20750,7 @@ } TRACING_TARGET(LIST_APPEND) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = LIST_APPEND; (void)(opcode); @@ -20640,6 +20777,7 @@ } TRACING_TARGET(LIST_EXTEND) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = LIST_EXTEND; (void)(opcode); @@ -20691,6 +20829,7 @@ } TRACING_TARGET(LOAD_ATTR) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = LOAD_ATTR; (void)(opcode); @@ -20774,6 +20913,7 @@ } TRACING_TARGET(LOAD_ATTR_CLASS) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = LOAD_ATTR_CLASS; (void)(opcode); @@ -20798,13 +20938,13 @@ if (!PyType_Check(owner_o)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } } /* Skip 2 cache entries */ @@ -20834,6 +20974,7 @@ } TRACING_TARGET(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = LOAD_ATTR_CLASS_WITH_METACLASS_CHECK; (void)(opcode); @@ -20858,13 +20999,13 @@ if (!PyType_Check(owner_o)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } } // _GUARD_TYPE_VERSION @@ -20875,7 +21016,7 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } } // _LOAD_ATTR_CLASS @@ -20904,6 +21045,7 @@ } TRACING_TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN; (void)(opcode); @@ -20927,14 +21069,14 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } PyTypeObject *cls = Py_TYPE(owner_o); assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); PyFunctionObject *f = (PyFunctionObject *)getattribute; @@ -20942,14 +21084,14 @@ if (f->func_version != func_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } PyCodeObject *code = (PyCodeObject *)f->func_code; assert(code->co_argcount == 2); if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } STAT_INC(LOAD_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); @@ -20964,6 +21106,7 @@ } TRACING_TARGET(LOAD_ATTR_INSTANCE_VALUE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = LOAD_ATTR_INSTANCE_VALUE; (void)(opcode); @@ -20989,7 +21132,7 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } } // _CHECK_MANAGED_OBJECT_HAS_VALUES @@ -21000,7 +21143,7 @@ if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } } // _LOAD_ATTR_INSTANCE_VALUE @@ -21012,7 +21155,7 @@ if (attr_o == NULL) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr); @@ -21020,7 +21163,7 @@ if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } } #else @@ -21046,6 +21189,7 @@ } TRACING_TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = LOAD_ATTR_METHOD_LAZY_DICT; (void)(opcode); @@ -21071,7 +21215,7 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } } // _CHECK_ATTR_METHOD_LAZY_DICT @@ -21082,7 +21226,7 @@ if (dict != NULL) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } } /* Skip 1 cache entry */ @@ -21104,6 +21248,7 @@ } TRACING_TARGET(LOAD_ATTR_METHOD_NO_DICT) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = LOAD_ATTR_METHOD_NO_DICT; (void)(opcode); @@ -21129,7 +21274,7 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } } /* Skip 2 cache entries */ @@ -21152,6 +21297,7 @@ } TRACING_TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = LOAD_ATTR_METHOD_WITH_VALUES; (void)(opcode); @@ -21177,7 +21323,7 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT @@ -21188,7 +21334,7 @@ if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } } // _GUARD_KEYS_VERSION @@ -21200,7 +21346,7 @@ if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } } // _LOAD_ATTR_METHOD_WITH_VALUES @@ -21221,6 +21367,7 @@ } TRACING_TARGET(LOAD_ATTR_MODULE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = LOAD_ATTR_MODULE; (void)(opcode); @@ -21246,7 +21393,7 @@ if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; assert(dict != NULL); @@ -21254,7 +21401,7 @@ if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } assert(keys->dk_kind == DICT_KEYS_UNICODE); assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(keys->dk_nentries)); @@ -21263,7 +21410,7 @@ if (attr_o == NULL) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); @@ -21271,7 +21418,7 @@ if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } } #else @@ -21297,6 +21444,7 @@ } TRACING_TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = LOAD_ATTR_NONDESCRIPTOR_NO_DICT; (void)(opcode); @@ -21321,7 +21469,7 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } } /* Skip 2 cache entries */ @@ -21346,6 +21494,7 @@ } TRACING_TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES; (void)(opcode); @@ -21370,7 +21519,7 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } } // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT @@ -21381,7 +21530,7 @@ if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } } // _GUARD_KEYS_VERSION @@ -21393,7 +21542,7 @@ if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } } // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES @@ -21416,6 +21565,7 @@ } TRACING_TARGET(LOAD_ATTR_PROPERTY) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = LOAD_ATTR_PROPERTY; (void)(opcode); @@ -21436,7 +21586,7 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } } // _GUARD_TYPE_VERSION @@ -21448,7 +21598,7 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } } /* Skip 2 cache entries */ @@ -21462,22 +21612,22 @@ if ((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } if (code->co_kwonlyargcount) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } if (code->co_argcount != 1) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } STAT_INC(LOAD_ATTR, hit); _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); @@ -21512,6 +21662,7 @@ } TRACING_TARGET(LOAD_ATTR_SLOT) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = LOAD_ATTR_SLOT; (void)(opcode); @@ -21537,7 +21688,7 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } } // _LOAD_ATTR_SLOT @@ -21549,14 +21700,14 @@ if (attr_o == NULL) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } #ifdef Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); if (!increfed) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } #else attr = PyStackRef_FromPyObjectNew(attr_o); @@ -21583,6 +21734,7 @@ } TRACING_TARGET(LOAD_ATTR_WITH_HINT) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = LOAD_ATTR_WITH_HINT; (void)(opcode); @@ -21608,7 +21760,7 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } } // _LOAD_ATTR_WITH_HINT @@ -21620,7 +21772,7 @@ if (dict == NULL) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } PyDictKeysObject *dk = FT_ATOMIC_LOAD_PTR(dict->ma_keys); assert(PyDict_CheckExact((PyObject *)dict)); @@ -21628,7 +21780,7 @@ if (!_Py_IsOwnedByCurrentThread((PyObject *)dict) && !_PyObject_GC_IS_SHARED(dict)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } #endif PyObject *attr_o; @@ -21636,7 +21788,7 @@ if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); @@ -21644,7 +21796,7 @@ if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dk) + hint; @@ -21652,7 +21804,7 @@ if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } } attr_o = FT_ATOMIC_LOAD_PTR(ep->me_value); @@ -21660,7 +21812,7 @@ if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } } STAT_INC(LOAD_ATTR, hit); @@ -21670,7 +21822,7 @@ if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(LOAD_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); } } #else @@ -21695,6 +21847,7 @@ } TRACING_TARGET(LOAD_BUILD_CLASS) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = LOAD_BUILD_CLASS; (void)(opcode); @@ -21729,6 +21882,7 @@ } TRACING_TARGET(LOAD_COMMON_CONSTANT) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = LOAD_COMMON_CONSTANT; (void)(opcode); @@ -21750,6 +21904,7 @@ } TRACING_TARGET(LOAD_CONST) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = LOAD_CONST; (void)(opcode); @@ -21771,6 +21926,7 @@ } TRACING_TARGET(LOAD_DEREF) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = LOAD_DEREF; (void)(opcode); @@ -21803,6 +21959,7 @@ } TRACING_TARGET(LOAD_FAST) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = LOAD_FAST; (void)(opcode); @@ -21824,6 +21981,7 @@ } TRACING_TARGET(LOAD_FAST_AND_CLEAR) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = LOAD_FAST_AND_CLEAR; (void)(opcode); @@ -21845,6 +22003,7 @@ } TRACING_TARGET(LOAD_FAST_BORROW) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = LOAD_FAST_BORROW; (void)(opcode); @@ -21866,6 +22025,7 @@ } TRACING_TARGET(LOAD_FAST_BORROW_LOAD_FAST_BORROW) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = LOAD_FAST_BORROW_LOAD_FAST_BORROW; (void)(opcode); @@ -21891,6 +22051,7 @@ } TRACING_TARGET(LOAD_FAST_CHECK) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = LOAD_FAST_CHECK; (void)(opcode); @@ -21921,6 +22082,7 @@ } TRACING_TARGET(LOAD_FAST_LOAD_FAST) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = LOAD_FAST_LOAD_FAST; (void)(opcode); @@ -21946,6 +22108,7 @@ } TRACING_TARGET(LOAD_FROM_DICT_OR_DEREF) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = LOAD_FROM_DICT_OR_DEREF; (void)(opcode); @@ -21995,6 +22158,7 @@ } TRACING_TARGET(LOAD_FROM_DICT_OR_GLOBALS) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = LOAD_FROM_DICT_OR_GLOBALS; (void)(opcode); @@ -22074,6 +22238,7 @@ } TRACING_TARGET(LOAD_GLOBAL) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = LOAD_GLOBAL; (void)(opcode); @@ -22132,6 +22297,7 @@ } TRACING_TARGET(LOAD_GLOBAL_BUILTIN) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = LOAD_GLOBAL_BUILTIN; (void)(opcode); @@ -22154,13 +22320,13 @@ if (!PyDict_CheckExact(dict)) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(LOAD_GLOBAL); + JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); } PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(LOAD_GLOBAL); + JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); } assert(DK_IS_UNICODE(keys)); } @@ -22172,13 +22338,13 @@ if (!PyDict_CheckExact(dict)) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(LOAD_GLOBAL); + JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); } PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(LOAD_GLOBAL); + JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); } assert(DK_IS_UNICODE(keys)); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); @@ -22186,14 +22352,14 @@ if (res_o == NULL) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(LOAD_GLOBAL); + JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); } #if Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); if (!increfed) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(LOAD_GLOBAL); + JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); } #else res = PyStackRef_FromPyObjectNew(res_o); @@ -22214,6 +22380,7 @@ } TRACING_TARGET(LOAD_GLOBAL_MODULE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = LOAD_GLOBAL_MODULE; (void)(opcode); @@ -22240,13 +22407,13 @@ if (!PyDict_CheckExact(dict)) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(LOAD_GLOBAL); + JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); } PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(LOAD_GLOBAL); + JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); } assert(DK_IS_UNICODE(keys)); PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); @@ -22255,14 +22422,14 @@ if (res_o == NULL) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(LOAD_GLOBAL); + JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); } #if Py_GIL_DISABLED int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); if (!increfed) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(LOAD_GLOBAL); + JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); } #else res = PyStackRef_FromPyObjectNew(res_o); @@ -22283,6 +22450,7 @@ } TRACING_TARGET(LOAD_LOCALS) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = LOAD_LOCALS; (void)(opcode); @@ -22311,6 +22479,7 @@ } TRACING_TARGET(LOAD_NAME) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = LOAD_NAME; (void)(opcode); @@ -22338,6 +22507,7 @@ } TRACING_TARGET(LOAD_SMALL_INT) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = LOAD_SMALL_INT; (void)(opcode); @@ -22360,6 +22530,7 @@ } TRACING_TARGET(LOAD_SPECIAL) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = LOAD_SPECIAL; (void)(opcode); @@ -22410,6 +22581,7 @@ } TRACING_TARGET(LOAD_SUPER_ATTR) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = LOAD_SUPER_ATTR; (void)(opcode); @@ -22549,6 +22721,7 @@ } TRACING_TARGET(LOAD_SUPER_ATTR_ATTR) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = LOAD_SUPER_ATTR_ATTR; (void)(opcode); @@ -22576,12 +22749,12 @@ if (global_super != (PyObject *)&PySuper_Type) { UPDATE_MISS_STATS(LOAD_SUPER_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); - JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_SUPER_ATTR); } if (!PyType_Check(class)) { UPDATE_MISS_STATS(LOAD_SUPER_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); - JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_SUPER_ATTR); } STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); @@ -22613,6 +22786,7 @@ } TRACING_TARGET(LOAD_SUPER_ATTR_METHOD) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = LOAD_SUPER_ATTR_METHOD; (void)(opcode); @@ -22641,12 +22815,12 @@ if (global_super != (PyObject *)&PySuper_Type) { UPDATE_MISS_STATS(LOAD_SUPER_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); - JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_SUPER_ATTR); } if (!PyType_Check(class)) { UPDATE_MISS_STATS(LOAD_SUPER_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); - JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); + JUMP_TO_PREDICTED(TRACING_LOAD_SUPER_ATTR); } STAT_INC(LOAD_SUPER_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); @@ -22693,6 +22867,7 @@ } TRACING_TARGET(MAKE_CELL) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = MAKE_CELL; (void)(opcode); @@ -22718,6 +22893,7 @@ } TRACING_TARGET(MAKE_FUNCTION) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = MAKE_FUNCTION; (void)(opcode); @@ -22755,6 +22931,7 @@ } TRACING_TARGET(MAP_ADD) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = MAP_ADD; (void)(opcode); @@ -22790,6 +22967,7 @@ } TRACING_TARGET(MATCH_CLASS) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = MATCH_CLASS; (void)(opcode); @@ -22846,6 +23024,7 @@ } TRACING_TARGET(MATCH_KEYS) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = MATCH_KEYS; (void)(opcode); @@ -22877,6 +23056,7 @@ } TRACING_TARGET(MATCH_MAPPING) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = MATCH_MAPPING; (void)(opcode); @@ -22900,6 +23080,7 @@ } TRACING_TARGET(MATCH_SEQUENCE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = MATCH_SEQUENCE; (void)(opcode); @@ -22923,6 +23104,7 @@ } TRACING_TARGET(NOP) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = NOP; (void)(opcode); @@ -22938,6 +23120,7 @@ } TRACING_TARGET(NOT_TAKEN) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = NOT_TAKEN; (void)(opcode); @@ -22953,6 +23136,7 @@ } TRACING_TARGET(POP_EXCEPT) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = POP_EXCEPT; (void)(opcode); @@ -22978,6 +23162,7 @@ } TRACING_TARGET(POP_ITER) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = POP_ITER; (void)(opcode); @@ -23003,6 +23188,7 @@ } TRACING_TARGET(POP_JUMP_IF_FALSE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = POP_JUMP_IF_FALSE; (void)(opcode); @@ -23019,13 +23205,15 @@ cond = stack_pointer[-1]; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsFalse(cond); - JUMPBY(flag ? oparg : 0); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); TRACING_DISPATCH(); } TRACING_TARGET(POP_JUMP_IF_NONE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = POP_JUMP_IF_NONE; (void)(opcode); @@ -23062,7 +23250,8 @@ cond = b; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsTrue(cond); - JUMPBY(flag ? oparg : 0); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -23070,6 +23259,7 @@ } TRACING_TARGET(POP_JUMP_IF_NOT_NONE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = POP_JUMP_IF_NOT_NONE; (void)(opcode); @@ -23106,7 +23296,8 @@ cond = b; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsFalse(cond); - JUMPBY(flag ? oparg : 0); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -23114,6 +23305,7 @@ } TRACING_TARGET(POP_JUMP_IF_TRUE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = POP_JUMP_IF_TRUE; (void)(opcode); @@ -23130,13 +23322,15 @@ cond = stack_pointer[-1]; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsTrue(cond); - JUMPBY(flag ? oparg : 0); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); TRACING_DISPATCH(); } TRACING_TARGET(POP_TOP) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = POP_TOP; (void)(opcode); @@ -23159,6 +23353,7 @@ } TRACING_TARGET(PUSH_EXC_INFO) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = PUSH_EXC_INFO; (void)(opcode); @@ -23192,6 +23387,7 @@ } TRACING_TARGET(PUSH_NULL) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = PUSH_NULL; (void)(opcode); @@ -23212,6 +23408,7 @@ } TRACING_TARGET(RAISE_VARARGS) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = RAISE_VARARGS; (void)(opcode); @@ -23243,6 +23440,7 @@ } TRACING_TARGET(RERAISE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = RERAISE; (void)(opcode); @@ -23273,6 +23471,7 @@ } TRACING_TARGET(RESERVED) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = RESERVED; (void)(opcode); @@ -23290,6 +23489,7 @@ } TRACING_TARGET(RESUME) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = RESUME; (void)(opcode); @@ -23368,6 +23568,7 @@ } TRACING_TARGET(RESUME_CHECK) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = RESUME_CHECK; (void)(opcode); @@ -23384,7 +23585,7 @@ if (_Py_emscripten_signal_clock == 0) { UPDATE_MISS_STATS(RESUME); assert(_PyOpcode_Deopt[opcode] == (RESUME)); - JUMP_TO_PREDICTED(RESUME); + JUMP_TO_PREDICTED(TRACING_RESUME); } _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif @@ -23394,20 +23595,21 @@ if (eval_breaker != version) { UPDATE_MISS_STATS(RESUME); assert(_PyOpcode_Deopt[opcode] == (RESUME)); - JUMP_TO_PREDICTED(RESUME); + JUMP_TO_PREDICTED(TRACING_RESUME); } #ifdef Py_GIL_DISABLED if (frame->tlbc_index != ((_PyThreadStateImpl *)tstate)->tlbc_index) { UPDATE_MISS_STATS(RESUME); assert(_PyOpcode_Deopt[opcode] == (RESUME)); - JUMP_TO_PREDICTED(RESUME); + JUMP_TO_PREDICTED(TRACING_RESUME); } #endif TRACING_DISPATCH(); } TRACING_TARGET(RETURN_GENERATOR) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = RETURN_GENERATOR; (void)(opcode); @@ -23454,6 +23656,7 @@ } TRACING_TARGET(RETURN_VALUE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = RETURN_VALUE; (void)(opcode); @@ -23492,6 +23695,7 @@ } TRACING_TARGET(SEND) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = SEND; (void)(opcode); @@ -23599,6 +23803,7 @@ } TRACING_TARGET(SEND_GEN) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = SEND_GEN; (void)(opcode); @@ -23621,7 +23826,7 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(SEND); assert(_PyOpcode_Deopt[opcode] == (SEND)); - JUMP_TO_PREDICTED(SEND); + JUMP_TO_PREDICTED(TRACING_SEND); } } // _SEND_GEN_FRAME @@ -23632,12 +23837,12 @@ if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { UPDATE_MISS_STATS(SEND); assert(_PyOpcode_Deopt[opcode] == (SEND)); - JUMP_TO_PREDICTED(SEND); + JUMP_TO_PREDICTED(TRACING_SEND); } if (gen->gi_frame_state >= FRAME_EXECUTING) { UPDATE_MISS_STATS(SEND); assert(_PyOpcode_Deopt[opcode] == (SEND)); - JUMP_TO_PREDICTED(SEND); + JUMP_TO_PREDICTED(TRACING_SEND); } STAT_INC(SEND, hit); _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; @@ -23670,6 +23875,7 @@ } TRACING_TARGET(SETUP_ANNOTATIONS) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = SETUP_ANNOTATIONS; (void)(opcode); @@ -23720,6 +23926,7 @@ } TRACING_TARGET(SET_ADD) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = SET_ADD; (void)(opcode); @@ -23748,6 +23955,7 @@ } TRACING_TARGET(SET_FUNCTION_ATTRIBUTE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = SET_FUNCTION_ATTRIBUTE; (void)(opcode); @@ -23780,6 +23988,7 @@ } TRACING_TARGET(SET_UPDATE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = SET_UPDATE; (void)(opcode); @@ -23811,6 +24020,7 @@ } TRACING_TARGET(STORE_ATTR) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = STORE_ATTR; (void)(opcode); @@ -23870,6 +24080,7 @@ } TRACING_TARGET(STORE_ATTR_INSTANCE_VALUE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = STORE_ATTR_INSTANCE_VALUE; (void)(opcode); @@ -23894,7 +24105,7 @@ if (!LOCK_OBJECT(owner_o)) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(STORE_ATTR); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); } PyTypeObject *tp = Py_TYPE(owner_o); if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { @@ -23902,7 +24113,7 @@ if (true) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(STORE_ATTR); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); } } } @@ -23917,7 +24128,7 @@ if (true) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(STORE_ATTR); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); } } } @@ -23948,6 +24159,7 @@ } TRACING_TARGET(STORE_ATTR_SLOT) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = STORE_ATTR_SLOT; (void)(opcode); @@ -23972,7 +24184,7 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(STORE_ATTR); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); } } // _STORE_ATTR_SLOT @@ -23983,7 +24195,7 @@ if (!LOCK_OBJECT(owner_o)) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(STORE_ATTR); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); } char *addr = (char *)owner_o + index; STAT_INC(STORE_ATTR, hit); @@ -24001,6 +24213,7 @@ } TRACING_TARGET(STORE_ATTR_WITH_HINT) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = STORE_ATTR_WITH_HINT; (void)(opcode); @@ -24025,7 +24238,7 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(STORE_ATTR); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); } } // _STORE_ATTR_WITH_HINT @@ -24038,12 +24251,12 @@ if (dict == NULL) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(STORE_ATTR); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); } if (!LOCK_OBJECT(dict)) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(STORE_ATTR); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); } assert(PyDict_CheckExact((PyObject *)dict)); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -24053,7 +24266,7 @@ if (true) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(STORE_ATTR); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); } } PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; @@ -24062,7 +24275,7 @@ if (true) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(STORE_ATTR); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); } } PyObject *old_value = ep->me_value; @@ -24071,7 +24284,7 @@ if (true) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(STORE_ATTR); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); } } _PyFrame_SetStackPointer(frame, stack_pointer); @@ -24091,6 +24304,7 @@ } TRACING_TARGET(STORE_DEREF) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = STORE_DEREF; (void)(opcode); @@ -24114,6 +24328,7 @@ } TRACING_TARGET(STORE_FAST) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = STORE_FAST; (void)(opcode); @@ -24138,6 +24353,7 @@ } TRACING_TARGET(STORE_FAST_LOAD_FAST) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = STORE_FAST_LOAD_FAST; (void)(opcode); @@ -24165,6 +24381,7 @@ } TRACING_TARGET(STORE_FAST_STORE_FAST) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = STORE_FAST_STORE_FAST; (void)(opcode); @@ -24200,6 +24417,7 @@ } TRACING_TARGET(STORE_GLOBAL) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = STORE_GLOBAL; (void)(opcode); @@ -24229,6 +24447,7 @@ } TRACING_TARGET(STORE_NAME) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = STORE_NAME; (void)(opcode); @@ -24279,6 +24498,7 @@ } TRACING_TARGET(STORE_SLICE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = STORE_SLICE; (void)(opcode); @@ -24343,6 +24563,7 @@ } TRACING_TARGET(STORE_SUBSCR) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = STORE_SUBSCR; (void)(opcode); @@ -24404,6 +24625,7 @@ } TRACING_TARGET(STORE_SUBSCR_DICT) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = STORE_SUBSCR_DICT; (void)(opcode); @@ -24427,7 +24649,7 @@ if (!PyDict_CheckExact(o)) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - JUMP_TO_PREDICTED(STORE_SUBSCR); + JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); } } /* Skip 1 cache entry */ @@ -24457,6 +24679,7 @@ } TRACING_TARGET(STORE_SUBSCR_LIST_INT) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = STORE_SUBSCR_LIST_INT; (void)(opcode); @@ -24480,7 +24703,7 @@ if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - JUMP_TO_PREDICTED(STORE_SUBSCR); + JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); } } // _GUARD_NOS_LIST @@ -24490,7 +24713,7 @@ if (!PyList_CheckExact(o)) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - JUMP_TO_PREDICTED(STORE_SUBSCR); + JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); } } /* Skip 1 cache entry */ @@ -24506,20 +24729,20 @@ if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - JUMP_TO_PREDICTED(STORE_SUBSCR); + JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; if (!LOCK_OBJECT(list)) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - JUMP_TO_PREDICTED(STORE_SUBSCR); + JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); } if (index >= PyList_GET_SIZE(list)) { UNLOCK_OBJECT(list); if (true) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - JUMP_TO_PREDICTED(STORE_SUBSCR); + JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); } } STAT_INC(STORE_SUBSCR, hit); @@ -24540,6 +24763,7 @@ } TRACING_TARGET(SWAP) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = SWAP; (void)(opcode); @@ -24564,6 +24788,7 @@ } TRACING_TARGET(TO_BOOL) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = TO_BOOL; (void)(opcode); @@ -24618,6 +24843,7 @@ } TRACING_TARGET(TO_BOOL_ALWAYS_TRUE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = TO_BOOL_ALWAYS_TRUE; (void)(opcode); @@ -24643,7 +24869,7 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - JUMP_TO_PREDICTED(TO_BOOL); + JUMP_TO_PREDICTED(TRACING_TO_BOOL); } } // _REPLACE_WITH_TRUE @@ -24663,6 +24889,7 @@ } TRACING_TARGET(TO_BOOL_BOOL) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = TO_BOOL_BOOL; (void)(opcode); @@ -24682,13 +24909,14 @@ if (!PyStackRef_BoolCheck(value)) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - JUMP_TO_PREDICTED(TO_BOOL); + JUMP_TO_PREDICTED(TRACING_TO_BOOL); } STAT_INC(TO_BOOL, hit); TRACING_DISPATCH(); } TRACING_TARGET(TO_BOOL_INT) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = TO_BOOL_INT; (void)(opcode); @@ -24710,7 +24938,7 @@ if (!PyLong_CheckExact(value_o)) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - JUMP_TO_PREDICTED(TO_BOOL); + JUMP_TO_PREDICTED(TRACING_TO_BOOL); } STAT_INC(TO_BOOL, hit); if (_PyLong_IsZero((PyLongObject *)value_o)) { @@ -24731,6 +24959,7 @@ } TRACING_TARGET(TO_BOOL_LIST) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = TO_BOOL_LIST; (void)(opcode); @@ -24753,7 +24982,7 @@ if (!PyList_CheckExact(o)) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - JUMP_TO_PREDICTED(TO_BOOL); + JUMP_TO_PREDICTED(TRACING_TO_BOOL); } } /* Skip 1 cache entry */ @@ -24776,6 +25005,7 @@ } TRACING_TARGET(TO_BOOL_NONE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = TO_BOOL_NONE; (void)(opcode); @@ -24796,7 +25026,7 @@ if (!PyStackRef_IsNone(value)) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - JUMP_TO_PREDICTED(TO_BOOL); + JUMP_TO_PREDICTED(TRACING_TO_BOOL); } STAT_INC(TO_BOOL, hit); res = PyStackRef_False; @@ -24805,6 +25035,7 @@ } TRACING_TARGET(TO_BOOL_STR) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = TO_BOOL_STR; (void)(opcode); @@ -24826,7 +25057,7 @@ if (!PyUnicode_CheckExact(value_o)) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - JUMP_TO_PREDICTED(TO_BOOL); + JUMP_TO_PREDICTED(TRACING_TO_BOOL); } } /* Skip 1 cache entry */ @@ -24855,6 +25086,7 @@ } TRACING_TARGET(UNARY_INVERT) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = UNARY_INVERT; (void)(opcode); @@ -24888,6 +25120,7 @@ } TRACING_TARGET(UNARY_NEGATIVE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = UNARY_NEGATIVE; (void)(opcode); @@ -24921,6 +25154,7 @@ } TRACING_TARGET(UNARY_NOT) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = UNARY_NOT; (void)(opcode); @@ -24943,6 +25177,7 @@ } TRACING_TARGET(UNPACK_EX) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = UNPACK_EX; (void)(opcode); @@ -24974,6 +25209,7 @@ } TRACING_TARGET(UNPACK_SEQUENCE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = UNPACK_SEQUENCE; (void)(opcode); @@ -25027,6 +25263,7 @@ } TRACING_TARGET(UNPACK_SEQUENCE_LIST) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = UNPACK_SEQUENCE_LIST; (void)(opcode); @@ -25049,7 +25286,7 @@ if (!PyList_CheckExact(o)) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); } } /* Skip 1 cache entry */ @@ -25062,14 +25299,14 @@ if (!LOCK_OBJECT(seq_o)) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); } if (PyList_GET_SIZE(seq_o) != oparg) { UNLOCK_OBJECT(seq_o); if (true) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); } } STAT_INC(UNPACK_SEQUENCE, hit); @@ -25088,6 +25325,7 @@ } TRACING_TARGET(UNPACK_SEQUENCE_TUPLE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = UNPACK_SEQUENCE_TUPLE; (void)(opcode); @@ -25110,7 +25348,7 @@ if (!PyTuple_CheckExact(o)) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); } } /* Skip 1 cache entry */ @@ -25123,7 +25361,7 @@ if (PyTuple_GET_SIZE(seq_o) != oparg) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); } STAT_INC(UNPACK_SEQUENCE, hit); PyObject **items = _PyTuple_ITEMS(seq_o); @@ -25140,6 +25378,7 @@ } TRACING_TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = UNPACK_SEQUENCE_TWO_TUPLE; (void)(opcode); @@ -25163,7 +25402,7 @@ if (!PyTuple_CheckExact(o)) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); } } /* Skip 1 cache entry */ @@ -25176,7 +25415,7 @@ if (PyTuple_GET_SIZE(seq_o) != 2) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(UNPACK_SEQUENCE); + JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); } STAT_INC(UNPACK_SEQUENCE, hit); val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); @@ -25193,6 +25432,7 @@ } TRACING_TARGET(WITH_EXCEPT_START) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = WITH_EXCEPT_START; (void)(opcode); @@ -25242,6 +25482,7 @@ } TRACING_TARGET(YIELD_VALUE) { + assert(IS_JIT_TRACING()); #if _Py_TAIL_CALL_INTERP int opcode = YIELD_VALUE; (void)(opcode); diff --git a/Python/optimizer.c b/Python/optimizer.c index ae1745a71de48b..21a0fb71651e2d 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -457,6 +457,18 @@ is_for_iter_test[MAX_UOP_ID + 1] = { [_FOR_ITER_TIER_TWO] = 1, }; +static const uint16_t +BRANCH_TO_GUARD[4][2] = { + [POP_JUMP_IF_FALSE - POP_JUMP_IF_FALSE][0] = _GUARD_IS_TRUE_POP, + [POP_JUMP_IF_FALSE - POP_JUMP_IF_FALSE][1] = _GUARD_IS_FALSE_POP, + [POP_JUMP_IF_TRUE - POP_JUMP_IF_FALSE][0] = _GUARD_IS_FALSE_POP, + [POP_JUMP_IF_TRUE - POP_JUMP_IF_FALSE][1] = _GUARD_IS_TRUE_POP, + [POP_JUMP_IF_NONE - POP_JUMP_IF_FALSE][0] = _GUARD_IS_NOT_NONE_POP, + [POP_JUMP_IF_NONE - POP_JUMP_IF_FALSE][1] = _GUARD_IS_NONE_POP, + [POP_JUMP_IF_NOT_NONE - POP_JUMP_IF_FALSE][0] = _GUARD_IS_NONE_POP, + [POP_JUMP_IF_NOT_NONE - POP_JUMP_IF_FALSE][1] = _GUARD_IS_NOT_NONE_POP, +}; + #define CONFIDENCE_RANGE 1000 #define CONFIDENCE_CUTOFF 333 @@ -525,10 +537,12 @@ add_to_trace( int _PyJIT_translate_single_bytecode_to_trace( PyThreadState *tstate, + _PyInterpreterFrame *frame, _Py_CODEUNIT *this_instr, _Py_CODEUNIT *next_instr, - PyCodeObject *code, + PyCodeObject *old_code, PyFunctionObject *func, + int opcode, int oparg) { if (Py_IsNone((PyObject *)func)) { @@ -536,7 +550,7 @@ _PyJIT_translate_single_bytecode_to_trace( } bool progress_needed = (tstate->interp->jit_tracer_initial_chain_depth % MAX_CHAIN_DEPTH) == 0;; _PyBloomFilter *dependencies = &tstate->interp->jit_tracer_dependencies; - _Py_BloomFilter_Add(dependencies, code); + _Py_BloomFilter_Add(dependencies, old_code); _Py_CODEUNIT *target_instr = this_instr; int trace_length = tstate->interp->jit_tracer_code_curr_size; _PyUOpInstruction *trace = tstate->interp->jit_tracer_code_buffer; @@ -552,10 +566,9 @@ _PyJIT_translate_single_bytecode_to_trace( uint32_t target = 0; - target = INSTR_IP(target_instr, code); + target = INSTR_IP(target_instr, old_code); // One for possible _DEOPT, one because _CHECK_VALIDITY itself might _DEOPT max_length-=2; - uint32_t opcode = this_instr->op.code; if ((uint16_t)oparg != (uint64_t)oparg) { goto full; } @@ -565,14 +578,16 @@ _PyJIT_translate_single_bytecode_to_trace( if (opcode == EXTENDED_ARG) { return 1; } - if (opcode == ENTER_EXECUTOR) { - goto full; - } assert(opcode != ENTER_EXECUTOR && opcode != EXTENDED_ARG); if (opcode == NOP) { return 1; } + // Unsupported opcodes + if (opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW) { + goto full; + } + RESERVE_RAW(1, "_CHECK_VALIDITY"); ADD_TO_TRACE(_CHECK_VALIDITY, 0, 0, target); @@ -583,7 +598,8 @@ _PyJIT_translate_single_bytecode_to_trace( bool needs_guard_ip = _PyOpcode_NeedsGuardIp[opcode] && !(opcode == FOR_ITER_RANGE || opcode == FOR_ITER_LIST || opcode == FOR_ITER_TUPLE) && - !(opcode == JUMP_BACKWARD_NO_INTERRUPT || opcode == JUMP_BACKWARD || opcode == JUMP_BACKWARD_JIT); + !(opcode == JUMP_BACKWARD_NO_INTERRUPT || opcode == JUMP_BACKWARD || opcode == JUMP_BACKWARD_JIT) && + !(opcode == POP_JUMP_IF_TRUE || opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_NONE || opcode == POP_JUMP_IF_NOT_NONE); if (needs_guard_ip) { RESERVE_RAW(1, "_GUARD_IP"); @@ -617,6 +633,19 @@ _PyJIT_translate_single_bytecode_to_trace( } switch (opcode) { + case POP_JUMP_IF_NONE: + case POP_JUMP_IF_NOT_NONE: + case POP_JUMP_IF_FALSE: + case POP_JUMP_IF_TRUE: + { + RESERVE(1); + int counter = this_instr[1].cache; + int bitcount = counter & 1; + int jump_likely = bitcount; + uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_likely]; + ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(target_instr, old_code)); + break; + } case JUMP_BACKWARD_JIT: case JUMP_BACKWARD: ADD_TO_TRACE(_CHECK_PERIODIC, 0, 0, target); @@ -678,8 +707,8 @@ _PyJIT_translate_single_bytecode_to_trace( #ifdef Py_DEBUG else { uint32_t jump_target = next_inst + oparg; - assert(_Py_GetBaseCodeUnit(code, jump_target).op.code == END_FOR); - assert(_Py_GetBaseCodeUnit(code, jump_target+1).op.code == POP_ITER); + assert(_Py_GetBaseCodeUnit(old_code, jump_target).op.code == END_FOR); + assert(_Py_GetBaseCodeUnit(old_code, jump_target+1).op.code == POP_ITER); } #endif break; @@ -707,11 +736,12 @@ _PyJIT_translate_single_bytecode_to_trace( Py_FatalError("garbled expansion"); } if (uop == _PUSH_FRAME || uop == _RETURN_VALUE || uop == _RETURN_GENERATOR || uop == _YIELD_VALUE) { + PyCodeObject *new_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); if (func != NULL) { operand = (uintptr_t)func; } - else if (code != NULL) { - operand = (uintptr_t)code | 1; + else if (new_code != NULL) { + operand = (uintptr_t)new_code | 1; } else { operand = 0; @@ -1091,7 +1121,7 @@ uop_optimize( char *env_var = Py_GETENV("PYTHON_UOPS_OPTIMIZE"); bool is_noopt = true; if (env_var == NULL || *env_var == '\0' || *env_var > '0') { - is_noopt = false; + is_noopt = true; } int curr_stackentries = tstate->interp->jit_tracer_initial_stack_depth; int length = interp->jit_tracer_code_curr_size; diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 1240be061b7828..74091012568ce2 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -2092,17 +2092,9 @@ break; } - case _POP_JUMP_IF_FALSE: { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - break; - } + /* _POP_JUMP_IF_FALSE is not a viable micro-op for tier 2 */ - case _POP_JUMP_IF_TRUE: { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - break; - } + /* _POP_JUMP_IF_TRUE is not a viable micro-op for tier 2 */ case _IS_NONE: { JitOptRef b; diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index c133a5e7b9fbd2..f3578fe1ffedb4 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -226,6 +226,8 @@ def generate_tier1_cases( for name, inst in sorted(analysis.instructions.items()): out.emit("\n") out.emit(f"{tracing_prepend}TARGET({name}) {{\n") + if is_tracing: + out.emit(f"assert(IS_JIT_TRACING());\n") popped = get_popped(inst, analysis) # We need to ifdef it because this breaks platforms # without computed gotos/tail calling. diff --git a/Tools/cases_generator/tracer_generator.py b/Tools/cases_generator/tracer_generator.py index e394807518f66c..1f2d161b70fe3a 100644 --- a/Tools/cases_generator/tracer_generator.py +++ b/Tools/cases_generator/tracer_generator.py @@ -57,6 +57,33 @@ def dispatch( self.emit("TRACING_DISPATCH") return False + def deopt_if( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ) -> bool: + self.out.start_line() + self.out.emit("if (") + lparen = next(tkn_iter) + assert lparen.kind == "LPAREN" + first_tkn = tkn_iter.peek() + emit_to(self.out, tkn_iter, "RPAREN") + self.emit(") {\n") + next(tkn_iter) # Semi colon + assert inst is not None + assert inst.family is not None + family_name = inst.family.name + self.emit(f"UPDATE_MISS_STATS({family_name});\n") + self.emit(f"assert(_PyOpcode_Deopt[opcode] == ({family_name}));\n") + self.emit(f"JUMP_TO_PREDICTED(TRACING_{family_name});\n") + self.emit("}\n") + return not always_true(first_tkn) + + exit_if = deopt_if + def generate_tracer_cases( analysis: Analysis, out: CWriter ): From 7192671f52b13a8748354e900609e36b08942e2e Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 20 Sep 2025 17:14:34 +0100 Subject: [PATCH 006/190] Fully working bm_generators --- Python/bytecodes.c | 3 +- Python/ceval.c | 7 ++-- Python/executor_cases.c.h | 3 +- Python/generated_cases.c.h | 4 +-- Python/optimizer.c | 44 ++++++++++++------------ Python/optimizer_analysis.c | 2 +- Tools/cases_generator/tier2_generator.py | 10 ------ 7 files changed, 33 insertions(+), 40 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index e972e4732b48a2..a0983a474af801 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3033,7 +3033,7 @@ dummy_func( /* If the eval breaker is set then stay in tier 1. * This avoids any potentially infinite loops * involving _RESUME_CHECK */ - if (IS_JIT_TRACING() || _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { opcode = executor->vm_data.opcode; oparg = (oparg & ~255) | executor->vm_data.oparg; next_instr = this_instr; @@ -3236,6 +3236,7 @@ dummy_func( } op(_FOR_ITER_TIER_TWO, (iter, null_or_index -- iter, null_or_index, next)) { + TIER2_JUMPBY(1 + INLINE_CACHE_ENTRIES_FOR_ITER); _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); if (!PyStackRef_IsValid(item)) { if (PyStackRef_IsError(item)) { diff --git a/Python/ceval.c b/Python/ceval.c index 940f27bd56611d..3eed68821021f0 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1195,8 +1195,11 @@ _PyTier2Interpreter( for (;;) { uopcode = next_uop->opcode; #ifdef Py_DEBUG - if (frame->lltrace >= 4 && next_uop->opcode != _YIELD_VALUE) { - // dump_stack(frame, stack_pointer); + if (frame->lltrace >= 4 && + next_uop->opcode != _YIELD_VALUE && + next_uop->opcode != _FOR_ITER_GEN_FRAME && + next_uop->opcode != _PUSH_FRAME) { + dump_stack(frame, stack_pointer); if (next_uop->opcode == _START_EXECUTOR) { printf("%4d uop: ", 0); } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 1000d7dedab068..f6c729f533504c 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4196,7 +4196,6 @@ } case _JUMP_BACKWARD_NO_INTERRUPT: { - TIER2_JUMPBY(2); oparg = CURRENT_OPARG(); #if TIER_ONE assert(oparg <= INSTR_OFFSET()); @@ -4402,6 +4401,7 @@ oparg = CURRENT_OPARG(); null_or_index = stack_pointer[-1]; iter = stack_pointer[-2]; + TIER2_JUMPBY(1 + INLINE_CACHE_ENTRIES_FOR_ITER); _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -4464,7 +4464,6 @@ } case _ITER_NEXT_LIST: { - TIER2_JUMPBY(2); _PyStackRef null_or_index; _PyStackRef iter; _PyStackRef next; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 007d97f8b11517..f7b890621e6980 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5640,7 +5640,7 @@ assert(executor->vm_data.code == code); assert(executor->vm_data.valid); assert(tstate->current_executor == NULL); - if (IS_JIT_TRACING() || _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { opcode = executor->vm_data.opcode; oparg = (oparg & ~255) | executor->vm_data.oparg; next_instr = this_instr; @@ -18323,7 +18323,7 @@ assert(executor->vm_data.code == code); assert(executor->vm_data.valid); assert(tstate->current_executor == NULL); - if (IS_JIT_TRACING() || _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { opcode = executor->vm_data.opcode; oparg = (oparg & ~255) | executor->vm_data.oparg; next_instr = this_instr; diff --git a/Python/optimizer.c b/Python/optimizer.c index 21a0fb71651e2d..4de08899456bae 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -568,7 +568,7 @@ _PyJIT_translate_single_bytecode_to_trace( target = INSTR_IP(target_instr, old_code); // One for possible _DEOPT, one because _CHECK_VALIDITY itself might _DEOPT - max_length-=2; + max_length -= 2; if ((uint16_t)oparg != (uint64_t)oparg) { goto full; } @@ -583,17 +583,13 @@ _PyJIT_translate_single_bytecode_to_trace( return 1; } - // Unsupported opcodes - if (opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW) { + if (opcode == ENTER_EXECUTOR) { goto full; } - RESERVE_RAW(1, "_CHECK_VALIDITY"); - ADD_TO_TRACE(_CHECK_VALIDITY, 0, 0, target); - - if (!OPCODE_HAS_NO_SAVE_IP(opcode)) { - RESERVE_RAW(2, "_SET_IP"); - ADD_TO_TRACE(_SET_IP, 0, (uintptr_t)target_instr, target); + // Unsupported opcodes + if (opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW) { + goto full; } bool needs_guard_ip = _PyOpcode_NeedsGuardIp[opcode] && @@ -601,8 +597,13 @@ _PyJIT_translate_single_bytecode_to_trace( !(opcode == JUMP_BACKWARD_NO_INTERRUPT || opcode == JUMP_BACKWARD || opcode == JUMP_BACKWARD_JIT) && !(opcode == POP_JUMP_IF_TRUE || opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_NONE || opcode == POP_JUMP_IF_NOT_NONE); - if (needs_guard_ip) { - RESERVE_RAW(1, "_GUARD_IP"); + const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; + RESERVE_RAW(expansion->nuops + needs_guard_ip + 3, "uop and various checks"); + + ADD_TO_TRACE(_CHECK_VALIDITY, 0, 0, target); + + if (!OPCODE_HAS_NO_SAVE_IP(opcode)) { + ADD_TO_TRACE(_SET_IP, 0, (uintptr_t)target_instr, target); } /* Special case the first instruction, @@ -623,12 +624,10 @@ _PyJIT_translate_single_bytecode_to_trace( if (OPCODE_HAS_EXIT(opcode)) { // Make space for side exit and final _EXIT_TRACE: - RESERVE_RAW(2, "_EXIT_TRACE"); max_length--; } if (OPCODE_HAS_ERROR(opcode)) { // Make space for error stub and final _EXIT_TRACE: - RESERVE_RAW(2, "_ERROR_POP_N"); max_length--; } @@ -643,7 +642,9 @@ _PyJIT_translate_single_bytecode_to_trace( int bitcount = counter & 1; int jump_likely = bitcount; uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_likely]; - ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(target_instr, old_code)); + _Py_CODEUNIT *next_instr = target_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; + _Py_CODEUNIT *false_target = next_instr + oparg; + ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(false_target, old_code)); break; } case JUMP_BACKWARD_JIT: @@ -704,13 +705,6 @@ _PyJIT_translate_single_bytecode_to_trace( if (uop == _TIER2_RESUME_CHECK) { target = next_inst; } -#ifdef Py_DEBUG - else { - uint32_t jump_target = next_inst + oparg; - assert(_Py_GetBaseCodeUnit(old_code, jump_target).op.code == END_FOR); - assert(_Py_GetBaseCodeUnit(old_code, jump_target+1).op.code == POP_ITER); - } -#endif break; case OPERAND1_1: assert(trace[trace_length-1].opcode == uop); @@ -772,6 +766,8 @@ _PyJIT_translate_single_bytecode_to_trace( if (!is_terminator(&tstate->interp->jit_tracer_code_buffer[trace_length-1])) { // Undo the last few instructions. trace_length = tstate->interp->jit_tracer_code_curr_size; + // We previously reversed one. + max_length += 1; ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); } tstate->interp->jit_tracer_code_curr_size = trace_length; @@ -1121,10 +1117,14 @@ uop_optimize( char *env_var = Py_GETENV("PYTHON_UOPS_OPTIMIZE"); bool is_noopt = true; if (env_var == NULL || *env_var == '\0' || *env_var > '0') { - is_noopt = true; + is_noopt = false; } int curr_stackentries = tstate->interp->jit_tracer_initial_stack_depth; int length = interp->jit_tracer_code_curr_size; + // Trace too short, don't bother. + if (length <= 8) { + return 0; + } assert(length > 0); assert(length < UOP_MAX_TRACE_LENGTH); OPT_STAT_INC(traces_created); diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 21410f1016ad5d..61a31adc6887dc 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -493,7 +493,7 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) } /* _PUSH_FRAME doesn't escape or error, but it * does need the IP for the return address */ - bool needs_ip = opcode == _PUSH_FRAME; + bool needs_ip = (opcode == _PUSH_FRAME || opcode == _YIELD_VALUE); if (_PyUop_Flags[opcode] & HAS_ESCAPES_FLAG) { needs_ip = true; may_have_escaped = true; diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index 35a10e7bebed1e..28adb132874d7a 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -226,16 +226,6 @@ def generate_tier2( ) continue out.emit(f"case {uop.name}: {{\n") - if uop.properties.jumps: - containing_inst = None - for inst in analysis.instructions.values(): - if uop in inst.parts: - print(uop.name, inst.name) - containing_inst = inst - break - assert containing_inst is not None, uop.name - size = containing_inst.size - out.emit(f"TIER2_JUMPBY({size});\n") declare_variables(uop, out) stack = Stack() stack = write_uop(uop, emitter, stack) From 07542dded54356d9bfcfc4c5752da5cb7338bc21 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 20 Sep 2025 17:18:58 +0100 Subject: [PATCH 007/190] fix jit build --- Tools/jit/template.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 8f71010a1aff58..d40217bc87ff51 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -56,11 +56,11 @@ do { \ } while (0) #undef GOTO_TIER_ONE -#define GOTO_TIER_ONE(TARGET) \ +#define GOTO_TIER_ONE(TARGET, SHOULD_CONTINUE_TRACING) \ do { \ tstate->current_executor = NULL; \ _PyFrame_SetStackPointer(frame, stack_pointer); \ - return TARGET; \ + return (_Py_CODEUNIT *)(((uintptr_t)(TARGET)) | SHOULD_CONTINUE_TRACING); \ } while (0) #undef LOAD_IP From 021fc4492a52f7ac6ea67daf3d6a12b832d91a1f Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 20 Sep 2025 22:17:47 +0100 Subject: [PATCH 008/190] Fix exception tracing --- Include/internal/pycore_optimizer.h | 3 ++- Include/internal/pycore_uop_metadata.h | 4 ++-- Python/bytecodes.c | 20 ++++++++++++++++++++ Python/ceval.c | 5 ++++- Python/executor_cases.c.h | 24 ++++++++++++++++++++++++ Python/optimizer.c | 20 +++++++++++++++----- Python/optimizer_analysis.c | 3 ++- 7 files changed, 69 insertions(+), 10 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index bea2838b8ba092..2eea92c9b64d45 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -345,7 +345,8 @@ static inline int is_terminator(const _PyUOpInstruction *uop) int opcode = uop->opcode; return ( opcode == _EXIT_TRACE || - opcode == _JUMP_TO_TOP + opcode == _JUMP_TO_TOP || + opcode == _DYNAMIC_EXIT ); } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 2b33e63ac6cf94..f5d7e1fafe8086 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -339,8 +339,8 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_ERROR_POP_N] = HAS_ARG_FLAG, [_TIER2_RESUME_CHECK] = HAS_PERIODIC_FLAG, [_COLD_EXIT] = HAS_ESCAPES_FLAG, - [_GUARD_IP] = 0, - [_DYNAMIC_EXIT] = 0, + [_GUARD_IP] = HAS_ESCAPES_FLAG, + [_DYNAMIC_EXIT] = HAS_ESCAPES_FLAG, }; const ReplicationRange _PyUop_Replication[MAX_UOP_ID+1] = { diff --git a/Python/bytecodes.c b/Python/bytecodes.c index a0983a474af801..5cec4f1112e437 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5446,11 +5446,31 @@ dummy_func( tier2 op(_GUARD_IP, (ip/4 --)) { if (frame->instr_ptr != (_Py_CODEUNIT *)ip) { +#ifdef Py_DEBUG + _Py_CODEUNIT *target = frame->instr_ptr; + if (frame->lltrace >= 2) { + printf("GUARD IP EXIT: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(", target %d -> %s]\n", + (int)(target - _PyFrame_GetBytecode(frame)), + _PyOpcode_OpName[target->op.code]); + } +#endif GOTO_TIER_ONE(frame->instr_ptr, 1); } } tier2 op(_DYNAMIC_EXIT, (ip/4 --)) { +#ifdef Py_DEBUG + _Py_CODEUNIT *target = frame->instr_ptr; + if (frame->lltrace >= 2) { + printf("GUARD IP EXIT: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(", target %d -> %s]\n", + (int)(target - _PyFrame_GetBytecode(frame)), + _PyOpcode_OpName[target->op.code]); + } +#endif GOTO_TIER_ONE(frame->instr_ptr, 1); } diff --git a/Python/ceval.c b/Python/ceval.c index 3eed68821021f0..afb69602a2574d 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1198,7 +1198,10 @@ _PyTier2Interpreter( if (frame->lltrace >= 4 && next_uop->opcode != _YIELD_VALUE && next_uop->opcode != _FOR_ITER_GEN_FRAME && - next_uop->opcode != _PUSH_FRAME) { + next_uop->opcode != _PUSH_FRAME && + next_uop->opcode != _PY_FRAME_KW && + next_uop->opcode != _SAVE_RETURN_OFFSET && + next_uop->opcode != _SAVE_RETURN_OFFSET) { dump_stack(frame, stack_pointer); if (next_uop->opcode == _START_EXECUTOR) { printf("%4d uop: ", 0); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index f6c729f533504c..013d7e78361698 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7551,6 +7551,18 @@ case _GUARD_IP: { PyObject *ip = (PyObject *)CURRENT_OPERAND0(); if (frame->instr_ptr != (_Py_CODEUNIT *)ip) { + #ifdef Py_DEBUG + _Py_CODEUNIT *target = frame->instr_ptr; + if (frame->lltrace >= 2) { + _PyFrame_SetStackPointer(frame, stack_pointer); + printf("GUARD IP EXIT: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(", target %d -> %s]\n", + (int)(target - _PyFrame_GetBytecode(frame)), + _PyOpcode_OpName[target->op.code]); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + #endif GOTO_TIER_ONE(frame->instr_ptr, 1); } break; @@ -7558,6 +7570,18 @@ case _DYNAMIC_EXIT: { PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + #ifdef Py_DEBUG + _Py_CODEUNIT *target = frame->instr_ptr; + if (frame->lltrace >= 2) { + _PyFrame_SetStackPointer(frame, stack_pointer); + printf("GUARD IP EXIT: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(", target %d -> %s]\n", + (int)(target - _PyFrame_GetBytecode(frame)), + _PyOpcode_OpName[target->op.code]); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + #endif GOTO_TIER_ONE(frame->instr_ptr, 1); break; } diff --git a/Python/optimizer.c b/Python/optimizer.c index 4de08899456bae..fa6a9616b332ce 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -587,16 +587,24 @@ _PyJIT_translate_single_bytecode_to_trace( goto full; } - // Unsupported opcodes - if (opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW) { - goto full; - } - bool needs_guard_ip = _PyOpcode_NeedsGuardIp[opcode] && !(opcode == FOR_ITER_RANGE || opcode == FOR_ITER_LIST || opcode == FOR_ITER_TUPLE) && !(opcode == JUMP_BACKWARD_NO_INTERRUPT || opcode == JUMP_BACKWARD || opcode == JUMP_BACKWARD_JIT) && !(opcode == POP_JUMP_IF_TRUE || opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_NONE || opcode == POP_JUMP_IF_NOT_NONE); + + // Strange control-flow + if (opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO) { + // Rewind to previous instruction and replace with _EXIT_TRACE. + _PyUOpInstruction *curr = &trace[trace_length-1]; + while (curr->opcode != _SET_IP && trace_length > 0) { + trace_length--; + curr = &trace[trace_length-1]; + } + curr->opcode = _EXIT_TRACE; + goto done; + } + const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; RESERVE_RAW(expansion->nuops + needs_guard_ip + 3, "uop and various checks"); @@ -618,6 +626,8 @@ _PyJIT_translate_single_bytecode_to_trace( // Loop back to the start if (tstate->interp->jit_tracer_initial_instr == this_instr && tstate->interp->jit_tracer_code_curr_size > 2) { + // Undo the last few instructions. + trace_length = tstate->interp->jit_tracer_code_curr_size; ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0, 0); goto done; } diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 61a31adc6887dc..1bb52188fba48e 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -493,7 +493,7 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) } /* _PUSH_FRAME doesn't escape or error, but it * does need the IP for the return address */ - bool needs_ip = (opcode == _PUSH_FRAME || opcode == _YIELD_VALUE); + bool needs_ip = (opcode == _PUSH_FRAME || opcode == _YIELD_VALUE || opcode == _DYNAMIC_EXIT); if (_PyUop_Flags[opcode] & HAS_ESCAPES_FLAG) { needs_ip = true; may_have_escaped = true; @@ -507,6 +507,7 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) } case _JUMP_TO_TOP: case _EXIT_TRACE: + case _DYNAMIC_EXIT: return pc + 1; } } From f886c4358507bc1600871b853f8631f4908b91e8 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 20 Sep 2025 23:19:39 +0100 Subject: [PATCH 009/190] Fix jump tracing --- Include/internal/pycore_optimizer.h | 3 +- Python/bytecodes.c | 4 + Python/ceval.c | 4 +- Python/ceval_macros.h | 4 +- Python/generated_cases.c.h | 904 +++++++++++++++++++++ Python/optimizer.c | 176 ++-- Tools/cases_generator/generators_common.py | 14 + Tools/cases_generator/tier1_generator.py | 2 + Tools/cases_generator/tracer_generator.py | 13 + 9 files changed, 1036 insertions(+), 88 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 2eea92c9b64d45..4b391d4d4fa926 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -366,7 +366,8 @@ _PyJIT_translate_single_bytecode_to_trace( PyCodeObject *code, PyFunctionObject *func, int opcode, - int oparg); + int oparg, + int jump_taken); void _PyJIT_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, int curr_stackdepth, int chain_depth); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 5cec4f1112e437..166383e4af6b82 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3230,6 +3230,7 @@ dummy_func( } // Jump forward by oparg and skip the following END_FOR JUMPBY(oparg + 1); + RECORD_JUMP_TAKEN(); DISPATCH(); } next = item; @@ -3291,6 +3292,7 @@ dummy_func( null_or_index = PyStackRef_TagInt(-1); /* Jump forward oparg, then skip following END_FOR instruction */ JUMPBY(oparg + 1); + RECORD_JUMP_TAKEN(); DISPATCH(); } #endif @@ -3368,6 +3370,7 @@ dummy_func( null_or_index = PyStackRef_TagInt(-1); /* Jump forward oparg, then skip following END_FOR instruction */ JUMPBY(oparg + 1); + RECORD_JUMP_TAKEN(); DISPATCH(); } } @@ -3412,6 +3415,7 @@ dummy_func( if (r->len <= 0) { // Jump over END_FOR instruction. JUMPBY(oparg + 1); + RECORD_JUMP_TAKEN(); DISPATCH(); } } diff --git a/Python/ceval.c b/Python/ceval.c index afb69602a2574d..a7edd45e1ea251 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -975,12 +975,12 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch) // 1 for trace full, 0 for successful write. static int -add_to_code_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, PyCodeObject *old_code, _Py_CODEUNIT *this_instr, _Py_CODEUNIT *next_instr, int opcode, int oparg) +add_to_code_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, PyCodeObject *old_code, _Py_CODEUNIT *this_instr, _Py_CODEUNIT *next_instr, int opcode, int oparg, int jump_taken) { assert(frame != NULL); assert(tstate->interp->jit_tracer_code_curr_size < UOP_MAX_TRACE_LENGTH); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - return !_PyJIT_translate_single_bytecode_to_trace(tstate, frame, this_instr, next_instr, old_code, func, opcode, oparg); + return !_PyJIT_translate_single_bytecode_to_trace(tstate, frame, this_instr, next_instr, old_code, func, opcode, oparg, jump_taken); } /* _PyEval_EvalFrameDefault is too large to optimize for speed with PGO on MSVC. diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 71e2e0baaeb57b..32815ad8c1a58f 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -144,7 +144,7 @@ DISPATCH(); # define RECORD_TRACE() do { \ frame->instr_ptr = next_instr; \ - if (add_to_code_trace(tstate, frame, old_code, this_instr, next_instr, opcode, oparg)) { \ + if (add_to_code_trace(tstate, frame, old_code, this_instr, next_instr, opcode, oparg, _jump_taken)) { \ BAIL_TRACING(); \ } \ } while (0); @@ -339,6 +339,8 @@ GETITEM(PyObject *v, Py_ssize_t i) { #define RECORD_BRANCH_TAKEN(bitset, flag) #endif +#define RECORD_JUMP_TAKEN() _jump_taken = 1; + #define UNBOUNDLOCAL_ERROR_MSG \ "cannot access local variable '%s' where it is not associated with a value" #define UNBOUNDFREE_ERROR_MSG \ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index f7b890621e6980..b612a51bc056df 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -25,6 +25,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP); @@ -90,6 +92,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -150,6 +154,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -212,6 +218,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -274,6 +282,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -340,6 +350,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -425,6 +437,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -485,6 +499,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -547,6 +563,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -615,6 +633,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -701,6 +721,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -793,6 +815,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -867,6 +891,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -947,6 +973,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -1022,6 +1050,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -1082,6 +1112,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -1144,6 +1176,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BINARY_SLICE); @@ -1202,6 +1236,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_INTERPOLATION); @@ -1262,6 +1298,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_LIST); @@ -1288,6 +1326,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_MAP); @@ -1342,6 +1382,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_SET); @@ -1401,6 +1443,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_SLICE); @@ -1438,6 +1482,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_STRING); @@ -1487,6 +1533,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_TEMPLATE); @@ -1527,6 +1575,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_TUPLE); @@ -1551,6 +1601,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CACHE); @@ -1566,6 +1618,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL); @@ -1746,6 +1800,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -1869,6 +1925,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -2014,6 +2072,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -2144,6 +2204,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -2251,6 +2313,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -2362,6 +2426,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -2473,6 +2539,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -2559,6 +2627,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -2728,6 +2798,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CALL_INTRINSIC_1); @@ -2760,6 +2832,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CALL_INTRINSIC_2); @@ -2801,6 +2875,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -2879,6 +2955,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW); @@ -3063,6 +3141,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -3195,6 +3275,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -3324,6 +3406,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -3436,6 +3520,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -3510,6 +3596,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -3600,6 +3688,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -3723,6 +3813,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -3848,6 +3940,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -3943,6 +4037,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -4051,6 +4147,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -4168,6 +4266,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -4283,6 +4383,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -4385,6 +4487,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -4459,6 +4563,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -4533,6 +4639,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -4590,6 +4698,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CHECK_EG_MATCH); @@ -4663,6 +4773,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CHECK_EXC_MATCH); @@ -4702,6 +4814,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -4763,6 +4877,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP); @@ -4838,6 +4954,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -4895,6 +5013,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -4956,6 +5076,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -5017,6 +5139,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP); @@ -5079,6 +5203,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -5139,6 +5265,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -5199,6 +5327,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CONVERT_VALUE); @@ -5233,6 +5363,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(COPY); @@ -5253,6 +5385,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(COPY_FREE_VARS); @@ -5276,6 +5410,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_ATTR); @@ -5303,6 +5439,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_DEREF); @@ -5327,6 +5465,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_FAST); @@ -5355,6 +5495,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_GLOBAL); @@ -5382,6 +5524,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_NAME); @@ -5416,6 +5560,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_SUBSCR); @@ -5450,6 +5596,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DICT_MERGE); @@ -5491,6 +5639,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DICT_UPDATE); @@ -5536,6 +5686,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -5583,6 +5735,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; next_instr += 1; INSTRUCTION_STATS(END_FOR); _PyStackRef value; @@ -5602,6 +5756,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(END_SEND); @@ -5627,6 +5783,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -5665,6 +5823,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(EXIT_INIT_CHECK); @@ -5690,6 +5850,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(EXTENDED_ARG); @@ -5708,6 +5870,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(FORMAT_SIMPLE); @@ -5746,6 +5910,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(FORMAT_WITH_SPEC); @@ -5784,6 +5950,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER); @@ -5840,6 +6008,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -5913,6 +6083,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -5999,6 +6171,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -6068,6 +6242,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -6126,6 +6302,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_AITER); @@ -6189,6 +6367,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_ANEXT); @@ -6215,6 +6395,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_AWAITABLE); @@ -6246,6 +6428,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_ITER); @@ -6294,6 +6478,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_LEN); @@ -6324,6 +6510,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_YIELD_FROM_ITER); @@ -6371,6 +6559,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IMPORT_FROM); @@ -6398,6 +6588,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IMPORT_NAME); @@ -6439,6 +6631,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -6629,6 +6823,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -6798,6 +6994,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -6986,6 +7184,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -7041,6 +7241,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; next_instr += 1; @@ -7072,6 +7274,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -7108,6 +7312,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -7146,6 +7352,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -7175,6 +7383,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -7204,6 +7414,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -7220,6 +7432,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -7262,6 +7476,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -7384,6 +7600,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -7402,6 +7620,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -7429,6 +7649,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -7455,6 +7677,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -7488,6 +7712,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -7519,6 +7745,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -7545,6 +7773,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -7627,6 +7857,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -7681,6 +7913,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -7754,6 +7988,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INTERPRETER_EXIT); @@ -7791,6 +8027,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IS_OP); @@ -7826,6 +8064,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(JUMP_BACKWARD); @@ -7869,6 +8109,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -7932,6 +8174,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); @@ -7949,6 +8193,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(JUMP_BACKWARD_NO_JIT); @@ -7980,6 +8226,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(JUMP_FORWARD); @@ -7994,6 +8242,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LIST_APPEND); @@ -8018,6 +8268,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LIST_EXTEND); @@ -8067,6 +8319,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR); @@ -8150,6 +8404,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -8210,6 +8466,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -8280,6 +8538,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -8340,6 +8600,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -8422,6 +8684,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -8480,6 +8744,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -8528,6 +8794,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -8597,6 +8865,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -8673,6 +8943,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -8722,6 +8994,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -8792,6 +9066,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -8888,6 +9164,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -8959,6 +9237,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -9071,6 +9351,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_BUILD_CLASS); @@ -9103,6 +9385,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); @@ -9122,6 +9406,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_CONST); @@ -9141,6 +9427,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_DEREF); @@ -9171,6 +9459,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST); @@ -9190,6 +9480,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); @@ -9209,6 +9501,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_BORROW); @@ -9228,6 +9522,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_BORROW_LOAD_FAST_BORROW); @@ -9251,6 +9547,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_CHECK); @@ -9279,6 +9577,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); @@ -9302,6 +9602,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); @@ -9349,6 +9651,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); @@ -9426,6 +9730,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL); @@ -9484,6 +9790,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -9566,6 +9874,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -9635,6 +9945,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_LOCALS); @@ -9661,6 +9973,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_NAME); @@ -9686,6 +10000,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_SMALL_INT); @@ -9706,6 +10022,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_SPECIAL); @@ -9754,6 +10072,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR); @@ -9893,6 +10213,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -9957,6 +10279,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -10037,6 +10361,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAKE_CELL); @@ -10060,6 +10386,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAKE_FUNCTION); @@ -10095,6 +10423,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAP_ADD); @@ -10128,6 +10458,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_CLASS); @@ -10182,6 +10514,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_KEYS); @@ -10211,6 +10545,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_MAPPING); @@ -10232,6 +10568,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_SEQUENCE); @@ -10253,6 +10591,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(NOP); @@ -10266,6 +10606,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(NOT_TAKEN); @@ -10279,6 +10621,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_EXCEPT); @@ -10302,6 +10646,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_ITER); @@ -10325,6 +10671,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -10349,6 +10697,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -10394,6 +10744,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -10439,6 +10791,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -10463,6 +10817,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_TOP); @@ -10483,6 +10839,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(PUSH_EXC_INFO); @@ -10514,6 +10872,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(PUSH_NULL); @@ -10532,6 +10892,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -10563,6 +10925,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -10593,6 +10957,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESERVED); @@ -10608,6 +10974,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESUME); @@ -10686,6 +11054,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -10726,6 +11096,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RETURN_GENERATOR); @@ -10770,6 +11142,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RETURN_VALUE); @@ -10806,6 +11180,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(SEND); @@ -10913,6 +11289,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -10984,6 +11362,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SETUP_ANNOTATIONS); @@ -11032,6 +11412,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_ADD); @@ -11058,6 +11440,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); @@ -11088,6 +11472,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_UPDATE); @@ -11117,6 +11503,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR); @@ -11176,6 +11564,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -11254,6 +11644,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -11307,6 +11699,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -11397,6 +11791,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_DEREF); @@ -11418,6 +11814,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_FAST); @@ -11440,6 +11838,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); @@ -11465,6 +11865,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_FAST_STORE_FAST); @@ -11498,6 +11900,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_GLOBAL); @@ -11525,6 +11929,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_NAME); @@ -11573,6 +11979,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_SLICE); @@ -11635,6 +12043,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR); @@ -11696,6 +12106,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -11749,6 +12161,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -11832,6 +12246,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SWAP); @@ -11854,6 +12270,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL); @@ -11908,6 +12326,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -11953,6 +12373,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -11979,6 +12401,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -12021,6 +12445,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -12066,6 +12492,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -12095,6 +12523,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -12145,6 +12575,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNARY_INVERT); @@ -12176,6 +12608,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNARY_NEGATIVE); @@ -12207,6 +12641,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNARY_NOT); @@ -12227,6 +12663,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNPACK_EX); @@ -12256,6 +12694,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE); @@ -12309,6 +12749,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -12370,6 +12812,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -12422,6 +12866,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -12475,6 +12921,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(WITH_EXCEPT_START); @@ -12522,6 +12970,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(YIELD_VALUE); @@ -12576,6 +13026,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP); @@ -12642,6 +13094,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -12703,6 +13157,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -12766,6 +13222,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -12829,6 +13287,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -12896,6 +13356,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -12982,6 +13444,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -13043,6 +13507,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -13106,6 +13572,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -13175,6 +13643,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -13262,6 +13732,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -13355,6 +13827,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -13430,6 +13904,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -13511,6 +13987,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -13587,6 +14065,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -13648,6 +14128,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -13711,6 +14193,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -13772,6 +14256,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -13835,6 +14321,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -13864,6 +14352,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -13921,6 +14411,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -13983,6 +14475,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -14023,6 +14517,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -14075,6 +14571,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -14118,6 +14616,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -14145,6 +14645,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -14163,6 +14665,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL); @@ -14344,6 +14848,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -14468,6 +14974,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -14614,6 +15122,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -14745,6 +15255,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -14853,6 +15365,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -14965,6 +15479,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -15077,6 +15593,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -15164,6 +15682,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -15334,6 +15854,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -15369,6 +15891,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -15413,6 +15937,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -15492,6 +16018,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW); @@ -15677,6 +16205,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -15810,6 +16340,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -15940,6 +16472,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -16053,6 +16587,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -16128,6 +16664,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -16219,6 +16757,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -16343,6 +16883,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -16469,6 +17011,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -16565,6 +17109,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -16674,6 +17220,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -16792,6 +17340,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -16908,6 +17458,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -17011,6 +17563,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -17086,6 +17640,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -17161,6 +17717,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -17219,6 +17777,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -17295,6 +17855,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -17337,6 +17899,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -17399,6 +17963,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP); @@ -17475,6 +18041,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -17533,6 +18101,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -17595,6 +18165,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -17657,6 +18229,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP); @@ -17720,6 +18294,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -17781,6 +18357,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -17842,6 +18420,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -17879,6 +18459,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -17902,6 +18484,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -17928,6 +18512,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -17958,6 +18544,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -17985,6 +18573,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -18016,6 +18606,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -18046,6 +18638,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -18083,6 +18677,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -18120,6 +18716,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -18164,6 +18762,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -18212,6 +18812,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -18260,6 +18862,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; next_instr += 1; @@ -18282,6 +18886,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -18310,6 +18916,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -18349,6 +18957,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -18377,6 +18987,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -18398,6 +19010,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -18439,6 +19053,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -18480,6 +19096,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER); @@ -18517,6 +19135,7 @@ JUMP_TO_LABEL(error); } JUMPBY(oparg + 1); + RECORD_JUMP_TAKEN(); stack_pointer[-1] = null_or_index; TRACING_DISPATCH(); } @@ -18537,6 +19156,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -18611,6 +19232,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -18651,6 +19274,7 @@ if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { null_or_index = PyStackRef_TagInt(-1); JUMPBY(oparg + 1); + RECORD_JUMP_TAKEN(); stack_pointer[-1] = null_or_index; TRACING_DISPATCH(); } @@ -18698,6 +19322,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -18734,6 +19360,7 @@ STAT_INC(FOR_ITER, hit); if (r->len <= 0) { JUMPBY(oparg + 1); + RECORD_JUMP_TAKEN(); TRACING_DISPATCH(); } } @@ -18768,6 +19395,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -18799,6 +19428,7 @@ if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { null_or_index = PyStackRef_TagInt(-1); JUMPBY(oparg + 1); + RECORD_JUMP_TAKEN(); stack_pointer[-1] = null_or_index; TRACING_DISPATCH(); } @@ -18827,6 +19457,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -18893,6 +19525,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -18922,6 +19556,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -18956,6 +19592,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -19007,6 +19645,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -19040,6 +19680,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -19090,6 +19732,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -19120,6 +19764,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -19164,6 +19810,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -19355,6 +20003,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -19525,6 +20175,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -19714,6 +20366,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -19770,6 +20424,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; next_instr += 1; @@ -19802,6 +20458,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -19839,6 +20497,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -19878,6 +20538,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -19908,6 +20570,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -19938,6 +20602,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -19955,6 +20621,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -19998,6 +20666,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -20121,6 +20791,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -20140,6 +20812,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; @@ -20168,6 +20842,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -20195,6 +20871,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -20229,6 +20907,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -20261,6 +20941,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -20288,6 +20970,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -20371,6 +21055,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -20426,6 +21112,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -20500,6 +21188,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -20540,6 +21230,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -20578,6 +21270,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(JUMP_BACKWARD); @@ -20622,6 +21316,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -20686,6 +21382,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -20706,6 +21404,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -20740,6 +21440,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -20757,6 +21459,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -20784,6 +21488,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -20836,6 +21542,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR); @@ -20920,6 +21628,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -20981,6 +21691,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -21052,6 +21764,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -21113,6 +21827,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -21196,6 +21912,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -21255,6 +21973,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -21304,6 +22024,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -21374,6 +22096,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -21451,6 +22175,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -21501,6 +22227,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -21572,6 +22300,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -21669,6 +22399,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -21741,6 +22473,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -21854,6 +22588,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -21889,6 +22625,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -21911,6 +22649,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -21933,6 +22673,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -21966,6 +22708,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -21988,6 +22732,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22010,6 +22756,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22032,6 +22780,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22058,6 +22808,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22089,6 +22841,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22115,6 +22869,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22165,6 +22921,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22245,6 +23003,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL); @@ -22304,6 +23064,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22387,6 +23149,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22457,6 +23221,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22486,6 +23252,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22514,6 +23282,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22537,6 +23307,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22588,6 +23360,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR); @@ -22728,6 +23502,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22793,6 +23569,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22874,6 +23652,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22900,6 +23680,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22938,6 +23720,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -22974,6 +23758,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23031,6 +23817,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23063,6 +23851,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23087,6 +23877,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23111,6 +23903,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23127,6 +23921,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23143,6 +23939,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23169,6 +23967,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23195,6 +23995,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23220,6 +24022,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23266,6 +24070,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23312,6 +24118,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23337,6 +24145,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23360,6 +24170,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23394,6 +24206,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23415,6 +24229,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23447,6 +24263,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23478,6 +24296,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23496,6 +24316,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESUME); @@ -23575,6 +24397,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23616,6 +24440,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23663,6 +24489,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23702,6 +24530,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(SEND); @@ -23810,6 +24640,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23882,6 +24714,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23933,6 +24767,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23962,6 +24798,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -23995,6 +24833,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -24027,6 +24867,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR); @@ -24087,6 +24929,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -24166,6 +25010,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -24220,6 +25066,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -24311,6 +25159,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -24335,6 +25185,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -24360,6 +25212,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -24388,6 +25242,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -24424,6 +25280,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -24454,6 +25312,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -24505,6 +25365,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -24570,6 +25432,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR); @@ -24632,6 +25496,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -24686,6 +25552,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -24770,6 +25638,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -24795,6 +25665,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL); @@ -24850,6 +25722,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -24896,6 +25770,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -24923,6 +25799,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -24966,6 +25844,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -25012,6 +25892,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -25042,6 +25924,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -25093,6 +25977,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -25127,6 +26013,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -25161,6 +26049,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -25184,6 +26074,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -25216,6 +26108,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE); @@ -25270,6 +26164,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -25332,6 +26228,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -25385,6 +26283,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -25439,6 +26339,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; @@ -25489,6 +26391,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + int _jump_taken = false; + (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; diff --git a/Python/optimizer.c b/Python/optimizer.c index fa6a9616b332ce..f0b207f721cde6 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -543,7 +543,8 @@ _PyJIT_translate_single_bytecode_to_trace( PyCodeObject *old_code, PyFunctionObject *func, int opcode, - int oparg) + int oparg, + int jump_taken) { if (Py_IsNone((PyObject *)func)) { func = NULL; @@ -570,6 +571,12 @@ _PyJIT_translate_single_bytecode_to_trace( // One for possible _DEOPT, one because _CHECK_VALIDITY itself might _DEOPT max_length -= 2; if ((uint16_t)oparg != (uint64_t)oparg) { + // Back up over EXTENDED_ARGs + _PyUOpInstruction *curr = &trace[trace_length-1]; + while (oparg > 0) { + oparg >>= 8; + trace_length--; + } goto full; } @@ -594,13 +601,14 @@ _PyJIT_translate_single_bytecode_to_trace( // Strange control-flow - if (opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO) { + if (jump_taken || opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO) { // Rewind to previous instruction and replace with _EXIT_TRACE. _PyUOpInstruction *curr = &trace[trace_length-1]; - while (curr->opcode != _SET_IP && trace_length > 0) { + while (curr->opcode != _SET_IP && trace_length > 1) { trace_length--; curr = &trace[trace_length-1]; } + assert(curr->opcode == _SET_IP || trace_length == 1); curr->opcode = _EXIT_TRACE; goto done; } @@ -673,91 +681,91 @@ _PyJIT_translate_single_bytecode_to_trace( default: { const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; - if (expansion->nuops > 0) { - // Reserve space for nuops (+ _SET_IP + _EXIT_TRACE) - int nuops = expansion->nuops; - RESERVE(nuops + 1); /* One extra for exit */ - uint32_t orig_oparg = oparg; // For OPARG_TOP/BOTTOM - for (int i = 0; i < nuops; i++) { - oparg = orig_oparg; - uint32_t uop = expansion->uops[i].uop; - uint64_t operand = 0; - // Add one to account for the actual opcode/oparg pair: - int offset = expansion->uops[i].offset + 1; - switch (expansion->uops[i].size) { - case OPARG_SIMPLE: - assert(opcode != _JUMP_BACKWARD_NO_INTERRUPT && opcode != JUMP_BACKWARD); - break; - case OPARG_CACHE_1: - operand = read_u16(&this_instr[offset].cache); - break; - case OPARG_CACHE_2: - operand = read_u32(&this_instr[offset].cache); - break; - case OPARG_CACHE_4: - operand = read_u64(&this_instr[offset].cache); - break; - case OPARG_TOP: // First half of super-instr - oparg = orig_oparg >> 4; - break; - case OPARG_BOTTOM: // Second half of super-instr - oparg = orig_oparg & 0xF; - break; - case OPARG_SAVE_RETURN_OFFSET: // op=_SAVE_RETURN_OFFSET; oparg=return_offset - oparg = offset; - assert(uop == _SAVE_RETURN_OFFSET); - break; - case OPARG_REPLACED: - uop = _PyUOp_Replacements[uop]; - assert(uop != 0); - - uint32_t next_inst = target + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + (oparg > 255); - if (uop == _TIER2_RESUME_CHECK) { - target = next_inst; - } - break; - case OPERAND1_1: - assert(trace[trace_length-1].opcode == uop); - operand = read_u16(&this_instr[offset].cache); - trace[trace_length-1].operand1 = operand; - continue; - case OPERAND1_2: - assert(trace[trace_length-1].opcode == uop); - operand = read_u32(&this_instr[offset].cache); - trace[trace_length-1].operand1 = operand; - continue; - case OPERAND1_4: - assert(trace[trace_length-1].opcode == uop); - operand = read_u64(&this_instr[offset].cache); - trace[trace_length-1].operand1 = operand; - continue; - default: - fprintf(stderr, - "opcode=%d, oparg=%d; nuops=%d, i=%d; size=%d, offset=%d\n", - opcode, oparg, nuops, i, - expansion->uops[i].size, - expansion->uops[i].offset); - Py_FatalError("garbled expansion"); - } - if (uop == _PUSH_FRAME || uop == _RETURN_VALUE || uop == _RETURN_GENERATOR || uop == _YIELD_VALUE) { - PyCodeObject *new_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - if (func != NULL) { - operand = (uintptr_t)func; - } - else if (new_code != NULL) { - operand = (uintptr_t)new_code | 1; - } - else { - operand = 0; + if (expansion->nuops == 0) { + DPRINTF(2, "Unsupported opcode %s\n", _PyOpcode_OpName[opcode]); + goto full; + } + // Reserve space for nuops (+ _SET_IP + _EXIT_TRACE) + int nuops = expansion->nuops; + RESERVE(nuops + 1); /* One extra for exit */ + uint32_t orig_oparg = oparg; // For OPARG_TOP/BOTTOM + for (int i = 0; i < nuops; i++) { + oparg = orig_oparg; + uint32_t uop = expansion->uops[i].uop; + uint64_t operand = 0; + // Add one to account for the actual opcode/oparg pair: + int offset = expansion->uops[i].offset + 1; + switch (expansion->uops[i].size) { + case OPARG_SIMPLE: + assert(opcode != _JUMP_BACKWARD_NO_INTERRUPT && opcode != JUMP_BACKWARD); + break; + case OPARG_CACHE_1: + operand = read_u16(&this_instr[offset].cache); + break; + case OPARG_CACHE_2: + operand = read_u32(&this_instr[offset].cache); + break; + case OPARG_CACHE_4: + operand = read_u64(&this_instr[offset].cache); + break; + case OPARG_TOP: // First half of super-instr + oparg = orig_oparg >> 4; + break; + case OPARG_BOTTOM: // Second half of super-instr + oparg = orig_oparg & 0xF; + break; + case OPARG_SAVE_RETURN_OFFSET: // op=_SAVE_RETURN_OFFSET; oparg=return_offset + oparg = offset; + assert(uop == _SAVE_RETURN_OFFSET); + break; + case OPARG_REPLACED: + uop = _PyUOp_Replacements[uop]; + assert(uop != 0); + + uint32_t next_inst = target + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + (oparg > 255); + if (uop == _TIER2_RESUME_CHECK) { + target = next_inst; } + break; + case OPERAND1_1: + assert(trace[trace_length-1].opcode == uop); + operand = read_u16(&this_instr[offset].cache); + trace[trace_length-1].operand1 = operand; + continue; + case OPERAND1_2: + assert(trace[trace_length-1].opcode == uop); + operand = read_u32(&this_instr[offset].cache); + trace[trace_length-1].operand1 = operand; + continue; + case OPERAND1_4: + assert(trace[trace_length-1].opcode == uop); + operand = read_u64(&this_instr[offset].cache); + trace[trace_length-1].operand1 = operand; + continue; + default: + fprintf(stderr, + "opcode=%d, oparg=%d; nuops=%d, i=%d; size=%d, offset=%d\n", + opcode, oparg, nuops, i, + expansion->uops[i].size, + expansion->uops[i].offset); + Py_FatalError("garbled expansion"); + } + if (uop == _PUSH_FRAME || uop == _RETURN_VALUE || uop == _RETURN_GENERATOR || uop == _YIELD_VALUE) { + PyCodeObject *new_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + if (func != NULL) { + operand = (uintptr_t)func; + } + else if (new_code != NULL) { + operand = (uintptr_t)new_code | 1; + } + else { + operand = 0; } - // All other instructions - ADD_TO_TRACE(uop, oparg, operand, target); } - break; + // All other instructions + ADD_TO_TRACE(uop, oparg, operand, target); } - DPRINTF(2, "Unsupported opcode %s\n", _PyOpcode_OpName[opcode]); - OPT_UNSUPPORTED_OPCODE(opcode); + break; } // End default } // End switch (opcode) diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 61e855eb003706..2ecf27a6585b0c 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -127,6 +127,7 @@ def __init__(self, out: CWriter, labels: dict[str, Label], cannot_escape: bool = "DISPATCH": self.dispatch, "INSTRUCTION_SIZE": self.instruction_size, "stack_pointer": self.stack_pointer, + "RECORD_JUMP_TAKEN": self.record_jump_taken, } self.out = out self.labels = labels @@ -473,6 +474,19 @@ def instruction_size(self, self.out.emit(f" {uop.instruction_size}u ") return True + def record_jump_taken( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ) -> bool: + next(tkn_iter); + next(tkn_iter); + next(tkn_iter); + return True + def _print_storage(self, reason:str, storage: Storage) -> None: if DEBUG: self.out.start_line() diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index f3578fe1ffedb4..176c4e5e3b3789 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -237,6 +237,8 @@ def generate_tier1_cases( out.emit(f"#endif\n") out.emit(f"PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable);\n") out.emit(f"(void)old_code;\n") + out.emit(f"int _jump_taken = false;\n") + out.emit(f"(void)_jump_taken;\n") needs_this = is_tracing or uses_this(inst) unused_guard = "(void)this_instr;\n" if inst.properties.needs_prev: diff --git a/Tools/cases_generator/tracer_generator.py b/Tools/cases_generator/tracer_generator.py index 1f2d161b70fe3a..910d2589b14eea 100644 --- a/Tools/cases_generator/tracer_generator.py +++ b/Tools/cases_generator/tracer_generator.py @@ -57,6 +57,19 @@ def dispatch( self.emit("TRACING_DISPATCH") return False + def record_jump_taken( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ) -> bool: + self.out.emit(tkn) + emit_to(self.out, tkn_iter, "SEMI") + self.out.emit(";\n") + return True + def deopt_if( self, tkn: Token, From 3066963b60873f08356c5917d2548182a651cb04 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 20 Sep 2025 23:55:09 +0100 Subject: [PATCH 010/190] Fix handling of ENTER_EXECUTOR --- Python/bytecodes.c | 6 +++++- Python/ceval_macros.h | 10 ++++++---- Python/generated_cases.c.h | 12 ++++++++++-- Python/optimizer.c | 3 ++- 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 166383e4af6b82..a377bb5a385f2b 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3044,7 +3044,11 @@ dummy_func( } assert(executor != tstate->interp->cold_executor); tstate->jit_exit = NULL; - TIER1_TO_TIER2(executor); + if (IS_JIT_TRACING()) { + RECORD_TRACE(); + BAIL_TRACING_NO_DISPATCH(); + } + TIER1_TO_TIER2(executor, 1); #else Py_FatalError("ENTER_EXECUTOR is not supported in this build"); #endif /* _Py_TIER2 */ diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 32815ad8c1a58f..47da165924536f 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -134,13 +134,15 @@ # define IS_JIT_TRACING() (DISPATCH_TABLE_VAR == TRACING_DISPATCH_TABLE) # define ENTER_TRACING() DISPATCH_TABLE_VAR = TRACING_DISPATCH_TABLE; # define LEAVE_TRACING() DISPATCH_TABLE_VAR = DISPATCH_TABLE; -# define BAIL_TRACING() \ +# define BAIL_TRACING_NO_DISPATCH() \ LEAVE_TRACING(); \ int err = _PyOptimizer_Optimize(frame, tstate); \ tstate->interp->jit_tracer_code_curr_size = 0; \ if (err < 0) { \ JUMP_TO_LABEL(error); \ - } \ + } +# define BAIL_TRACING() \ + BAIL_TRACING_NO_DISPATCH() \ DISPATCH(); # define RECORD_TRACE() do { \ frame->instr_ptr = next_instr; \ @@ -395,7 +397,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer) /* Tier-switching macros. */ -#define TIER1_TO_TIER2(EXECUTOR) \ +#define TIER1_TO_TIER2(EXECUTOR, IN_ENTER_EXECUTOR) \ do { \ OPT_STAT_INC(traces_executed); \ next_instr = _Py_jit_entry((EXECUTOR), frame, stack_pointer, tstate); \ @@ -407,7 +409,7 @@ do { \ next_instr = frame->instr_ptr; \ JUMP_TO_LABEL(error); \ } \ - if (keep_tracing_bit) { \ + if (!IN_ENTER_EXECUTOR && keep_tracing_bit) { \ ENTER_TRACING(); \ _PyJIT_InitializeTracing(tstate, frame, next_instr, STACK_LEVEL(), 0); \ } \ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index b612a51bc056df..4183e23cdf1802 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5809,7 +5809,11 @@ } assert(executor != tstate->interp->cold_executor); tstate->jit_exit = NULL; - TIER1_TO_TIER2(executor); + if (IS_JIT_TRACING()) { + RECORD_TRACE(); + BAIL_TRACING_NO_DISPATCH(); + } + TIER1_TO_TIER2(executor, 1); #else Py_FatalError("ENTER_EXECUTOR is not supported in this build"); #endif /* _Py_TIER2 */ @@ -18942,7 +18946,11 @@ } assert(executor != tstate->interp->cold_executor); tstate->jit_exit = NULL; - TIER1_TO_TIER2(executor); + if (IS_JIT_TRACING()) { + RECORD_TRACE(); + BAIL_TRACING_NO_DISPATCH(); + } + TIER1_TO_TIER2(executor, 1); #else Py_FatalError("ENTER_EXECUTOR is not supported in this build"); #endif /* _Py_TIER2 */ diff --git a/Python/optimizer.c b/Python/optimizer.c index f0b207f721cde6..b973fdd7ac4159 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -585,7 +585,6 @@ _PyJIT_translate_single_bytecode_to_trace( if (opcode == EXTENDED_ARG) { return 1; } - assert(opcode != ENTER_EXECUTOR && opcode != EXTENDED_ARG); if (opcode == NOP) { return 1; } @@ -594,6 +593,8 @@ _PyJIT_translate_single_bytecode_to_trace( goto full; } + assert(opcode != ENTER_EXECUTOR && opcode != EXTENDED_ARG); + bool needs_guard_ip = _PyOpcode_NeedsGuardIp[opcode] && !(opcode == FOR_ITER_RANGE || opcode == FOR_ITER_LIST || opcode == FOR_ITER_TUPLE) && !(opcode == JUMP_BACKWARD_NO_INTERRUPT || opcode == JUMP_BACKWARD || opcode == JUMP_BACKWARD_JIT) && From 20b283b7b521f62cc4b5096e2a78b349be17cc80 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 21 Sep 2025 11:23:50 +0100 Subject: [PATCH 011/190] Fix ENTER_EXECUTOR bug --- Python/bytecodes.c | 9 +++++++-- Python/ceval_macros.h | 7 ++++++- Python/generated_cases.c.h | 18 ++++++++++++++---- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index a377bb5a385f2b..98ff74b0d6afb1 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3045,8 +3045,13 @@ dummy_func( assert(executor != tstate->interp->cold_executor); tstate->jit_exit = NULL; if (IS_JIT_TRACING()) { - RECORD_TRACE(); - BAIL_TRACING_NO_DISPATCH(); + int old_opcode = executor->vm_data.opcode; + int old_oparg = (oparg & ~255) | executor->vm_data.oparg; + RECORD_TRACE_NO_DISPATCH(); + opcode = old_opcode; + oparg = old_oparg; + next_instr = this_instr; + DISPATCH_GOTO(); } TIER1_TO_TIER2(executor, 1); #else diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 47da165924536f..a13c9427804485 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -137,7 +137,6 @@ # define BAIL_TRACING_NO_DISPATCH() \ LEAVE_TRACING(); \ int err = _PyOptimizer_Optimize(frame, tstate); \ - tstate->interp->jit_tracer_code_curr_size = 0; \ if (err < 0) { \ JUMP_TO_LABEL(error); \ } @@ -150,6 +149,12 @@ BAIL_TRACING(); \ } \ } while (0); +# define RECORD_TRACE_NO_DISPATCH() do { \ + frame->instr_ptr = next_instr; \ + if (add_to_code_trace(tstate, frame, old_code, this_instr, next_instr, opcode, oparg, _jump_taken)) { \ + BAIL_TRACING_NO_DISPATCH(); \ + } \ + } while (0); #endif diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 4183e23cdf1802..9551acb4d64bb9 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5810,8 +5810,13 @@ assert(executor != tstate->interp->cold_executor); tstate->jit_exit = NULL; if (IS_JIT_TRACING()) { - RECORD_TRACE(); - BAIL_TRACING_NO_DISPATCH(); + int old_opcode = executor->vm_data.opcode; + int old_oparg = (oparg & ~255) | executor->vm_data.oparg; + RECORD_TRACE_NO_DISPATCH(); + opcode = old_opcode; + oparg = old_oparg; + next_instr = this_instr; + DISPATCH_GOTO(); } TIER1_TO_TIER2(executor, 1); #else @@ -18947,8 +18952,13 @@ assert(executor != tstate->interp->cold_executor); tstate->jit_exit = NULL; if (IS_JIT_TRACING()) { - RECORD_TRACE(); - BAIL_TRACING_NO_DISPATCH(); + int old_opcode = executor->vm_data.opcode; + int old_oparg = (oparg & ~255) | executor->vm_data.oparg; + RECORD_TRACE_NO_DISPATCH(); + opcode = old_opcode; + oparg = old_oparg; + next_instr = this_instr; + DISPATCH_GOTO(); } TIER1_TO_TIER2(executor, 1); #else From 36554a514c9be979479b040e9c6270781b69ce59 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 21 Sep 2025 12:15:07 +0100 Subject: [PATCH 012/190] Fix over-tracing bug --- Include/internal/pycore_uop_metadata.h | 2 +- Python/bytecodes.c | 47 ++++++++++++++++---------- Python/ceval_macros.h | 5 +-- Python/executor_cases.c.h | 46 ++++++++++++++++--------- Python/generated_cases.c.h | 12 ++++--- Python/optimizer.c | 6 ++-- 6 files changed, 74 insertions(+), 44 deletions(-) diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index f5d7e1fafe8086..a7eebe9ad0d5a0 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -339,7 +339,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_ERROR_POP_N] = HAS_ARG_FLAG, [_TIER2_RESUME_CHECK] = HAS_PERIODIC_FLAG, [_COLD_EXIT] = HAS_ESCAPES_FLAG, - [_GUARD_IP] = HAS_ESCAPES_FLAG, + [_GUARD_IP] = HAS_EXIT_FLAG, [_DYNAMIC_EXIT] = HAS_ESCAPES_FLAG, }; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 98ff74b0d6afb1..61e97cb99230a6 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2961,7 +2961,9 @@ dummy_func( tier1 op(_JIT, (--)) { #ifdef _Py_TIER2 _Py_BackoffCounter counter = this_instr[1].counter; - if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD_JIT) { + if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && + this_instr->op.code == JUMP_BACKWARD_JIT && + next_instr->op.code != ENTER_EXECUTOR) { _Py_CODEUNIT *start = this_instr; /* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */ int curr_oparg = oparg; @@ -3053,7 +3055,7 @@ dummy_func( next_instr = this_instr; DISPATCH_GOTO(); } - TIER1_TO_TIER2(executor, 1); + TIER1_TO_TIER2(executor); #else Py_FatalError("ENTER_EXECUTOR is not supported in this build"); #endif /* _Py_TIER2 */ @@ -5458,24 +5460,12 @@ dummy_func( } tier2 op(_GUARD_IP, (ip/4 --)) { - if (frame->instr_ptr != (_Py_CODEUNIT *)ip) { -#ifdef Py_DEBUG - _Py_CODEUNIT *target = frame->instr_ptr; - if (frame->lltrace >= 2) { - printf("GUARD IP EXIT: [UOp "); - _PyUOpPrint(&next_uop[-1]); - printf(", target %d -> %s]\n", - (int)(target - _PyFrame_GetBytecode(frame)), - _PyOpcode_OpName[target->op.code]); - } -#endif - GOTO_TIER_ONE(frame->instr_ptr, 1); - } + EXIT_IF(frame->instr_ptr != (_Py_CODEUNIT *)ip); } - tier2 op(_DYNAMIC_EXIT, (ip/4 --)) { -#ifdef Py_DEBUG + tier2 op(_DYNAMIC_EXIT, (exit_p/4 --)) { _Py_CODEUNIT *target = frame->instr_ptr; +#ifdef Py_DEBUG if (frame->lltrace >= 2) { printf("GUARD IP EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); @@ -5484,7 +5474,28 @@ dummy_func( _PyOpcode_OpName[target->op.code]); } #endif - GOTO_TIER_ONE(frame->instr_ptr, 1); + _PyExitData *exit = (_PyExitData *)exit_p; + tstate->jit_exit = exit_p; + _Py_BackoffCounter temperature = exit->temperature; + _PyExecutorObject *executor; + if (target->op.code == ENTER_EXECUTOR) { + PyCodeObject *code = _PyFrame_GetCode(frame); + executor = code->co_executors->executors[target->op.arg]; + Py_INCREF(executor); + } + else { + if (!backoff_counter_triggers(temperature)) { + exit->temperature = advance_backoff_counter(temperature); + GOTO_TIER_ONE(frame->instr_ptr, 0); + } + _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); + assert(tstate->current_executor == (PyObject *)previous_executor); + int chain_depth = previous_executor->vm_data.chain_depth + 1; + _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), chain_depth); + GOTO_TIER_ONE(target, 1); + } + exit->executor = executor; + TIER2_TO_TIER2(exit->executor); } label(pop_2_error) { diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index a13c9427804485..0e7e5e81a7f9ff 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -402,7 +402,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer) /* Tier-switching macros. */ -#define TIER1_TO_TIER2(EXECUTOR, IN_ENTER_EXECUTOR) \ +#define TIER1_TO_TIER2(EXECUTOR) \ do { \ OPT_STAT_INC(traces_executed); \ next_instr = _Py_jit_entry((EXECUTOR), frame, stack_pointer, tstate); \ @@ -414,7 +414,8 @@ do { \ next_instr = frame->instr_ptr; \ JUMP_TO_LABEL(error); \ } \ - if (!IN_ENTER_EXECUTOR && keep_tracing_bit) { \ + if (keep_tracing_bit) { \ + assert(next_instr->op.code != ENTER_EXECUTOR); \ ENTER_TRACING(); \ _PyJIT_InitializeTracing(tstate, frame, next_instr, STACK_LEVEL(), 0); \ } \ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 013d7e78361698..62cda4cf98fbba 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7551,27 +7551,16 @@ case _GUARD_IP: { PyObject *ip = (PyObject *)CURRENT_OPERAND0(); if (frame->instr_ptr != (_Py_CODEUNIT *)ip) { - #ifdef Py_DEBUG - _Py_CODEUNIT *target = frame->instr_ptr; - if (frame->lltrace >= 2) { - _PyFrame_SetStackPointer(frame, stack_pointer); - printf("GUARD IP EXIT: [UOp "); - _PyUOpPrint(&next_uop[-1]); - printf(", target %d -> %s]\n", - (int)(target - _PyFrame_GetBytecode(frame)), - _PyOpcode_OpName[target->op.code]); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - #endif - GOTO_TIER_ONE(frame->instr_ptr, 1); + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); } break; } case _DYNAMIC_EXIT: { - PyObject *ip = (PyObject *)CURRENT_OPERAND0(); - #ifdef Py_DEBUG + PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); _Py_CODEUNIT *target = frame->instr_ptr; + #ifdef Py_DEBUG if (frame->lltrace >= 2) { _PyFrame_SetStackPointer(frame, stack_pointer); printf("GUARD IP EXIT: [UOp "); @@ -7582,7 +7571,32 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } #endif - GOTO_TIER_ONE(frame->instr_ptr, 1); + _PyExitData *exit = (_PyExitData *)exit_p; + tstate->jit_exit = exit_p; + _Py_BackoffCounter temperature = exit->temperature; + _PyExecutorObject *executor; + if (target->op.code == ENTER_EXECUTOR) { + PyCodeObject *code = _PyFrame_GetCode(frame); + executor = code->co_executors->executors[target->op.arg]; + Py_INCREF(executor); + } + else { + if (!backoff_counter_triggers(temperature)) { + exit->temperature = advance_backoff_counter(temperature); + GOTO_TIER_ONE(frame->instr_ptr, 0); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); + stack_pointer = _PyFrame_GetStackPointer(frame); + assert(tstate->current_executor == (PyObject *)previous_executor); + int chain_depth = previous_executor->vm_data.chain_depth + 1; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), chain_depth); + stack_pointer = _PyFrame_GetStackPointer(frame); + GOTO_TIER_ONE(target, 1); + } + exit->executor = executor; + TIER2_TO_TIER2(exit->executor); break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 9551acb4d64bb9..8201d2c236c1d8 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5818,7 +5818,7 @@ next_instr = this_instr; DISPATCH_GOTO(); } - TIER1_TO_TIER2(executor, 1); + TIER1_TO_TIER2(executor); #else Py_FatalError("ENTER_EXECUTOR is not supported in this build"); #endif /* _Py_TIER2 */ @@ -8147,7 +8147,9 @@ { #ifdef _Py_TIER2 _Py_BackoffCounter counter = this_instr[1].counter; - if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD_JIT) { + if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && + this_instr->op.code == JUMP_BACKWARD_JIT && + next_instr->op.code != ENTER_EXECUTOR) { _Py_CODEUNIT *start = this_instr; int curr_oparg = oparg; while (curr_oparg > 255) { @@ -18960,7 +18962,7 @@ next_instr = this_instr; DISPATCH_GOTO(); } - TIER1_TO_TIER2(executor, 1); + TIER1_TO_TIER2(executor); #else Py_FatalError("ENTER_EXECUTOR is not supported in this build"); #endif /* _Py_TIER2 */ @@ -21363,7 +21365,9 @@ { #ifdef _Py_TIER2 _Py_BackoffCounter counter = this_instr[1].counter; - if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD_JIT) { + if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && + this_instr->op.code == JUMP_BACKWARD_JIT && + next_instr->op.code != ENTER_EXECUTOR) { _Py_CODEUNIT *start = this_instr; int curr_oparg = oparg; while (curr_oparg > 255) { diff --git a/Python/optimizer.c b/Python/optimizer.c index b973fdd7ac4159..f59beda5f8a427 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -842,7 +842,7 @@ count_exits(_PyUOpInstruction *buffer, int length) int exit_count = 0; for (int i = 0; i < length; i++) { int opcode = buffer[i].opcode; - if (opcode == _EXIT_TRACE) { + if (opcode == _EXIT_TRACE || opcode == _DYNAMIC_EXIT) { exit_count++; } } @@ -898,7 +898,7 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) else if (exit_flags & HAS_PERIODIC_FLAG) { exit_op = _HANDLE_PENDING_AND_DEOPT; } - if (opcode == _FOR_ITER_TIER_TWO) { + if (opcode == _FOR_ITER_TIER_TWO || opcode == _GUARD_IP) { exit_op = _DYNAMIC_EXIT; } int32_t jump_target = target; @@ -1053,7 +1053,7 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil int opcode = buffer[i].opcode; dest--; *dest = buffer[i]; - if (opcode == _EXIT_TRACE) { + if (opcode == _EXIT_TRACE || opcode == _DYNAMIC_EXIT) { _PyExitData *exit = &executor->exits[next_exit]; exit->target = buffer[i].target; dest->operand0 = (uint64_t)exit; From 92bba64f1b43582934ff7c94fe7a2e6e444691d2 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 21 Sep 2025 12:29:43 +0100 Subject: [PATCH 013/190] fix JIT + debug builds --- Python/bytecodes.c | 2 +- Python/executor_cases.c.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 61e97cb99230a6..8d3464586e2942 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5465,7 +5465,7 @@ dummy_func( tier2 op(_DYNAMIC_EXIT, (exit_p/4 --)) { _Py_CODEUNIT *target = frame->instr_ptr; -#ifdef Py_DEBUG +#if defined(Py_DEBUG) && !defined(_Py_JIT) if (frame->lltrace >= 2) { printf("GUARD IP EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 62cda4cf98fbba..efe3c6f52464c1 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7560,7 +7560,7 @@ case _DYNAMIC_EXIT: { PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); _Py_CODEUNIT *target = frame->instr_ptr; - #ifdef Py_DEBUG + #if defined(Py_DEBUG) && !defined(_Py_JIT) if (frame->lltrace >= 2) { _PyFrame_SetStackPointer(frame, stack_pointer); printf("GUARD IP EXIT: [UOp "); From 2e3ddc14e3c984a39adb7ce6e32c0ab21e26ba23 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 21 Sep 2025 12:50:06 +0100 Subject: [PATCH 014/190] Fix double-initialization --- Python/bytecodes.c | 4 ++-- Python/ceval_macros.h | 1 - Python/executor_cases.c.h | 3 +-- Python/optimizer.c | 2 +- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 8d3464586e2942..c1912300bda141 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5490,8 +5490,8 @@ dummy_func( } _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); assert(tstate->current_executor == (PyObject *)previous_executor); - int chain_depth = previous_executor->vm_data.chain_depth + 1; - _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), chain_depth); + // Set chain_depth to 0 because we want to keep tracing whatever we see next. + _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), 0); GOTO_TIER_ONE(target, 1); } exit->executor = executor; diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 0e7e5e81a7f9ff..48f08868d0b07a 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -417,7 +417,6 @@ do { \ if (keep_tracing_bit) { \ assert(next_instr->op.code != ENTER_EXECUTOR); \ ENTER_TRACING(); \ - _PyJIT_InitializeTracing(tstate, frame, next_instr, STACK_LEVEL(), 0); \ } \ else { \ LEAVE_TRACING(); \ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index efe3c6f52464c1..a8224df7f55e2b 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7589,9 +7589,8 @@ _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); stack_pointer = _PyFrame_GetStackPointer(frame); assert(tstate->current_executor == (PyObject *)previous_executor); - int chain_depth = previous_executor->vm_data.chain_depth + 1; _PyFrame_SetStackPointer(frame, stack_pointer); - _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), chain_depth); + _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), 0); stack_pointer = _PyFrame_GetStackPointer(frame); GOTO_TIER_ONE(target, 1); } diff --git a/Python/optimizer.c b/Python/optimizer.c index f59beda5f8a427..597bc6d2dc1bc1 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -805,7 +805,7 @@ _PyJIT_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_ lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that } DPRINTF(2, - "Optimizing %s (%s:%d) at byte offset %d\n", + "Tracing %s (%s:%d) at byte offset %d\n", PyUnicode_AsUTF8(code->co_qualname), PyUnicode_AsUTF8(code->co_filename), code->co_firstlineno, From aada168b9c7aded941af49c8cb9d0ad2186f2465 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 21 Sep 2025 19:36:54 +0100 Subject: [PATCH 015/190] fix exception bug --- Include/internal/pycore_optimizer.h | 2 + Include/internal/pycore_uop_metadata.h | 2 +- Python/bytecodes.c | 82 +- Python/ceval.c | 12 +- Python/ceval_macros.h | 16 +- Python/executor_cases.c.h | 63 +- Python/generated_cases.c.h | 1314 ++++++++++++++++---- Python/optimizer.c | 42 +- Tools/cases_generator/analyzer.py | 2 + Tools/cases_generator/generators_common.py | 10 +- Tools/cases_generator/tier1_generator.py | 2 + Tools/cases_generator/tracer_generator.py | 2 +- 12 files changed, 1207 insertions(+), 342 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 4b391d4d4fa926..ac509b890fd31d 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -37,6 +37,7 @@ typedef struct { typedef struct _PyExitData { uint32_t target; uint16_t index; + char is_dynamic; _Py_BackoffCounter temperature; struct _PyExecutorObject *executor; } _PyExitData; @@ -44,6 +45,7 @@ typedef struct _PyExitData { typedef struct _PyExecutorObject { PyObject_VAR_HEAD const _PyUOpInstruction *trace; + _Py_CODEUNIT *expected_entrypoint; _PyVMData vm_data; /* Used by the VM, but opaque to the optimizer */ uint32_t exit_count; uint32_t code_size; diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index a7eebe9ad0d5a0..1b599091f781fd 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -338,7 +338,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_HANDLE_PENDING_AND_DEOPT] = HAS_ESCAPES_FLAG, [_ERROR_POP_N] = HAS_ARG_FLAG, [_TIER2_RESUME_CHECK] = HAS_PERIODIC_FLAG, - [_COLD_EXIT] = HAS_ESCAPES_FLAG, + [_COLD_EXIT] = 0, [_GUARD_IP] = HAS_EXIT_FLAG, [_DYNAMIC_EXIT] = HAS_ESCAPES_FLAG, }; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index c1912300bda141..a3854fbe0cc190 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5414,6 +5414,7 @@ dummy_func( } tier2 op(_ERROR_POP_N, (target/2 --)) { + assert(target != 0); assert(oparg == 0); frame->instr_ptr = _PyFrame_GetBytecode(frame) + target; SYNC_SP(); @@ -5435,28 +5436,35 @@ dummy_func( tier2 op(_COLD_EXIT, ( -- )) { _PyExitData *exit = tstate->jit_exit; assert(exit != NULL); - _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; - _Py_BackoffCounter temperature = exit->temperature; - if (!backoff_counter_triggers(temperature)) { - exit->temperature = advance_backoff_counter(temperature); - GOTO_TIER_ONE(target, 0); - } - _PyExecutorObject *executor; - if (target->op.code == ENTER_EXECUTOR) { - PyCodeObject *code = _PyFrame_GetCode(frame); - executor = code->co_executors->executors[target->op.arg]; - Py_INCREF(executor); - } - else { - _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); - assert(tstate->current_executor == (PyObject *)previous_executor); - int chain_depth = previous_executor->vm_data.chain_depth + 1; - _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), chain_depth); - GOTO_TIER_ONE(target, 1); - } - assert(tstate->jit_exit == exit); - exit->executor = executor; - TIER2_TO_TIER2(exit->executor); + bool is_dynamic = exit->is_dynamic; + _Py_CODEUNIT *target = is_dynamic ? frame->instr_ptr : (_PyFrame_GetBytecode(frame) + exit->target); + GOTO_TIER_ONE(target, 0); + // _Py_BackoffCounter temperature = exit->temperature; + // if (target->op.code == ENTER_EXECUTOR) { + // PyCodeObject *code = _PyFrame_GetCode(frame); + // _PyExecutorObject *executor = code->co_executors->executors[target->op.arg]; + // if (is_dynamic && executor->expected_entrypoint != target) { + // GOTO_TIER_ONE(target, 0); + // } + // Py_INCREF(executor); + // assert(tstate->jit_exit == exit); + // exit->executor = executor; + // TIER2_TO_TIER2(exit->executor); + // } + // else { + // if (!backoff_counter_triggers(temperature)) { + // exit->temperature = advance_backoff_counter(temperature); + // GOTO_TIER_ONE(target, 0); + // } + // if (is_dynamic) { + // GOTO_TIER_ONE(target, 0); + // } + // _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); + // assert(tstate->current_executor == (PyObject *)previous_executor); + // int chain_depth = 0; + // _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), chain_depth); + // GOTO_TIER_ONE(target, 1); + // } } tier2 op(_GUARD_IP, (ip/4 --)) { @@ -5465,36 +5473,20 @@ dummy_func( tier2 op(_DYNAMIC_EXIT, (exit_p/4 --)) { _Py_CODEUNIT *target = frame->instr_ptr; + _PyExitData *exit = (_PyExitData *)exit_p; #if defined(Py_DEBUG) && !defined(_Py_JIT) + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); if (frame->lltrace >= 2) { - printf("GUARD IP EXIT: [UOp "); + printf("DYNAMIC EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); - printf(", target %d -> %s]\n", + printf(", exit %tu, temp %d, target %d -> %s]\n", + exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyFrame_GetBytecode(frame)), _PyOpcode_OpName[target->op.code]); } #endif - _PyExitData *exit = (_PyExitData *)exit_p; - tstate->jit_exit = exit_p; - _Py_BackoffCounter temperature = exit->temperature; - _PyExecutorObject *executor; - if (target->op.code == ENTER_EXECUTOR) { - PyCodeObject *code = _PyFrame_GetCode(frame); - executor = code->co_executors->executors[target->op.arg]; - Py_INCREF(executor); - } - else { - if (!backoff_counter_triggers(temperature)) { - exit->temperature = advance_backoff_counter(temperature); - GOTO_TIER_ONE(frame->instr_ptr, 0); - } - _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); - assert(tstate->current_executor == (PyObject *)previous_executor); - // Set chain_depth to 0 because we want to keep tracing whatever we see next. - _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), 0); - GOTO_TIER_ONE(target, 1); - } - exit->executor = executor; + assert(exit->is_dynamic); + tstate->jit_exit = exit; TIER2_TO_TIER2(exit->executor); } diff --git a/Python/ceval.c b/Python/ceval.c index a7edd45e1ea251..a6219e5c26d670 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -975,12 +975,11 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch) // 1 for trace full, 0 for successful write. static int -add_to_code_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, PyCodeObject *old_code, _Py_CODEUNIT *this_instr, _Py_CODEUNIT *next_instr, int opcode, int oparg, int jump_taken) +add_to_code_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, PyCodeObject *old_code, PyFunctionObject *old_func, _Py_CODEUNIT *this_instr, _Py_CODEUNIT *next_instr, int opcode, int oparg, int jump_taken) { assert(frame != NULL); assert(tstate->interp->jit_tracer_code_curr_size < UOP_MAX_TRACE_LENGTH); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - return !_PyJIT_translate_single_bytecode_to_trace(tstate, frame, this_instr, next_instr, old_code, func, opcode, oparg, jump_taken); + return !_PyJIT_translate_single_bytecode_to_trace(tstate, frame, this_instr, next_instr, old_code, old_func, opcode, oparg, jump_taken); } /* _PyEval_EvalFrameDefault is too large to optimize for speed with PGO on MSVC. @@ -1195,14 +1194,15 @@ _PyTier2Interpreter( for (;;) { uopcode = next_uop->opcode; #ifdef Py_DEBUG - if (frame->lltrace >= 4 && - next_uop->opcode != _YIELD_VALUE && + if (frame->lltrace >= 4) { + if (next_uop->opcode != _YIELD_VALUE && next_uop->opcode != _FOR_ITER_GEN_FRAME && next_uop->opcode != _PUSH_FRAME && next_uop->opcode != _PY_FRAME_KW && next_uop->opcode != _SAVE_RETURN_OFFSET && next_uop->opcode != _SAVE_RETURN_OFFSET) { - dump_stack(frame, stack_pointer); + dump_stack(frame, stack_pointer); + } if (next_uop->opcode == _START_EXECUTOR) { printf("%4d uop: ", 0); } diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 48f08868d0b07a..2f53366915dc72 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -130,14 +130,20 @@ # define LABEL(name) name: #endif +#define TRACING_JUMP_TO_LABEL(label) \ + RECORD_JUMP_TAKEN() \ + RECORD_TRACE() \ + BAIL_TRACING_NO_DISPATCH() \ + JUMP_TO_LABEL(label); + #if _Py_TAIL_CALL_INTERP || USE_COMPUTED_GOTOS # define IS_JIT_TRACING() (DISPATCH_TABLE_VAR == TRACING_DISPATCH_TABLE) # define ENTER_TRACING() DISPATCH_TABLE_VAR = TRACING_DISPATCH_TABLE; # define LEAVE_TRACING() DISPATCH_TABLE_VAR = DISPATCH_TABLE; # define BAIL_TRACING_NO_DISPATCH() \ LEAVE_TRACING(); \ - int err = _PyOptimizer_Optimize(frame, tstate); \ - if (err < 0) { \ + int _err = _PyOptimizer_Optimize(frame, tstate); \ + if (_err < 0) { \ JUMP_TO_LABEL(error); \ } # define BAIL_TRACING() \ @@ -145,13 +151,13 @@ DISPATCH(); # define RECORD_TRACE() do { \ frame->instr_ptr = next_instr; \ - if (add_to_code_trace(tstate, frame, old_code, this_instr, next_instr, opcode, oparg, _jump_taken)) { \ + if (add_to_code_trace(tstate, frame, old_code, old_func, this_instr, next_instr, opcode, oparg, _jump_taken)) { \ BAIL_TRACING(); \ } \ } while (0); # define RECORD_TRACE_NO_DISPATCH() do { \ frame->instr_ptr = next_instr; \ - if (add_to_code_trace(tstate, frame, old_code, this_instr, next_instr, opcode, oparg, _jump_taken)) { \ + if (add_to_code_trace(tstate, frame, old_code, old_func, this_instr, next_instr, opcode, oparg, _jump_taken)) { \ BAIL_TRACING_NO_DISPATCH(); \ } \ } while (0); @@ -415,7 +421,9 @@ do { \ JUMP_TO_LABEL(error); \ } \ if (keep_tracing_bit) { \ + assert(next_instr == frame->instr_ptr); \ assert(next_instr->op.code != ENTER_EXECUTOR); \ + assert(tstate->interp->jit_tracer_code_curr_size == 2); \ ENTER_TRACING(); \ } \ else { \ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index a8224df7f55e2b..0a94baa0557d37 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7493,6 +7493,7 @@ case _ERROR_POP_N: { oparg = CURRENT_OPARG(); uint32_t target = (uint32_t)CURRENT_OPERAND0(); + assert(target != 0); assert(oparg == 0); frame->instr_ptr = _PyFrame_GetBytecode(frame) + target; GOTO_TIER_ONE(NULL, 0); @@ -7519,32 +7520,9 @@ case _COLD_EXIT: { _PyExitData *exit = tstate->jit_exit; assert(exit != NULL); - _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; - _Py_BackoffCounter temperature = exit->temperature; - if (!backoff_counter_triggers(temperature)) { - exit->temperature = advance_backoff_counter(temperature); - GOTO_TIER_ONE(target, 0); - } - _PyExecutorObject *executor; - if (target->op.code == ENTER_EXECUTOR) { - PyCodeObject *code = _PyFrame_GetCode(frame); - executor = code->co_executors->executors[target->op.arg]; - Py_INCREF(executor); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); - stack_pointer = _PyFrame_GetStackPointer(frame); - assert(tstate->current_executor == (PyObject *)previous_executor); - int chain_depth = previous_executor->vm_data.chain_depth + 1; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), chain_depth); - stack_pointer = _PyFrame_GetStackPointer(frame); - GOTO_TIER_ONE(target, 1); - } - assert(tstate->jit_exit == exit); - exit->executor = executor; - TIER2_TO_TIER2(exit->executor); + bool is_dynamic = exit->is_dynamic; + _Py_CODEUNIT *target = is_dynamic ? frame->instr_ptr : (_PyFrame_GetBytecode(frame) + exit->target); + GOTO_TIER_ONE(target, 0); break; } @@ -7560,41 +7538,22 @@ case _DYNAMIC_EXIT: { PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); _Py_CODEUNIT *target = frame->instr_ptr; + _PyExitData *exit = (_PyExitData *)exit_p; #if defined(Py_DEBUG) && !defined(_Py_JIT) + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); if (frame->lltrace >= 2) { _PyFrame_SetStackPointer(frame, stack_pointer); - printf("GUARD IP EXIT: [UOp "); + printf("DYNAMIC EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); - printf(", target %d -> %s]\n", + printf(", exit %tu, temp %d, target %d -> %s]\n", + exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyFrame_GetBytecode(frame)), _PyOpcode_OpName[target->op.code]); stack_pointer = _PyFrame_GetStackPointer(frame); } #endif - _PyExitData *exit = (_PyExitData *)exit_p; - tstate->jit_exit = exit_p; - _Py_BackoffCounter temperature = exit->temperature; - _PyExecutorObject *executor; - if (target->op.code == ENTER_EXECUTOR) { - PyCodeObject *code = _PyFrame_GetCode(frame); - executor = code->co_executors->executors[target->op.arg]; - Py_INCREF(executor); - } - else { - if (!backoff_counter_triggers(temperature)) { - exit->temperature = advance_backoff_counter(temperature); - GOTO_TIER_ONE(frame->instr_ptr, 0); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); - stack_pointer = _PyFrame_GetStackPointer(frame); - assert(tstate->current_executor == (PyObject *)previous_executor); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), 0); - stack_pointer = _PyFrame_GetStackPointer(frame); - GOTO_TIER_ONE(target, 1); - } - exit->executor = executor; + assert(exit->is_dynamic); + tstate->jit_exit = exit; TIER2_TO_TIER2(exit->executor); break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 8201d2c236c1d8..78a458b4899eff 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -25,6 +25,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -92,6 +94,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -154,6 +158,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -218,6 +224,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -282,6 +290,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -350,6 +360,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -437,6 +449,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -499,6 +513,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -563,6 +579,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -633,6 +651,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -721,6 +741,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -815,6 +837,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -891,6 +915,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -973,6 +999,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -1050,6 +1078,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -1112,6 +1142,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -1176,6 +1208,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -1236,6 +1270,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -1298,6 +1334,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -1326,6 +1364,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -1382,6 +1422,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -1443,6 +1485,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -1482,6 +1526,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -1533,6 +1579,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -1575,6 +1623,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -1601,6 +1651,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -1618,6 +1670,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -1800,6 +1854,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -1925,6 +1981,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -2072,6 +2130,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -2204,6 +2264,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -2313,6 +2375,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -2426,6 +2490,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -2539,6 +2605,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -2627,6 +2695,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -2798,6 +2868,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -2832,6 +2904,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -2875,6 +2949,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -2955,6 +3031,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -3141,6 +3219,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -3275,6 +3355,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -3406,6 +3488,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -3520,6 +3604,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -3596,6 +3682,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -3688,6 +3776,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -3813,6 +3903,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -3940,6 +4032,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -4037,6 +4131,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -4147,6 +4243,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -4266,6 +4364,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -4383,6 +4483,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -4487,6 +4589,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -4563,6 +4667,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -4639,6 +4745,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -4698,6 +4806,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -4773,6 +4883,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -4814,6 +4926,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -4877,6 +4991,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -4954,6 +5070,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -5013,6 +5131,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -5076,6 +5196,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -5139,6 +5261,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -5203,6 +5327,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -5265,6 +5391,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -5327,6 +5455,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -5363,6 +5493,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -5385,6 +5517,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -5410,6 +5544,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -5439,6 +5575,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -5465,6 +5603,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -5495,6 +5635,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -5524,6 +5666,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -5560,6 +5704,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -5596,6 +5742,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -5639,6 +5787,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -5686,6 +5836,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -5735,6 +5887,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; next_instr += 1; @@ -5756,6 +5910,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -5783,6 +5939,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -5832,6 +5990,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -5859,6 +6019,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -5879,6 +6041,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -5919,6 +6083,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -5959,6 +6125,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -6017,6 +6185,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -6092,6 +6262,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -6180,6 +6352,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -6251,6 +6425,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -6311,6 +6487,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -6376,6 +6554,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -6404,6 +6584,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -6437,6 +6619,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -6487,6 +6671,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -6519,6 +6705,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -6568,6 +6756,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -6597,6 +6787,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -6640,6 +6832,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -6832,6 +7026,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -7003,6 +7199,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -7193,6 +7391,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -7250,6 +7450,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -7283,6 +7485,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -7321,6 +7525,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -7361,6 +7567,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -7392,6 +7600,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -7423,6 +7633,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -7441,6 +7653,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const prev_instr = frame->instr_ptr; @@ -7485,6 +7699,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -7609,6 +7825,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const prev_instr = frame->instr_ptr; @@ -7629,6 +7847,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const prev_instr = frame->instr_ptr; @@ -7658,6 +7878,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -7686,6 +7908,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -7721,6 +7945,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -7754,6 +7980,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -7782,6 +8010,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -7866,6 +8096,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -7922,6 +8154,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -7997,6 +8231,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -8036,6 +8272,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -8073,6 +8311,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -8118,6 +8358,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -8164,9 +8406,7 @@ DISPATCH(); } } - _PyFrame_SetStackPointer(frame, stack_pointer); _PyJIT_InitializeTracing(tstate, frame, next_instr, STACK_LEVEL(), 0); - stack_pointer = _PyFrame_GetStackPointer(frame); ENTER_TRACING(); DISPATCH(); } @@ -8185,6 +8425,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -8204,6 +8446,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -8237,6 +8481,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -8253,6 +8499,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -8279,6 +8527,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -8330,6 +8580,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -8415,6 +8667,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -8477,6 +8731,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -8549,6 +8805,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -8611,6 +8869,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -8695,6 +8955,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -8755,6 +9017,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -8805,6 +9069,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -8876,6 +9142,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -8954,6 +9222,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -9005,6 +9275,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -9077,6 +9349,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -9175,6 +9449,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -9248,6 +9524,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -9362,6 +9640,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -9396,6 +9676,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -9417,6 +9699,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -9438,6 +9722,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -9470,6 +9756,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -9491,6 +9779,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -9512,6 +9802,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -9533,6 +9825,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -9558,6 +9852,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -9588,6 +9884,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -9613,6 +9911,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -9662,6 +9962,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -9741,6 +10043,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -9801,6 +10105,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -9885,6 +10191,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -9956,6 +10264,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -9984,6 +10294,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -10011,6 +10323,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -10033,6 +10347,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -10083,6 +10399,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -10224,6 +10542,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -10290,6 +10610,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -10372,6 +10694,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -10397,6 +10721,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -10434,6 +10760,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -10469,6 +10797,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -10525,6 +10855,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -10556,6 +10888,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -10579,6 +10913,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -10602,6 +10938,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -10617,6 +10955,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -10632,6 +10972,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -10657,6 +10999,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -10682,6 +11026,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -10708,6 +11054,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -10755,6 +11103,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -10802,6 +11152,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -10828,6 +11180,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -10850,6 +11204,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -10883,6 +11239,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -10903,6 +11261,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -10936,6 +11296,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -10968,6 +11330,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -10985,6 +11349,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -11065,6 +11431,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -11107,6 +11475,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -11153,6 +11523,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -11191,6 +11563,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -11300,6 +11674,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -11373,6 +11749,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -11423,6 +11801,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -11451,6 +11831,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -11483,6 +11865,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -11514,6 +11898,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -11575,6 +11961,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -11655,6 +12043,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -11710,6 +12100,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -11802,6 +12194,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -11825,6 +12219,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -11849,6 +12245,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -11876,6 +12274,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -11911,6 +12311,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -11940,6 +12342,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -11990,6 +12394,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -12054,6 +12460,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -12117,6 +12525,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -12172,6 +12582,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -12257,6 +12669,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -12281,6 +12695,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -12337,6 +12753,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -12384,6 +12802,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -12412,6 +12832,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -12456,6 +12878,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -12503,6 +12927,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -12534,6 +12960,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -12586,6 +13014,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -12619,6 +13049,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -12652,6 +13084,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -12674,6 +13108,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -12705,6 +13141,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -12760,6 +13198,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -12823,6 +13263,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -12877,6 +13319,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -12932,6 +13376,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -12981,6 +13427,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -13037,6 +13485,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -13078,7 +13528,7 @@ PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -13105,6 +13555,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -13151,7 +13603,7 @@ ((PyFloatObject *)right_o)->ob_fval; res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); if (PyStackRef_IsNull(res)) { - JUMP_TO_LABEL(pop_2_error); + TRACING_JUMP_TO_LABEL(pop_2_error); } } stack_pointer[-2] = res; @@ -13168,6 +13620,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -13233,6 +13687,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -13280,7 +13736,7 @@ PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); if (res_o == NULL) { - JUMP_TO_LABEL(pop_2_error); + TRACING_JUMP_TO_LABEL(pop_2_error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -13298,6 +13754,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -13367,6 +13825,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -13436,7 +13896,7 @@ Py_DECREF(right_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (PyStackRef_IsNull(*target_local)) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } #if TIER_ONE @@ -13455,6 +13915,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -13501,7 +13963,7 @@ ((PyFloatObject *)right_o)->ob_fval; res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); if (PyStackRef_IsNull(res)) { - JUMP_TO_LABEL(pop_2_error); + TRACING_JUMP_TO_LABEL(pop_2_error); } } stack_pointer[-2] = res; @@ -13518,6 +13980,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -13583,6 +14047,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -13636,7 +14102,7 @@ stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); if (rc <= 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -13654,6 +14120,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -13743,6 +14211,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -13838,6 +14308,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -13897,7 +14369,7 @@ stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -13915,6 +14387,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -13998,6 +14472,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -14076,6 +14552,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -14122,7 +14600,7 @@ ((PyFloatObject *)right_o)->ob_fval; res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); if (PyStackRef_IsNull(res)) { - JUMP_TO_LABEL(pop_2_error); + TRACING_JUMP_TO_LABEL(pop_2_error); } } stack_pointer[-2] = res; @@ -14139,6 +14617,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -14204,6 +14684,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -14249,7 +14731,7 @@ PyStackRef_CLOSE(container); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -14267,6 +14749,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -14315,7 +14799,7 @@ PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); if (interpolation_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } interpolation = PyStackRef_FromPyObjectSteal(interpolation_o); stack_pointer[0] = interpolation; @@ -14332,6 +14816,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -14346,7 +14832,7 @@ PyObject *list_o = _PyList_FromStackRefStealOnSuccess(values, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); if (list_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } list = PyStackRef_FromPyObjectStealMortal(list_o); stack_pointer[-oparg] = list; @@ -14363,6 +14849,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -14385,7 +14873,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -oparg*2; assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *map_o = _PyDict_FromItems( @@ -14405,7 +14893,7 @@ stack_pointer += -oparg*2; assert(WITHIN_STACK_BOUNDS()); if (map_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } map = PyStackRef_FromPyObjectStealMortal(map_o); stack_pointer[0] = map; @@ -14422,6 +14910,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -14446,7 +14936,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } int err = 0; for (Py_ssize_t i = 0; i < oparg; i++) { @@ -14469,7 +14959,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(set_o); stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } set = PyStackRef_FromPyObjectStealMortal(set_o); stack_pointer[-oparg] = set; @@ -14486,6 +14976,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -14511,7 +15003,7 @@ stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); if (slice_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } slice = PyStackRef_FromPyObjectStealMortal(slice_o); stack_pointer[0] = slice; @@ -14528,6 +15020,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -14550,7 +15044,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); @@ -14565,7 +15059,7 @@ stack_pointer += -oparg; assert(WITHIN_STACK_BOUNDS()); if (str_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } str = PyStackRef_FromPyObjectSteal(str_o); stack_pointer[0] = str; @@ -14582,6 +15076,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -14610,7 +15106,7 @@ PyStackRef_CLOSE(strings); stack_pointer = _PyFrame_GetStackPointer(frame); if (template_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } template = PyStackRef_FromPyObjectSteal(template_o); stack_pointer[0] = template; @@ -14627,6 +15123,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -14639,7 +15137,7 @@ values = &stack_pointer[-oparg]; PyObject *tup_o = _PyTuple_FromStackRefStealOnSuccess(values, oparg); if (tup_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } tup = PyStackRef_FromPyObjectStealMortal(tup_o); stack_pointer[-oparg] = tup; @@ -14656,6 +15154,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -14676,6 +15176,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -14751,7 +15253,7 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (new_frame == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); @@ -14778,7 +15280,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } stack_pointer[-2 - oparg] = callable; stack_pointer[-1 - oparg] = self_or_null; @@ -14832,7 +15334,7 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -14845,7 +15347,7 @@ int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } } TRACING_DISPATCH(); @@ -14859,6 +15361,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -14921,7 +15425,7 @@ PyObject *self_o = PyType_GenericAlloc(tp, 0); stack_pointer = _PyFrame_GetStackPointer(frame); if (self_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } self_or_null = PyStackRef_FromPyObjectSteal(self_o); _PyStackRef temp = callable; @@ -14954,7 +15458,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FrameClearAndPop(tstate, shim); stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; tstate->py_recursion_remaining--; @@ -14985,6 +15489,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -15133,6 +15639,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -15229,7 +15737,7 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (temp == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } new_frame = PyStackRef_Wrap(temp); } @@ -15266,6 +15774,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -15324,7 +15834,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); @@ -15349,7 +15859,7 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -15362,7 +15872,7 @@ int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } } TRACING_DISPATCH(); @@ -15376,6 +15886,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -15434,7 +15946,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = _PyCFunctionFast_CAST(cfunc)( @@ -15463,7 +15975,7 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -15476,7 +15988,7 @@ int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } } TRACING_DISPATCH(); @@ -15490,6 +16002,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -15551,7 +16065,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); @@ -15577,7 +16091,7 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -15590,7 +16104,7 @@ int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } } TRACING_DISPATCH(); @@ -15604,6 +16118,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -15666,7 +16182,7 @@ PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -15679,7 +16195,7 @@ int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } } TRACING_DISPATCH(); @@ -15693,6 +16209,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -15718,13 +16236,13 @@ int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *tuple_o = PySequence_Tuple(callargs_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (tuple_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } _PyStackRef temp = callargs; callargs = PyStackRef_FromPyObjectSteal(tuple_o); @@ -15759,7 +16277,7 @@ frame, this_instr, func, arg); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); result_o = PyObject_Call(func, callargs, kwargs); @@ -15807,7 +16325,7 @@ stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); if (new_frame == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } assert( 1u == 1); frame->return_offset = 1; @@ -15838,7 +16356,7 @@ PyStackRef_CLOSE(func_st); stack_pointer = _PyFrame_GetStackPointer(frame); if (result_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } result = PyStackRef_FromPyObjectSteal(result_o); } @@ -15851,7 +16369,7 @@ int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } } TRACING_DISPATCH(); @@ -15865,6 +16383,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -15885,7 +16405,7 @@ PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; @@ -15902,6 +16422,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -15931,7 +16453,7 @@ stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; @@ -15948,6 +16470,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -15994,7 +16518,7 @@ int retval = PyObject_IsInstance(inst_o, cls_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (retval < 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } (void)null; stack_pointer += -1; @@ -16029,6 +16553,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -16111,7 +16637,7 @@ PyStackRef_CLOSE(kwnames); stack_pointer = _PyFrame_GetStackPointer(frame); if (new_frame == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); frame->return_offset = 4u ; @@ -16142,7 +16668,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } stack_pointer[-3 - oparg] = callable; stack_pointer[-2 - oparg] = self_or_null; @@ -16198,7 +16724,7 @@ stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -16216,6 +16742,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -16314,7 +16842,7 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (temp == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } new_frame = PyStackRef_Wrap(temp); } @@ -16351,6 +16879,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -16420,7 +16950,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); @@ -16456,7 +16986,7 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -16469,7 +16999,7 @@ int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } } TRACING_DISPATCH(); @@ -16483,6 +17013,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -16561,7 +17093,7 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (temp == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } new_frame = PyStackRef_Wrap(temp); } @@ -16598,6 +17130,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -16642,12 +17176,12 @@ Py_ssize_t len_i = PyObject_Length(arg_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (len_i < 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } PyObject *res_o = PyLong_FromSsize_t(len_i); assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); if (res_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -16675,6 +17209,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -16749,7 +17285,7 @@ PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } #if TIER_ONE @@ -16768,6 +17304,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -16840,7 +17378,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyCFunctionFast cfunc = _PyCFunctionFast_CAST(meth->ml_meth); @@ -16867,7 +17405,7 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -16880,7 +17418,7 @@ int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } } TRACING_DISPATCH(); @@ -16894,6 +17432,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -16967,7 +17507,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyCFunctionFastWithKeywords cfunc = @@ -16995,7 +17535,7 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -17008,7 +17548,7 @@ int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } } TRACING_DISPATCH(); @@ -17022,6 +17562,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -17093,7 +17635,7 @@ PyStackRef_CLOSE(callable); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -17106,7 +17648,7 @@ int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } } TRACING_DISPATCH(); @@ -17120,6 +17662,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -17204,7 +17748,7 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -17217,7 +17761,7 @@ int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } } TRACING_DISPATCH(); @@ -17231,6 +17775,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -17295,7 +17841,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Vectorcall( @@ -17324,7 +17870,7 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -17337,7 +17883,7 @@ int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } } TRACING_DISPATCH(); @@ -17351,6 +17897,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -17469,6 +18017,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -17537,7 +18087,7 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (temp == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } new_frame = PyStackRef_Wrap(temp); } @@ -17574,6 +18124,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -17624,7 +18176,7 @@ PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -17637,7 +18189,7 @@ int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } } TRACING_DISPATCH(); @@ -17651,6 +18203,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -17701,7 +18255,7 @@ PyStackRef_CLOSE(arg); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -17714,7 +18268,7 @@ int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } } TRACING_DISPATCH(); @@ -17728,6 +18282,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -17788,6 +18344,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -17819,7 +18377,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } PyObject *match_o = NULL; PyObject *rest_o = NULL; @@ -17838,11 +18396,11 @@ stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); if (res < 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } assert((match_o == NULL) == (rest_o == NULL)); if (match_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } if (!Py_IsNone(match_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -17866,6 +18424,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -17885,7 +18445,7 @@ int err = _PyEval_CheckExceptTypeValid(tstate, right_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyErr_GivenExceptionMatches(left_o, right_o); @@ -17910,6 +18470,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -17957,7 +18519,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); monitor_reraise(tstate, frame, this_instr); - JUMP_TO_LABEL(exception_unwind); + TRACING_JUMP_TO_LABEL(exception_unwind); } stack_pointer[0] = none; stack_pointer[1] = value; @@ -17974,6 +18536,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -18022,7 +18586,7 @@ stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } if (oparg & 16) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -18030,7 +18594,7 @@ Py_DECREF(res_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_bool < 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } res = res_bool ? PyStackRef_True : PyStackRef_False; } @@ -18052,6 +18616,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -18112,6 +18678,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -18176,6 +18744,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -18240,6 +18810,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -18287,7 +18859,7 @@ stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); if (res < 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; } @@ -18305,6 +18877,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -18350,7 +18924,7 @@ stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); if (res < 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; } @@ -18368,6 +18942,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -18413,7 +18989,7 @@ stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); if (res < 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; } @@ -18431,6 +19007,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -18453,7 +19031,7 @@ PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); if (result_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } result = PyStackRef_FromPyObjectSteal(result_o); stack_pointer[0] = result; @@ -18470,6 +19048,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -18495,6 +19075,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -18523,6 +19105,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -18542,7 +19126,7 @@ PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } TRACING_DISPATCH(); } @@ -18555,6 +19139,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -18568,7 +19154,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(oldobj); @@ -18584,6 +19170,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -18599,7 +19187,7 @@ PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = PyStackRef_NULL; @@ -18617,6 +19205,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -18629,14 +19219,14 @@ int err = PyDict_Pop(GLOBALS(), name, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } if (err == 0) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, NAME_ERROR_MSG, name); stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } TRACING_DISPATCH(); } @@ -18649,6 +19239,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -18664,7 +19256,7 @@ _PyErr_Format(tstate, PyExc_SystemError, "no locals when deleting %R", name); stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_DelItem(ns, name); @@ -18675,7 +19267,7 @@ NAME_ERROR_MSG, name); stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } TRACING_DISPATCH(); } @@ -18688,6 +19280,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -18714,7 +19308,7 @@ stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); if (err) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } TRACING_DISPATCH(); } @@ -18727,6 +19321,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -18755,7 +19351,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(update); stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -18773,6 +19369,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -18805,7 +19403,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(update); stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -18823,6 +19421,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -18860,7 +19460,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetRaisedException(tstate, exc); monitor_reraise(tstate, frame, this_instr); - JUMP_TO_LABEL(exception_unwind); + TRACING_JUMP_TO_LABEL(exception_unwind); } TRACING_DISPATCH(); } @@ -18873,6 +19473,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -18897,6 +19499,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -18927,6 +19531,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -18977,6 +19583,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -18992,7 +19600,7 @@ "__init__() should return None, not '%.200s'", Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -19007,6 +19615,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -19030,6 +19640,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -19051,7 +19663,7 @@ PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -19073,6 +19685,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -19099,7 +19713,7 @@ stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; @@ -19116,6 +19730,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -19152,7 +19768,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (!PyStackRef_IsValid(item)) { if (PyStackRef_IsError(item)) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } JUMPBY(oparg + 1); RECORD_JUMP_TAKEN(); @@ -19176,6 +19792,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -19252,6 +19870,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -19342,6 +19962,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -19397,7 +20019,7 @@ r->len--; PyObject *res = PyLong_FromLong(value); if (res == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } next = PyStackRef_FromPyObjectSteal(res); } @@ -19415,6 +20037,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -19477,6 +20101,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -19506,7 +20132,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(obj); stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); iter_o = (*getter)(obj_o); @@ -19517,7 +20143,7 @@ PyStackRef_CLOSE(obj); stack_pointer = _PyFrame_GetStackPointer(frame); if (iter_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } if (Py_TYPE(iter_o)->tp_as_async == NULL || Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { @@ -19528,7 +20154,7 @@ Py_TYPE(iter_o)->tp_name); Py_DECREF(iter_o); stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[0] = iter; @@ -19545,6 +20171,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -19559,7 +20187,7 @@ PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); stack_pointer = _PyFrame_GetStackPointer(frame); if (awaitable_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); stack_pointer[0] = awaitable; @@ -19576,6 +20204,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -19595,7 +20225,7 @@ PyStackRef_CLOSE(iterable); stack_pointer = _PyFrame_GetStackPointer(frame); if (iter_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } iter = PyStackRef_FromPyObjectSteal(iter_o); stack_pointer[0] = iter; @@ -19612,6 +20242,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -19644,7 +20276,7 @@ PyStackRef_CLOSE(iterable); stack_pointer = _PyFrame_GetStackPointer(frame); if (iter_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } iter = PyStackRef_FromPyObjectSteal(iter_o); index_or_null = PyStackRef_NULL; @@ -19665,6 +20297,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -19679,11 +20313,11 @@ Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); stack_pointer = _PyFrame_GetStackPointer(frame); if (len_i < 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } PyObject *len_o = PyLong_FromSsize_t(len_i); if (len_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } len = PyStackRef_FromPyObjectSteal(len_o); stack_pointer[0] = len; @@ -19700,6 +20334,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -19718,7 +20354,7 @@ "cannot 'yield from' a coroutine object " "in a non-coroutine generator"); stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } iter = iterable; } @@ -19730,7 +20366,7 @@ PyObject *iter_o = PyObject_GetIter(iterable_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (iter_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } iter = PyStackRef_FromPyObjectSteal(iter_o); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -19752,6 +20388,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -19767,7 +20405,7 @@ PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; @@ -19784,6 +20422,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -19813,7 +20453,7 @@ stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; @@ -19830,6 +20470,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -19889,7 +20531,7 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } } // _DO_CALL @@ -19919,7 +20561,7 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (new_frame == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); @@ -19944,7 +20586,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Vectorcall( @@ -19996,7 +20638,7 @@ stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -20009,7 +20651,7 @@ int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } } TRACING_DISPATCH(); @@ -20023,6 +20665,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -20048,13 +20692,13 @@ int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *tuple_o = PySequence_Tuple(callargs_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (tuple_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } _PyStackRef temp = callargs; callargs = PyStackRef_FromPyObjectSteal(tuple_o); @@ -20089,7 +20733,7 @@ frame, this_instr, func, arg); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); result_o = PyObject_Call(func, callargs, kwargs); @@ -20137,7 +20781,7 @@ stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); if (new_frame == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } assert( 1u == 1); frame->return_offset = 1; @@ -20168,7 +20812,7 @@ PyStackRef_CLOSE(func_st); stack_pointer = _PyFrame_GetStackPointer(frame); if (result_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } result = PyStackRef_FromPyObjectSteal(result_o); } @@ -20181,7 +20825,7 @@ int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } } TRACING_DISPATCH(); @@ -20195,6 +20839,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -20251,7 +20897,7 @@ frame, this_instr, function, arg); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } } // _DO_CALL_KW @@ -20285,7 +20931,7 @@ PyStackRef_CLOSE(kwnames); stack_pointer = _PyFrame_GetStackPointer(frame); if (new_frame == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); frame->return_offset = 4u ; @@ -20314,7 +20960,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); PyObject *res_o = PyObject_Vectorcall( @@ -20368,7 +21014,7 @@ stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); if (res_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); } @@ -20386,6 +21032,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -20430,7 +21078,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetRaisedException(tstate, exc); monitor_reraise(tstate, frame, this_instr); - JUMP_TO_LABEL(exception_unwind); + TRACING_JUMP_TO_LABEL(exception_unwind); } } TRACING_DISPATCH(); @@ -20444,6 +21092,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -20459,7 +21109,7 @@ int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } } stack_pointer += -1; @@ -20478,6 +21128,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -20496,7 +21148,7 @@ int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } } val = value; @@ -20517,6 +21169,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -20535,7 +21189,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (!PyStackRef_IsValid(item)) { if (PyStackRef_IsError(item)) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } JUMPBY(oparg + 1); stack_pointer[-1] = null_or_index; @@ -20558,6 +21212,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -20571,7 +21227,7 @@ tstate, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); if (next_opcode < 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } next_instr = this_instr; if (_PyOpcode_Caches[next_opcode]) { @@ -20590,6 +21246,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -20604,7 +21262,7 @@ int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } } // _MONITOR_JUMP_BACKWARD @@ -20622,6 +21280,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -20641,6 +21301,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const prev_instr = frame->instr_ptr; @@ -20663,7 +21325,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); if (original_opcode < 0) { next_instr = this_instr+1; - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } next_instr = frame->instr_ptr; if (next_instr != this_instr) { @@ -20686,6 +21348,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -20732,7 +21396,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } } PyObject *stack[] = {class, self}; @@ -20778,7 +21442,7 @@ stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); if (super == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -20786,7 +21450,7 @@ Py_DECREF(super); stack_pointer = _PyFrame_GetStackPointer(frame); if (attr_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } attr = PyStackRef_FromPyObjectSteal(attr_o); } @@ -20811,6 +21475,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const prev_instr = frame->instr_ptr; @@ -20832,6 +21498,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const prev_instr = frame->instr_ptr; @@ -20862,6 +21530,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -20891,6 +21561,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -20927,6 +21599,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -20961,6 +21635,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -20990,6 +21666,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -21007,7 +21685,7 @@ _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); stack_pointer = _PyFrame_GetStackPointer(frame); if (bytecode == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; @@ -21033,7 +21711,7 @@ int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } next_instr = this_instr; TRACING_DISPATCH(); @@ -21047,7 +21725,7 @@ int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } } } @@ -21058,7 +21736,7 @@ tstate, oparg > 0, frame, this_instr); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } if (frame->instr_ptr != this_instr) { next_instr = frame->instr_ptr; @@ -21075,6 +21753,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -21094,7 +21774,7 @@ frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } } // _RETURN_VALUE @@ -21132,6 +21812,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -21151,7 +21833,7 @@ frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } if (frame->instr_ptr != this_instr) { next_instr = frame->instr_ptr; @@ -21208,6 +21890,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -21250,6 +21934,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -21290,6 +21976,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -21315,7 +22003,7 @@ int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } } // _JUMP_BACKWARD_NO_INTERRUPT @@ -21336,6 +22024,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -21351,7 +22041,7 @@ int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } } // _JUMP_BACKWARD_NO_INTERRUPT @@ -21382,9 +22072,7 @@ TRACING_DISPATCH(); } } - _PyFrame_SetStackPointer(frame, stack_pointer); _PyJIT_InitializeTracing(tstate, frame, next_instr, STACK_LEVEL(), 0); - stack_pointer = _PyFrame_GetStackPointer(frame); ENTER_TRACING(); TRACING_DISPATCH(); } @@ -21404,6 +22092,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -21426,6 +22116,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -21441,7 +22133,7 @@ int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } } // _JUMP_BACKWARD_NO_INTERRUPT @@ -21462,6 +22154,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -21481,6 +22175,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -21495,7 +22191,7 @@ int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), PyStackRef_AsPyObjectSteal(v)); if (err < 0) { - JUMP_TO_LABEL(pop_1_error); + TRACING_JUMP_TO_LABEL(pop_1_error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -21510,6 +22206,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -21545,7 +22243,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(iterable_st); stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } assert(Py_IsNone(none_val)); stack_pointer += -1; @@ -21564,6 +22262,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -21615,7 +22315,7 @@ PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); if (PyStackRef_IsNull(*attr)) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } self_or_null[0] = PyStackRef_NULL; stack_pointer += 1; @@ -21631,7 +22331,7 @@ PyStackRef_CLOSE(owner); stack_pointer = _PyFrame_GetStackPointer(frame); if (attr_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } *attr = PyStackRef_FromPyObjectSteal(attr_o); stack_pointer += 1; @@ -21650,6 +22350,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -21713,6 +22415,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -21786,6 +22490,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -21849,6 +22555,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -21934,6 +22642,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -21995,6 +22705,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -22046,6 +22758,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -22118,6 +22832,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -22197,6 +22913,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -22249,6 +22967,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -22322,6 +23042,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -22421,6 +23143,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -22495,6 +23219,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -22610,6 +23336,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -22623,14 +23351,14 @@ int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } if (bc_o == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetString(tstate, PyExc_NameError, "__build_class__ not found"); stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } bc = PyStackRef_FromPyObjectSteal(bc_o); stack_pointer[0] = bc; @@ -22647,6 +23375,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -22671,6 +23401,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -22695,6 +23427,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -22714,7 +23448,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } stack_pointer[0] = value; stack_pointer += 1; @@ -22730,6 +23464,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -22754,6 +23490,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -22778,6 +23516,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -22802,6 +23542,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -22830,6 +23572,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -22846,7 +23590,7 @@ PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) ); stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } value = PyStackRef_DUP(value_s); stack_pointer[0] = value; @@ -22863,6 +23607,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -22891,6 +23637,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -22911,7 +23659,7 @@ int err = PyMapping_GetOptionalItem(class_dict, name, &value_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } if (!value_o) { PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); @@ -22920,7 +23668,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } } stack_pointer += -1; @@ -22943,6 +23691,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -22964,7 +23714,7 @@ PyStackRef_CLOSE(mod_or_class_dict); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } if (v_o == NULL) { if (PyDict_CheckExact(GLOBALS()) @@ -22982,7 +23732,7 @@ NAME_ERROR_MSG, name); stack_pointer = _PyFrame_GetStackPointer(frame); } - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } } else { @@ -22990,14 +23740,14 @@ int err = PyMapping_GetOptionalItem(GLOBALS(), name, &v_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } if (v_o == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyMapping_GetOptionalItem(BUILTINS(), name, &v_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } if (v_o == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -23005,7 +23755,7 @@ tstate, PyExc_NameError, NAME_ERROR_MSG, name); stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } } } @@ -23025,6 +23775,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -23063,7 +23815,7 @@ _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); stack_pointer = _PyFrame_GetStackPointer(frame); if (PyStackRef_IsNull(*res)) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } } // _PUSH_NULL_CONDITIONAL @@ -23086,6 +23838,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -23171,6 +23925,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -23243,6 +23999,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -23257,7 +24015,7 @@ _PyErr_SetString(tstate, PyExc_SystemError, "no locals found"); stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } locals = PyStackRef_FromPyObjectNew(l); stack_pointer[0] = locals; @@ -23274,6 +24032,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -23287,7 +24047,7 @@ PyObject *v_o = _PyEval_LoadName(tstate, frame, name); stack_pointer = _PyFrame_GetStackPointer(frame); if (v_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } v = PyStackRef_FromPyObjectSteal(v_o); stack_pointer[0] = v; @@ -23304,6 +24064,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -23329,6 +24091,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -23368,7 +24132,7 @@ _PyErr_Format(tstate, PyExc_TypeError, errfmt, owner); stack_pointer = _PyFrame_GetStackPointer(frame); } - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } } TRACING_DISPATCH(); @@ -23382,6 +24146,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -23445,7 +24211,7 @@ stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } } PyObject *stack[] = {class, self}; @@ -23491,7 +24257,7 @@ stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); if (super == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -23499,7 +24265,7 @@ Py_DECREF(super); stack_pointer = _PyFrame_GetStackPointer(frame); if (attr_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } attr = PyStackRef_FromPyObjectSteal(attr_o); } @@ -23524,6 +24290,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -23574,7 +24342,7 @@ stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); if (attr == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } attr_st = PyStackRef_FromPyObjectSteal(attr); stack_pointer[0] = attr_st; @@ -23591,6 +24359,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -23631,7 +24401,7 @@ Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); stack_pointer = _PyFrame_GetStackPointer(frame); if (attr_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } if (method_found) { self_or_null = self_st; @@ -23674,6 +24444,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -23684,7 +24456,7 @@ PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); PyObject *cell = PyCell_New(initial); if (cell == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } _PyStackRef tmp = GETLOCAL(oparg); GETLOCAL(oparg) = PyStackRef_FromPyObjectSteal(cell); @@ -23702,6 +24474,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -23723,7 +24497,7 @@ PyStackRef_CLOSE(codeobj_st); stack_pointer = _PyFrame_GetStackPointer(frame); if (func_obj == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } _PyFunction_SetVersion( func_obj, ((PyCodeObject *)codeobj)->co_version); @@ -23742,6 +24516,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -23765,7 +24541,7 @@ ); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - JUMP_TO_LABEL(pop_2_error); + TRACING_JUMP_TO_LABEL(pop_2_error); } stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -23780,6 +24556,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -23821,7 +24599,7 @@ } else { if (_PyErr_Occurred(tstate)) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } attrs = PyStackRef_None; } @@ -23839,6 +24617,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -23856,7 +24636,7 @@ PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); stack_pointer = _PyFrame_GetStackPointer(frame); if (values_or_none_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); stack_pointer[0] = values_or_none; @@ -23873,6 +24653,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -23899,6 +24681,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -23925,6 +24709,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -23943,6 +24729,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -23961,6 +24749,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -23989,6 +24779,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -24017,6 +24809,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -24044,6 +24838,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -24092,6 +24888,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -24140,6 +24938,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -24167,6 +24967,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -24192,6 +24994,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -24228,6 +25032,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -24251,6 +25057,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -24272,9 +25080,9 @@ assert(oparg == 0); _PyFrame_SetStackPointer(frame, stack_pointer); monitor_reraise(tstate, frame, this_instr); - JUMP_TO_LABEL(exception_unwind); + TRACING_JUMP_TO_LABEL(exception_unwind); } - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } TRACING_TARGET(RERAISE) { @@ -24285,6 +25093,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -24307,7 +25117,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyErr_SetRaisedException(tstate, exc); monitor_reraise(tstate, frame, this_instr); - JUMP_TO_LABEL(exception_unwind); + TRACING_JUMP_TO_LABEL(exception_unwind); } TRACING_TARGET(RESERVED) { @@ -24318,6 +25128,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -24338,6 +25150,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -24356,7 +25170,7 @@ _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); stack_pointer = _PyFrame_GetStackPointer(frame); if (bytecode == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; @@ -24382,7 +25196,7 @@ int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } next_instr = this_instr; TRACING_DISPATCH(); @@ -24404,7 +25218,7 @@ int err = check_periodics(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); if (err != 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } } } @@ -24419,6 +25233,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -24462,6 +25278,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -24476,7 +25294,7 @@ PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); stack_pointer = _PyFrame_GetStackPointer(frame); if (gen == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } assert(STACK_LEVEL() == 0); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -24511,6 +25329,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -24552,6 +25372,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -24638,7 +25460,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(v); stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } } stack_pointer += -1; @@ -24662,6 +25484,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -24736,6 +25560,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -24749,20 +25575,20 @@ _PyErr_Format(tstate, PyExc_SystemError, "no locals found when setting up annotations"); stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } if (ann_dict == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); ann_dict = PyDict_New(); stack_pointer = _PyFrame_GetStackPointer(frame); if (ann_dict == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } _PyFrame_SetStackPointer(frame, stack_pointer); err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), @@ -24770,7 +25596,7 @@ Py_DECREF(ann_dict); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } } else { @@ -24789,6 +25615,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -24805,7 +25633,7 @@ PyStackRef_AsPyObjectSteal(v)); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - JUMP_TO_LABEL(pop_1_error); + TRACING_JUMP_TO_LABEL(pop_1_error); } stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -24820,6 +25648,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -24855,6 +25685,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -24876,7 +25708,7 @@ PyStackRef_CLOSE(iterable); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } TRACING_DISPATCH(); } @@ -24889,6 +25721,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -24937,7 +25771,7 @@ stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); if (err) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } } TRACING_DISPATCH(); @@ -24951,6 +25785,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -25032,6 +25868,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -25088,6 +25926,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -25181,6 +26021,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -25207,6 +26049,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -25234,6 +26078,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -25264,6 +26110,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -25302,6 +26150,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -25321,7 +26171,7 @@ PyStackRef_CLOSE(v); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } TRACING_DISPATCH(); } @@ -25334,6 +26184,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -25356,7 +26208,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); PyStackRef_CLOSE(v); stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } if (PyDict_CheckExact(ns)) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -25374,7 +26226,7 @@ PyStackRef_CLOSE(v); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } TRACING_DISPATCH(); } @@ -25387,6 +26239,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -25440,7 +26294,7 @@ stack_pointer += -4; assert(WITHIN_STACK_BOUNDS()); if (err) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } } TRACING_DISPATCH(); @@ -25454,6 +26308,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -25504,7 +26360,7 @@ stack_pointer += -3; assert(WITHIN_STACK_BOUNDS()); if (err) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } } TRACING_DISPATCH(); @@ -25518,6 +26374,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -25560,7 +26418,7 @@ PyStackRef_CLOSE(dict_st); stack_pointer = _PyFrame_GetStackPointer(frame); if (err) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } } TRACING_DISPATCH(); @@ -25574,6 +26432,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -25660,6 +26520,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -25687,6 +26549,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -25726,7 +26590,7 @@ PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } res = err ? PyStackRef_True : PyStackRef_False; } @@ -25744,6 +26608,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -25792,6 +26658,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -25821,6 +26689,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -25866,6 +26736,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -25914,6 +26786,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -25946,6 +26820,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -25999,6 +26875,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -26018,7 +26896,7 @@ PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; @@ -26035,6 +26913,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -26054,7 +26934,7 @@ PyStackRef_CLOSE(value); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; @@ -26071,6 +26951,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -26096,6 +26978,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -26115,7 +26999,7 @@ Py_DECREF(seq_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (res == 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } stack_pointer += 1 + (oparg & 0xFF) + (oparg >> 8); assert(WITHIN_STACK_BOUNDS()); @@ -26130,6 +27014,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; frame->instr_ptr = next_instr; @@ -26170,7 +27056,7 @@ Py_DECREF(seq_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (res == 0) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } } stack_pointer += oparg; @@ -26186,6 +27072,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -26250,6 +27138,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -26305,6 +27195,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -26361,6 +27253,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; @@ -26396,7 +27290,7 @@ Py_XDECREF(original_tb); stack_pointer = _PyFrame_GetStackPointer(frame); if (res_o == NULL) { - JUMP_TO_LABEL(error); + TRACING_JUMP_TO_LABEL(error); } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; @@ -26413,6 +27307,8 @@ #endif PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; int _jump_taken = false; (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; diff --git a/Python/optimizer.c b/Python/optimizer.c index 597bc6d2dc1bc1..2f6f48271710f9 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -144,24 +144,19 @@ _PyOptimizer_Optimize( return err; } assert(executor != NULL); - if (progress_needed) { - int index = get_index_for_executor(code, start); - if (index < 0) { - /* Out of memory. Don't raise and assume that the - * error will show up elsewhere. - * - * If an optimizer has already produced an executor, - * it might get confused by the executor disappearing, - * but there is not much we can do about that here. */ - Py_DECREF(executor); - interp->compiling = false; - return 0; - } - insert_executor(code, start, index, executor); - } - else { - executor->vm_data.code = NULL; + int index = get_index_for_executor(code, start); + if (index < 0) { + /* Out of memory. Don't raise and assume that the + * error will show up elsewhere. + * + * If an optimizer has already produced an executor, + * it might get confused by the executor disappearing, + * but there is not much we can do about that here. */ + Py_DECREF(executor); + interp->compiling = false; + return 0; } + insert_executor(code, start, index, executor); executor->vm_data.chain_depth = chain_depth; assert(executor->vm_data.valid); interp->compiling = false; @@ -590,11 +585,12 @@ _PyJIT_translate_single_bytecode_to_trace( } if (opcode == ENTER_EXECUTOR) { + ADD_TO_TRACE(_CHECK_VALIDITY, 0, 0, target); + ADD_TO_TRACE(_SET_IP, 0, (uintptr_t)target_instr, target); + ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); goto full; } - assert(opcode != ENTER_EXECUTOR && opcode != EXTENDED_ARG); - bool needs_guard_ip = _PyOpcode_NeedsGuardIp[opcode] && !(opcode == FOR_ITER_RANGE || opcode == FOR_ITER_LIST || opcode == FOR_ITER_TUPLE) && !(opcode == JUMP_BACKWARD_NO_INTERRUPT || opcode == JUMP_BACKWARD || opcode == JUMP_BACKWARD_JIT) && @@ -614,6 +610,8 @@ _PyJIT_translate_single_bytecode_to_trace( goto done; } + assert(opcode != ENTER_EXECUTOR && opcode != EXTENDED_ARG); + const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; RESERVE_RAW(expansion->nuops + needs_guard_ip + 3, "uop and various checks"); @@ -674,6 +672,7 @@ _PyJIT_translate_single_bytecode_to_trace( break; case RESUME: + case RESUME_CHECK: /* Use a special tier 2 version of RESUME_CHECK to allow traces to * start with RESUME_CHECK */ ADD_TO_TRACE(_TIER2_RESUME_CHECK, 0, 0, target); @@ -819,6 +818,7 @@ _PyJIT_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_ tstate->interp->jit_tracer_initial_code = code; tstate->interp->jit_tracer_initial_func = _PyFrame_GetFunction(frame); tstate->interp->jit_tracer_seen_initial_before = 0; + memset(&tstate->interp->jit_tracer_dependencies.bits, 0, sizeof(tstate->interp->jit_tracer_dependencies.bits)); tstate->interp->jit_completed_loop = false; tstate->interp->jit_tracer_initial_stack_depth = curr_stackdepth; tstate->interp->jit_tracer_initial_chain_depth = chain_depth; @@ -898,10 +898,11 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) else if (exit_flags & HAS_PERIODIC_FLAG) { exit_op = _HANDLE_PENDING_AND_DEOPT; } + int32_t jump_target = target; if (opcode == _FOR_ITER_TIER_TWO || opcode == _GUARD_IP) { exit_op = _DYNAMIC_EXIT; + jump_target = current_jump_target + 1; } - int32_t jump_target = target; if (is_for_iter_test[opcode]) { /* Target the POP_TOP immediately after the END_FOR, * leaving only the iterator on the stack. */ @@ -1057,6 +1058,7 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil _PyExitData *exit = &executor->exits[next_exit]; exit->target = buffer[i].target; dest->operand0 = (uint64_t)exit; + exit->is_dynamic = (char)(opcode == _DYNAMIC_EXIT); next_exit--; } } diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 75bff34aa74275..facdabef83e72b 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -695,6 +695,8 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "PyStackRef_Wrap", "PyStackRef_Unwrap", "_PyLong_CheckExactAndCompact", + "_PyExecutor_FromExit", + "_PyJIT_InitializeTracing", ) diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 2ecf27a6585b0c..8a95e4ea6a345b 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -107,8 +107,9 @@ class Emitter: labels: dict[str, Label] _replacers: dict[str, ReplacementFunctionType] cannot_escape: bool + tracing: str - def __init__(self, out: CWriter, labels: dict[str, Label], cannot_escape: bool = False): + def __init__(self, out: CWriter, labels: dict[str, Label], cannot_escape: bool = False, is_tracing: bool = False): self._replacers = { "EXIT_IF": self.exit_if, "AT_END_EXIT_IF": self.exit_if_after, @@ -132,6 +133,7 @@ def __init__(self, out: CWriter, labels: dict[str, Label], cannot_escape: bool = self.out = out self.labels = labels self.cannot_escape = cannot_escape + self.tracing = "TRACING_" if is_tracing else "" def dispatch( self, @@ -199,10 +201,10 @@ def exit_if_after( def goto_error(self, offset: int, storage: Storage) -> str: if offset > 0: - return f"JUMP_TO_LABEL(pop_{offset}_error);" + return f"{self.tracing}JUMP_TO_LABEL(pop_{offset}_error);" if offset < 0: storage.copy().flush(self.out) - return f"JUMP_TO_LABEL(error);" + return f"{self.tracing}JUMP_TO_LABEL(error);" def error_if( self, @@ -422,7 +424,7 @@ def goto_label(self, goto: Token, label: Token, storage: Storage) -> None: elif storage.spilled: raise analysis_error("Cannot jump from spilled label without reloading the stack pointer", goto) self.out.start_line() - self.out.emit("JUMP_TO_LABEL(") + self.out.emit(f"{self.tracing}JUMP_TO_LABEL(") self.out.emit(label) self.out.emit(")") diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 176c4e5e3b3789..4738cac33df8db 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -237,6 +237,8 @@ def generate_tier1_cases( out.emit(f"#endif\n") out.emit(f"PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable);\n") out.emit(f"(void)old_code;\n") + out.emit(f"PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj);\n") + out.emit(f"(void)old_func;\n") out.emit(f"int _jump_taken = false;\n") out.emit(f"(void)_jump_taken;\n") needs_this = is_tracing or uses_this(inst) diff --git a/Tools/cases_generator/tracer_generator.py b/Tools/cases_generator/tracer_generator.py index 910d2589b14eea..690c555b8c083b 100644 --- a/Tools/cases_generator/tracer_generator.py +++ b/Tools/cases_generator/tracer_generator.py @@ -36,7 +36,7 @@ class TracerEmitter(Emitter): cannot_escape: bool def __init__(self, out: CWriter, labels: dict[str, Label], cannot_escape: bool = False): - super().__init__(out, labels, cannot_escape) + super().__init__(out, labels, cannot_escape, is_tracing=True) self._replacers = { **self._replacers, "DISPATCH": self.dispatch, From 7b5c655246cc6cf5903f93b450e3413d186bda10 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 22 Sep 2025 12:27:52 +0100 Subject: [PATCH 016/190] Fix dispatch_inlined --- Include/internal/pycore_optimizer.h | 1 - Python/bytecodes.c | 47 ++++++++++------------- Python/ceval_macros.h | 5 ++- Python/executor_cases.c.h | 21 +++++++++- Python/generated_cases.c.h | 16 ++++---- Python/optimizer.c | 14 +++++-- Tools/cases_generator/tracer_generator.py | 15 ++++++++ 7 files changed, 78 insertions(+), 41 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index ac509b890fd31d..5dc0a34268696e 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -45,7 +45,6 @@ typedef struct _PyExitData { typedef struct _PyExecutorObject { PyObject_VAR_HEAD const _PyUOpInstruction *trace; - _Py_CODEUNIT *expected_entrypoint; _PyVMData vm_data; /* Used by the VM, but opaque to the optimizer */ uint32_t exit_count; uint32_t code_size; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index a3854fbe0cc190..43f81bfdf746cc 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5438,33 +5438,26 @@ dummy_func( assert(exit != NULL); bool is_dynamic = exit->is_dynamic; _Py_CODEUNIT *target = is_dynamic ? frame->instr_ptr : (_PyFrame_GetBytecode(frame) + exit->target); - GOTO_TIER_ONE(target, 0); - // _Py_BackoffCounter temperature = exit->temperature; - // if (target->op.code == ENTER_EXECUTOR) { - // PyCodeObject *code = _PyFrame_GetCode(frame); - // _PyExecutorObject *executor = code->co_executors->executors[target->op.arg]; - // if (is_dynamic && executor->expected_entrypoint != target) { - // GOTO_TIER_ONE(target, 0); - // } - // Py_INCREF(executor); - // assert(tstate->jit_exit == exit); - // exit->executor = executor; - // TIER2_TO_TIER2(exit->executor); - // } - // else { - // if (!backoff_counter_triggers(temperature)) { - // exit->temperature = advance_backoff_counter(temperature); - // GOTO_TIER_ONE(target, 0); - // } - // if (is_dynamic) { - // GOTO_TIER_ONE(target, 0); - // } - // _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); - // assert(tstate->current_executor == (PyObject *)previous_executor); - // int chain_depth = 0; - // _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), chain_depth); - // GOTO_TIER_ONE(target, 1); - // } + _Py_BackoffCounter temperature = exit->temperature; + if (target->op.code == ENTER_EXECUTOR) { + PyCodeObject *code = _PyFrame_GetCode(frame); + _PyExecutorObject *executor = code->co_executors->executors[target->op.arg]; + Py_INCREF(executor); + assert(tstate->jit_exit == exit); + exit->executor = executor; + TIER2_TO_TIER2(exit->executor); + } + else { + if (!backoff_counter_triggers(temperature)) { + exit->temperature = advance_backoff_counter(temperature); + GOTO_TIER_ONE(target, 0); + } + _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); + assert(tstate->current_executor == (PyObject *)previous_executor); + int chain_depth = 0; + _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), chain_depth); + GOTO_TIER_ONE(target, 1); + } } tier2 op(_GUARD_IP, (ip/4 --)) { diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 2f53366915dc72..540785414e7334 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -217,6 +217,10 @@ do { \ JUMP_TO_LABEL(start_frame); \ } while (0) +#define TRACING_DISPATCH_INLINED(NEW_FRAME) \ + RECORD_TRACE_NO_DISPATCH(); \ + DISPATCH_INLINED(NEW_FRAME); + #define TRACING_DISPATCH() \ { \ assert(frame->stackpointer == NULL); \ @@ -421,7 +425,6 @@ do { \ JUMP_TO_LABEL(error); \ } \ if (keep_tracing_bit) { \ - assert(next_instr == frame->instr_ptr); \ assert(next_instr->op.code != ENTER_EXECUTOR); \ assert(tstate->interp->jit_tracer_code_curr_size == 2); \ ENTER_TRACING(); \ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 0a94baa0557d37..f2a100210d4b86 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7522,7 +7522,26 @@ assert(exit != NULL); bool is_dynamic = exit->is_dynamic; _Py_CODEUNIT *target = is_dynamic ? frame->instr_ptr : (_PyFrame_GetBytecode(frame) + exit->target); - GOTO_TIER_ONE(target, 0); + _Py_BackoffCounter temperature = exit->temperature; + if (target->op.code == ENTER_EXECUTOR) { + PyCodeObject *code = _PyFrame_GetCode(frame); + _PyExecutorObject *executor = code->co_executors->executors[target->op.arg]; + Py_INCREF(executor); + assert(tstate->jit_exit == exit); + exit->executor = executor; + TIER2_TO_TIER2(exit->executor); + } + else { + if (!backoff_counter_triggers(temperature)) { + exit->temperature = advance_backoff_counter(temperature); + GOTO_TIER_ONE(target, 0); + } + _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); + assert(tstate->current_executor == (PyObject *)previous_executor); + int chain_depth = 0; + _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), chain_depth); + GOTO_TIER_ONE(target, 1); + } break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 78a458b4899eff..1f77215f582219 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -15256,7 +15256,7 @@ TRACING_JUMP_TO_LABEL(error); } frame->return_offset = 4u ; - DISPATCH_INLINED(new_frame); + TRACING_DISPATCH_INLINED(new_frame); } STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { @@ -16329,7 +16329,7 @@ } assert( 1u == 1); frame->return_offset = 1; - DISPATCH_INLINED(new_frame); + TRACING_DISPATCH_INLINED(new_frame); } PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); assert(PyTuple_CheckExact(callargs)); @@ -16641,7 +16641,7 @@ } assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); frame->return_offset = 4u ; - DISPATCH_INLINED(new_frame); + TRACING_DISPATCH_INLINED(new_frame); } STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { @@ -20564,7 +20564,7 @@ TRACING_JUMP_TO_LABEL(error); } frame->return_offset = 4u ; - DISPATCH_INLINED(new_frame); + TRACING_DISPATCH_INLINED(new_frame); } STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { @@ -20785,7 +20785,7 @@ } assert( 1u == 1); frame->return_offset = 1; - DISPATCH_INLINED(new_frame); + TRACING_DISPATCH_INLINED(new_frame); } PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); assert(PyTuple_CheckExact(callargs)); @@ -20935,7 +20935,7 @@ } assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); frame->return_offset = 4u ; - DISPATCH_INLINED(new_frame); + TRACING_DISPATCH_INLINED(new_frame); } STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); if (CONVERSION_FAILED(args_o)) { @@ -22544,7 +22544,7 @@ assert(WITHIN_STACK_BOUNDS()); new_frame->localsplus[1] = PyStackRef_FromPyObjectNew(name); frame->return_offset = 10u ; - DISPATCH_INLINED(new_frame); + TRACING_DISPATCH_INLINED(new_frame); } TRACING_TARGET(LOAD_ATTR_INSTANCE_VALUE) { @@ -25424,7 +25424,7 @@ frame->return_offset = (uint16_t)( 2u + oparg); assert(gen_frame->previous == NULL); gen_frame->previous = frame; - DISPATCH_INLINED(gen_frame); + TRACING_DISPATCH_INLINED(gen_frame); } if (PyStackRef_IsNone(v) && PyIter_Check(receiver_o)) { _PyFrame_SetStackPointer(frame, stack_pointer); diff --git a/Python/optimizer.c b/Python/optimizer.c index 2f6f48271710f9..8c69bd10662aa6 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -584,6 +584,10 @@ _PyJIT_translate_single_bytecode_to_trace( return 1; } + if (opcode == JUMP_BACKWARD_NO_INTERRUPT) { + return 1; + } + if (opcode == ENTER_EXECUTOR) { ADD_TO_TRACE(_CHECK_VALIDITY, 0, 0, target); ADD_TO_TRACE(_SET_IP, 0, (uintptr_t)target_instr, target); @@ -899,9 +903,13 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) exit_op = _HANDLE_PENDING_AND_DEOPT; } int32_t jump_target = target; - if (opcode == _FOR_ITER_TIER_TWO || opcode == _GUARD_IP) { + bool unique_target = false; + if (opcode == _FOR_ITER_TIER_TWO) { + exit_op = _DYNAMIC_EXIT; + } + else if (opcode == _GUARD_IP) { exit_op = _DYNAMIC_EXIT; - jump_target = current_jump_target + 1; + unique_target = true; } if (is_for_iter_test[opcode]) { /* Target the POP_TOP immediately after the END_FOR, @@ -910,7 +918,7 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) int32_t next_inst = target + 1 + INLINE_CACHE_ENTRIES_FOR_ITER + extended_arg; jump_target = next_inst + inst->oparg + 1; } - if (jump_target != current_jump_target || current_exit_op != exit_op) { + if (unique_target || jump_target != current_jump_target || current_exit_op != exit_op) { make_exit(&buffer[next_spare], exit_op, jump_target); current_exit_op = exit_op; current_jump_target = jump_target; diff --git a/Tools/cases_generator/tracer_generator.py b/Tools/cases_generator/tracer_generator.py index 690c555b8c083b..61088ff185e2d5 100644 --- a/Tools/cases_generator/tracer_generator.py +++ b/Tools/cases_generator/tracer_generator.py @@ -40,6 +40,7 @@ def __init__(self, out: CWriter, labels: dict[str, Label], cannot_escape: bool = self._replacers = { **self._replacers, "DISPATCH": self.dispatch, + "DISPATCH_INLINED": self.dispatch_inlined, } def dispatch( @@ -57,6 +58,20 @@ def dispatch( self.emit("TRACING_DISPATCH") return False + def dispatch_inlined( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ) -> bool: + if storage.spilled: + raise analysis_error("stack_pointer needs reloading before dispatch", tkn) + storage.stack.flush(self.out) + self.out.start_line() + self.emit("TRACING_DISPATCH_INLINED") + return False def record_jump_taken( self, tkn: Token, From 3e9f78211d6baec04a471855a4786f16fdb39026 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 22 Sep 2025 14:05:57 +0100 Subject: [PATCH 017/190] Fix handling of EXTENDED_ARG --- Python/optimizer.c | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/Python/optimizer.c b/Python/optimizer.c index 8c69bd10662aa6..c7d8784f2bef73 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -436,12 +436,12 @@ PyTypeObject _PyUOpExecutor_Type = { /* TO DO -- Generate these tables */ static const uint16_t _PyUOp_Replacements[MAX_UOP_ID + 1] = { - [_CHECK_PERIODIC_AT_END] = _TIER2_RESUME_CHECK, [_ITER_JUMP_RANGE] = _GUARD_NOT_EXHAUSTED_RANGE, [_ITER_JUMP_LIST] = _GUARD_NOT_EXHAUSTED_LIST, [_ITER_JUMP_TUPLE] = _GUARD_NOT_EXHAUSTED_TUPLE, [_FOR_ITER] = _FOR_ITER_TIER_TWO, [_ITER_NEXT_LIST] = _ITER_NEXT_LIST_TIER_TWO, + [_CHECK_PERIODIC_AT_END] = _TIER2_RESUME_CHECK, }; static const uint8_t @@ -563,6 +563,9 @@ _PyJIT_translate_single_bytecode_to_trace( uint32_t target = 0; target = INSTR_IP(target_instr, old_code); + + DPRINTF(2, "%d: %s(%d)\n", target, _PyOpcode_OpName[opcode], oparg); + // One for possible _DEOPT, one because _CHECK_VALIDITY itself might _DEOPT max_length -= 2; if ((uint16_t)oparg != (uint64_t)oparg) { @@ -575,7 +578,6 @@ _PyJIT_translate_single_bytecode_to_trace( goto full; } - DPRINTF(2, "%d: %s(%d)\n", target, _PyOpcode_OpName[opcode], oparg); if (opcode == EXTENDED_ARG) { return 1; @@ -726,10 +728,17 @@ _PyJIT_translate_single_bytecode_to_trace( uop = _PyUOp_Replacements[uop]; assert(uop != 0); - uint32_t next_inst = target + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]] + (oparg > 255); + uint32_t next_inst = target + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; if (uop == _TIER2_RESUME_CHECK) { target = next_inst; } +#ifdef Py_DEBUG + else if (uop != _FOR_ITER_TIER_TWO) { + uint32_t jump_target = next_inst + oparg; + assert(_Py_GetBaseCodeUnit(old_code, jump_target).op.code == END_FOR); + assert(_Py_GetBaseCodeUnit(old_code, jump_target+1).op.code == POP_ITER); + } +#endif break; case OPERAND1_1: assert(trace[trace_length-1].opcode == uop); @@ -914,8 +923,7 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) if (is_for_iter_test[opcode]) { /* Target the POP_TOP immediately after the END_FOR, * leaving only the iterator on the stack. */ - int extended_arg = inst->oparg > 255; - int32_t next_inst = target + 1 + INLINE_CACHE_ENTRIES_FOR_ITER + extended_arg; + int32_t next_inst = target + 1 + INLINE_CACHE_ENTRIES_FOR_ITER; jump_target = next_inst + inst->oparg + 1; } if (unique_target || jump_target != current_jump_target || current_exit_op != exit_op) { From 9a66605da2b8f3328260213172cd20b79d12ca88 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 23 Sep 2025 00:05:28 +0100 Subject: [PATCH 018/190] Fix chain depth bug --- Python/bytecodes.c | 3 ++- Python/executor_cases.c.h | 3 ++- Python/optimizer.c | 47 ++++++++++++++++++++++----------------- 3 files changed, 31 insertions(+), 22 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 43f81bfdf746cc..e0150aafaf52e8 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5289,6 +5289,7 @@ dummy_func( } #endif tstate->jit_exit = exit; + assert(!exit->is_dynamic); TIER2_TO_TIER2(exit->executor); } @@ -5454,7 +5455,7 @@ dummy_func( } _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); assert(tstate->current_executor == (PyObject *)previous_executor); - int chain_depth = 0; + int chain_depth = is_dynamic ? 0 : current_executor->vm_data.chain_depth + 1; _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), chain_depth); GOTO_TIER_ONE(target, 1); } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index f2a100210d4b86..56a1c9b7a52e68 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7178,6 +7178,7 @@ } #endif tstate->jit_exit = exit; + assert(!exit->is_dynamic); TIER2_TO_TIER2(exit->executor); break; } @@ -7538,7 +7539,7 @@ } _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); assert(tstate->current_executor == (PyObject *)previous_executor); - int chain_depth = 0; + int chain_depth = is_dynamic ? 0 : current_executor->vm_data.chain_depth + 1; _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), chain_depth); GOTO_TIER_ONE(target, 1); } diff --git a/Python/optimizer.c b/Python/optimizer.c index c7d8784f2bef73..5ab6ef4e91bf13 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -102,7 +102,7 @@ insert_executor(PyCodeObject *code, _Py_CODEUNIT *instr, int index, _PyExecutorO } static _PyExecutorObject * -make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies); +make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies, int chain_depth); static int uop_optimize(_PyInterpreterFrame *frame, PyThreadState *tstate, @@ -128,8 +128,7 @@ _PyOptimizer_Optimize( // make progress in order to avoid infinite loops or excessively-long // side-exit chains. We can only insert the executor into the bytecode if // this is true, since a deopt won't infinitely re-enter the executor: - chain_depth %= MAX_CHAIN_DEPTH; - bool progress_needed = chain_depth == 0; + bool progress_needed = (chain_depth % MAX_CHAIN_DEPTH) == 0; PyCodeObject *code = (PyCodeObject *)tstate->interp->jit_tracer_initial_func->func_code; assert(PyCode_Check(code)); _Py_CODEUNIT *start = tstate->interp->jit_tracer_initial_instr; @@ -144,19 +143,24 @@ _PyOptimizer_Optimize( return err; } assert(executor != NULL); - int index = get_index_for_executor(code, start); - if (index < 0) { - /* Out of memory. Don't raise and assume that the - * error will show up elsewhere. - * - * If an optimizer has already produced an executor, - * it might get confused by the executor disappearing, - * but there is not much we can do about that here. */ - Py_DECREF(executor); - interp->compiling = false; - return 0; + if (progress_needed) { + int index = get_index_for_executor(code, start); + if (index < 0) { + /* Out of memory. Don't raise and assume that the + * error will show up elsewhere. + * + * If an optimizer has already produced an executor, + * it might get confused by the executor disappearing, + * but there is not much we can do about that here. */ + Py_DECREF(executor); + interp->compiling = false; + return 0; + } + insert_executor(code, start, index, executor); + } + else { + executor->vm_data.code = NULL; } - insert_executor(code, start, index, executor); executor->vm_data.chain_depth = chain_depth; assert(executor->vm_data.valid); interp->compiling = false; @@ -544,7 +548,8 @@ _PyJIT_translate_single_bytecode_to_trace( if (Py_IsNone((PyObject *)func)) { func = NULL; } - bool progress_needed = (tstate->interp->jit_tracer_initial_chain_depth % MAX_CHAIN_DEPTH) == 0;; + int is_first_instr = tstate->interp->jit_tracer_initial_instr == this_instr ; + bool progress_needed = (tstate->interp->jit_tracer_initial_chain_depth % MAX_CHAIN_DEPTH) == 0 && is_first_instr;; _PyBloomFilter *dependencies = &tstate->interp->jit_tracer_dependencies; _Py_BloomFilter_Add(dependencies, old_code); _Py_CODEUNIT *target_instr = this_instr; @@ -629,7 +634,7 @@ _PyJIT_translate_single_bytecode_to_trace( /* Special case the first instruction, * so that we can guarantee forward progress */ - if (progress_needed && tstate->interp->jit_tracer_initial_instr == this_instr && tstate->interp->jit_tracer_code_curr_size == 0) { + if (progress_needed && is_first_instr && tstate->interp->jit_tracer_code_curr_size == 0) { if (OPCODE_HAS_EXIT(opcode) || OPCODE_HAS_DEOPT(opcode)) { opcode = _PyOpcode_Deopt[opcode]; } @@ -638,7 +643,7 @@ _PyJIT_translate_single_bytecode_to_trace( } // Loop back to the start - if (tstate->interp->jit_tracer_initial_instr == this_instr && tstate->interp->jit_tracer_code_curr_size > 2) { + if (is_first_instr && tstate->interp->jit_tracer_code_curr_size > 2) { // Undo the last few instructions. trace_length = tstate->interp->jit_tracer_code_curr_size; ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0, 0); @@ -1047,7 +1052,7 @@ sanity_check(_PyExecutorObject *executor) * and not a NOP. */ static _PyExecutorObject * -make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies) +make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies, int chain_depth) { int exit_count = count_exits(buffer, length); _PyExecutorObject *executor = allocate_executor(exit_count, length); @@ -1057,6 +1062,8 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil /* Initialize exits */ _PyExecutorObject *cold = _PyExecutor_GetColdExecutor(); + fprintf(stdout, "CHAIN DEPTH %d;\n", chain_depth); + cold->vm_data.chain_depth = chain_depth; for (int i = 0; i < exit_count; i++) { executor->exits[i].index = i; executor->exits[i].temperature = initial_temperature_backoff_counter(); @@ -1191,7 +1198,7 @@ uop_optimize( OPT_HIST(effective_trace_length(buffer, length), optimized_trace_length_hist); length = prepare_for_execution(buffer, length); assert(length <= UOP_MAX_TRACE_LENGTH); - _PyExecutorObject *executor = make_executor_from_uops(buffer, length, &dependencies); + _PyExecutorObject *executor = make_executor_from_uops(buffer, length, &dependencies, tstate->interp->jit_tracer_initial_chain_depth); if (executor == NULL) { return -1; } From 108ab7f223af9c57ec1d3693fbc2a72a328f6684 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 23 Sep 2025 00:07:13 +0100 Subject: [PATCH 019/190] remove printf --- Python/optimizer.c | 1 - 1 file changed, 1 deletion(-) diff --git a/Python/optimizer.c b/Python/optimizer.c index 5ab6ef4e91bf13..1badc9127987a3 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1062,7 +1062,6 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil /* Initialize exits */ _PyExecutorObject *cold = _PyExecutor_GetColdExecutor(); - fprintf(stdout, "CHAIN DEPTH %d;\n", chain_depth); cold->vm_data.chain_depth = chain_depth; for (int i = 0; i < exit_count; i++) { executor->exits[i].index = i; From fac8c748a5baefd1c3272bf53a9d3372df726bf5 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 23 Sep 2025 23:17:36 +0100 Subject: [PATCH 020/190] fix problem with jumping labels --- Python/ceval_macros.h | 3 +-- Python/generated_cases.c.h | 20 +++++++++++++++++++- Python/optimizer.c | 19 +++++++++---------- Python/optimizer_analysis.c | 6 +++--- 4 files changed, 32 insertions(+), 16 deletions(-) diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 540785414e7334..b15b6788205915 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -132,8 +132,7 @@ #define TRACING_JUMP_TO_LABEL(label) \ RECORD_JUMP_TAKEN() \ - RECORD_TRACE() \ - BAIL_TRACING_NO_DISPATCH() \ + RECORD_TRACE_NO_DISPATCH() \ JUMP_TO_LABEL(label); #if _Py_TAIL_CALL_INTERP || USE_COMPUTED_GOTOS diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 1f77215f582219..92ddfeb3a982f9 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5976,7 +5976,25 @@ next_instr = this_instr; DISPATCH_GOTO(); } - TIER1_TO_TIER2(executor); + OPT_STAT_INC(traces_executed); \ + next_instr = _Py_jit_entry((executor), frame, stack_pointer, tstate); \ + frame = tstate->current_frame; \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ + int keep_tracing_bit = (uintptr_t)next_instr & 1; \ + next_instr = (_Py_CODEUNIT *)(((uintptr_t)next_instr) >> 1 << 1); \ + if (next_instr == NULL) { \ + next_instr = frame->instr_ptr; \ + JUMP_TO_LABEL(error); \ + } \ + if (keep_tracing_bit) { \ + assert(next_instr->op.code != ENTER_EXECUTOR); \ + assert(tstate->interp->jit_tracer_code_curr_size == 2); \ + ENTER_TRACING(); \ + } \ + else { \ + LEAVE_TRACING(); \ + } \ + DISPATCH(); #else Py_FatalError("ENTER_EXECUTOR is not supported in this build"); #endif /* _Py_TIER2 */ diff --git a/Python/optimizer.c b/Python/optimizer.c index 1badc9127987a3..7789558a29b9e4 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -607,9 +607,12 @@ _PyJIT_translate_single_bytecode_to_trace( !(opcode == JUMP_BACKWARD_NO_INTERRUPT || opcode == JUMP_BACKWARD || opcode == JUMP_BACKWARD_JIT) && !(opcode == POP_JUMP_IF_TRUE || opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_NONE || opcode == POP_JUMP_IF_NOT_NONE); + assert(opcode != ENTER_EXECUTOR && opcode != EXTENDED_ARG); + + const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; - // Strange control-flow - if (jump_taken || opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO) { + // Strange control-flow, unsupported opcode, etc. + if (jump_taken || expansion->nuops == 0 || opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO) { // Rewind to previous instruction and replace with _EXIT_TRACE. _PyUOpInstruction *curr = &trace[trace_length-1]; while (curr->opcode != _SET_IP && trace_length > 1) { @@ -620,10 +623,6 @@ _PyJIT_translate_single_bytecode_to_trace( curr->opcode = _EXIT_TRACE; goto done; } - - assert(opcode != ENTER_EXECUTOR && opcode != EXTENDED_ARG); - - const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; RESERVE_RAW(expansion->nuops + needs_guard_ip + 3, "uop and various checks"); ADD_TO_TRACE(_CHECK_VALIDITY, 0, 0, target); @@ -692,12 +691,9 @@ _PyJIT_translate_single_bytecode_to_trace( default: { const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; - if (expansion->nuops == 0) { - DPRINTF(2, "Unsupported opcode %s\n", _PyOpcode_OpName[opcode]); - goto full; - } // Reserve space for nuops (+ _SET_IP + _EXIT_TRACE) int nuops = expansion->nuops; + assert(nuops > 0); RESERVE(nuops + 1); /* One extra for exit */ uint32_t orig_oparg = oparg; // For OPARG_TOP/BOTTOM for (int i = 0; i < nuops; i++) { @@ -906,6 +902,9 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) for (int i = 0; i < length; i++) { _PyUOpInstruction *inst = &buffer[i]; int opcode = inst->opcode; + if (inst->format != UOP_FORMAT_TARGET) { + fprintf(stdout, "I: %d\n", i); + } int32_t target = (int32_t)uop_get_target(inst); uint16_t exit_flags = _PyUop_Flags[opcode] & (HAS_EXIT_FLAG | HAS_DEOPT_FLAG | HAS_PERIODIC_FLAG); if (exit_flags) { diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 1bb52188fba48e..b90575abf51cdd 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -528,12 +528,12 @@ _Py_uop_analyze_and_optimize( { OPT_STAT_INC(optimizer_attempts); - // length = optimize_uops( + // int err = optimize_uops( // _PyFrame_GetFunction(frame), buffer, // length, curr_stacklen, dependencies); // - // if (length == 0) { - // return length; + // if (err == 0) { + // return err; // } assert(length > 0); From fd3bb482885d6703e55570af86669978b9f517b6 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 23 Sep 2025 23:27:01 +0100 Subject: [PATCH 021/190] Point to previous executor when side-exiting --- Include/internal/pycore_interp_structs.h | 2 +- Include/internal/pycore_optimizer.h | 2 +- Python/bytecodes.c | 4 ++-- Python/executor_cases.c.h | 2 +- Python/generated_cases.c.h | 24 +++--------------------- Python/optimizer.c | 9 +++++++-- 6 files changed, 15 insertions(+), 28 deletions(-) diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 5834a6f8edda3a..9671325362d5c3 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -953,7 +953,7 @@ struct _is { int jit_tracer_initial_chain_depth; PyCodeObject *jit_tracer_initial_code; // Borrowed PyFunctionObject *jit_tracer_initial_func; // Borrowed - int jit_tracer_seen_initial_before; + struct _PyExitData *jit_tracer_previous_exit; bool jit_completed_loop; bool jit; bool compiling; diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 5dc0a34268696e..79b1931d070467 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -371,7 +371,7 @@ _PyJIT_translate_single_bytecode_to_trace( int jump_taken); void -_PyJIT_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, int curr_stackdepth, int chain_depth); +_PyJIT_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit); #ifdef __cplusplus } #endif diff --git a/Python/bytecodes.c b/Python/bytecodes.c index e0150aafaf52e8..d033900c30839d 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2978,7 +2978,7 @@ dummy_func( DISPATCH(); } } - _PyJIT_InitializeTracing(tstate, frame, next_instr, STACK_LEVEL(), 0); + _PyJIT_InitializeTracing(tstate, frame, next_instr, STACK_LEVEL(), 0, NULL); ENTER_TRACING(); // Don't add the JUMP_BACKWARD_JIT instruction to the trace. DISPATCH(); @@ -5456,7 +5456,7 @@ dummy_func( _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); assert(tstate->current_executor == (PyObject *)previous_executor); int chain_depth = is_dynamic ? 0 : current_executor->vm_data.chain_depth + 1; - _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), chain_depth); + _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), chain_depth, exit); GOTO_TIER_ONE(target, 1); } } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 56a1c9b7a52e68..d4c7c6ad5687a3 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7540,7 +7540,7 @@ _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); assert(tstate->current_executor == (PyObject *)previous_executor); int chain_depth = is_dynamic ? 0 : current_executor->vm_data.chain_depth + 1; - _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), chain_depth); + _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), chain_depth, exit); GOTO_TIER_ONE(target, 1); } break; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 92ddfeb3a982f9..229d75a8dc0dc7 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5976,25 +5976,7 @@ next_instr = this_instr; DISPATCH_GOTO(); } - OPT_STAT_INC(traces_executed); \ - next_instr = _Py_jit_entry((executor), frame, stack_pointer, tstate); \ - frame = tstate->current_frame; \ - stack_pointer = _PyFrame_GetStackPointer(frame); \ - int keep_tracing_bit = (uintptr_t)next_instr & 1; \ - next_instr = (_Py_CODEUNIT *)(((uintptr_t)next_instr) >> 1 << 1); \ - if (next_instr == NULL) { \ - next_instr = frame->instr_ptr; \ - JUMP_TO_LABEL(error); \ - } \ - if (keep_tracing_bit) { \ - assert(next_instr->op.code != ENTER_EXECUTOR); \ - assert(tstate->interp->jit_tracer_code_curr_size == 2); \ - ENTER_TRACING(); \ - } \ - else { \ - LEAVE_TRACING(); \ - } \ - DISPATCH(); + TIER1_TO_TIER2(executor); #else Py_FatalError("ENTER_EXECUTOR is not supported in this build"); #endif /* _Py_TIER2 */ @@ -8424,7 +8406,7 @@ DISPATCH(); } } - _PyJIT_InitializeTracing(tstate, frame, next_instr, STACK_LEVEL(), 0); + _PyJIT_InitializeTracing(tstate, frame, next_instr, STACK_LEVEL(), 0, NULL); ENTER_TRACING(); DISPATCH(); } @@ -22090,7 +22072,7 @@ TRACING_DISPATCH(); } } - _PyJIT_InitializeTracing(tstate, frame, next_instr, STACK_LEVEL(), 0); + _PyJIT_InitializeTracing(tstate, frame, next_instr, STACK_LEVEL(), 0, NULL); ENTER_TRACING(); TRACING_DISPATCH(); } diff --git a/Python/optimizer.c b/Python/optimizer.c index 7789558a29b9e4..8c7028b2a5a399 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -161,6 +161,11 @@ _PyOptimizer_Optimize( else { executor->vm_data.code = NULL; } + if (chain_depth > 0) { + _PyExitData *prev_exit = tstate->interp->jit_tracer_previous_exit; + assert(prev_exit != NULL); + prev_exit->executor = executor;; + } executor->vm_data.chain_depth = chain_depth; assert(executor->vm_data.valid); interp->compiling = false; @@ -808,7 +813,7 @@ _PyJIT_translate_single_bytecode_to_trace( } void -_PyJIT_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, int curr_stackdepth, int chain_depth) +_PyJIT_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit) { PyCodeObject *code = _PyFrame_GetCode(frame); #ifdef Py_DEBUG @@ -831,7 +836,7 @@ _PyJIT_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_ tstate->interp->jit_tracer_initial_instr = next_instr; tstate->interp->jit_tracer_initial_code = code; tstate->interp->jit_tracer_initial_func = _PyFrame_GetFunction(frame); - tstate->interp->jit_tracer_seen_initial_before = 0; + tstate->interp->jit_tracer_previous_exit = exit; memset(&tstate->interp->jit_tracer_dependencies.bits, 0, sizeof(tstate->interp->jit_tracer_dependencies.bits)); tstate->interp->jit_completed_loop = false; tstate->interp->jit_tracer_initial_stack_depth = curr_stackdepth; From 96b7bb2761502081765abeee357671915e59d4e4 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 24 Sep 2025 00:08:33 +0100 Subject: [PATCH 022/190] Fix progress needed and warmup --- Python/bytecodes.c | 7 +------ Python/executor_cases.c.h | 1 + Python/generated_cases.c.h | 12 ------------ Python/optimizer.c | 2 +- 4 files changed, 3 insertions(+), 19 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index d033900c30839d..658177c041dc6e 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3047,13 +3047,7 @@ dummy_func( assert(executor != tstate->interp->cold_executor); tstate->jit_exit = NULL; if (IS_JIT_TRACING()) { - int old_opcode = executor->vm_data.opcode; - int old_oparg = (oparg & ~255) | executor->vm_data.oparg; RECORD_TRACE_NO_DISPATCH(); - opcode = old_opcode; - oparg = old_oparg; - next_instr = this_instr; - DISPATCH_GOTO(); } TIER1_TO_TIER2(executor); #else @@ -5453,6 +5447,7 @@ dummy_func( exit->temperature = advance_backoff_counter(temperature); GOTO_TIER_ONE(target, 0); } + exit->temperature = initial_temperature_backoff_counter(); _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); assert(tstate->current_executor == (PyObject *)previous_executor); int chain_depth = is_dynamic ? 0 : current_executor->vm_data.chain_depth + 1; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index d4c7c6ad5687a3..b25fb61bd5209d 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7537,6 +7537,7 @@ exit->temperature = advance_backoff_counter(temperature); GOTO_TIER_ONE(target, 0); } + exit->temperature = initial_temperature_backoff_counter(); _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); assert(tstate->current_executor == (PyObject *)previous_executor); int chain_depth = is_dynamic ? 0 : current_executor->vm_data.chain_depth + 1; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 229d75a8dc0dc7..70e3b798563920 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5968,13 +5968,7 @@ assert(executor != tstate->interp->cold_executor); tstate->jit_exit = NULL; if (IS_JIT_TRACING()) { - int old_opcode = executor->vm_data.opcode; - int old_oparg = (oparg & ~255) | executor->vm_data.oparg; RECORD_TRACE_NO_DISPATCH(); - opcode = old_opcode; - oparg = old_oparg; - next_instr = this_instr; - DISPATCH_GOTO(); } TIER1_TO_TIER2(executor); #else @@ -19560,13 +19554,7 @@ assert(executor != tstate->interp->cold_executor); tstate->jit_exit = NULL; if (IS_JIT_TRACING()) { - int old_opcode = executor->vm_data.opcode; - int old_oparg = (oparg & ~255) | executor->vm_data.oparg; RECORD_TRACE_NO_DISPATCH(); - opcode = old_opcode; - oparg = old_oparg; - next_instr = this_instr; - DISPATCH_GOTO(); } TIER1_TO_TIER2(executor); #else diff --git a/Python/optimizer.c b/Python/optimizer.c index 8c7028b2a5a399..1afbadc207d592 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -638,7 +638,7 @@ _PyJIT_translate_single_bytecode_to_trace( /* Special case the first instruction, * so that we can guarantee forward progress */ - if (progress_needed && is_first_instr && tstate->interp->jit_tracer_code_curr_size == 0) { + if (progress_needed && is_first_instr) { if (OPCODE_HAS_EXIT(opcode) || OPCODE_HAS_DEOPT(opcode)) { opcode = _PyOpcode_Deopt[opcode]; } From 396818b9fc7baeae8016794450067023e1d19070 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 24 Sep 2025 11:41:37 +0100 Subject: [PATCH 023/190] Fix unsupported opcode bug, turn off optimizer again --- Python/optimizer.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Python/optimizer.c b/Python/optimizer.c index 1afbadc207d592..c916ffd0e84e39 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -617,7 +617,8 @@ _PyJIT_translate_single_bytecode_to_trace( const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; // Strange control-flow, unsupported opcode, etc. - if (jump_taken || expansion->nuops == 0 || opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO) { + if (jump_taken || opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO) { + unsupported: // Rewind to previous instruction and replace with _EXIT_TRACE. _PyUOpInstruction *curr = &trace[trace_length-1]; while (curr->opcode != _SET_IP && trace_length > 1) { @@ -698,6 +699,10 @@ _PyJIT_translate_single_bytecode_to_trace( const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; // Reserve space for nuops (+ _SET_IP + _EXIT_TRACE) int nuops = expansion->nuops; + if (nuops == 0) { + DPRINTF(2, "Unsupported opcode %s\n", _PyOpcode_OpName[opcode]); + goto unsupported; + } assert(nuops > 0); RESERVE(nuops + 1); /* One extra for exit */ uint32_t orig_oparg = oparg; // For OPARG_TOP/BOTTOM @@ -1169,7 +1174,7 @@ uop_optimize( int curr_stackentries = tstate->interp->jit_tracer_initial_stack_depth; int length = interp->jit_tracer_code_curr_size; // Trace too short, don't bother. - if (length <= 8) { + if (length <= 4) { return 0; } assert(length > 0); From 57f417e91bd3278fcec090a026f0168408408317 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 24 Sep 2025 11:55:11 +0100 Subject: [PATCH 024/190] fix branch tracing --- Python/bytecodes.c | 4 ++-- Python/generated_cases.c.h | 20 ++++---------------- Python/optimizer.c | 4 +--- 3 files changed, 7 insertions(+), 21 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 658177c041dc6e..c30e314e653e82 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3059,7 +3059,7 @@ dummy_func( assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsFalse(cond); DEAD(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + RECORD_JUMP_TAKEN(); JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } @@ -3067,7 +3067,7 @@ dummy_func( assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsTrue(cond); DEAD(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + RECORD_JUMP_TAKEN(); JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 70e3b798563920..83f5fde87e6a88 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -11024,8 +11024,6 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_FALSE); @@ -11034,7 +11032,6 @@ cond = stack_pointer[-1]; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsFalse(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -11052,8 +11049,6 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_NONE); @@ -11082,7 +11077,6 @@ cond = b; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsTrue(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } stack_pointer += -1; @@ -11101,8 +11095,6 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); @@ -11131,7 +11123,6 @@ cond = b; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsFalse(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } stack_pointer += -1; @@ -11150,8 +11141,6 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_TRUE); @@ -11160,7 +11149,6 @@ cond = stack_pointer[-1]; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsTrue(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -24811,7 +24799,7 @@ cond = stack_pointer[-1]; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsFalse(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + RECORD_JUMP_TAKEN(); JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -24860,7 +24848,7 @@ cond = b; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsTrue(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + RECORD_JUMP_TAKEN(); JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } stack_pointer += -1; @@ -24910,7 +24898,7 @@ cond = b; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsFalse(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + RECORD_JUMP_TAKEN(); JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } stack_pointer += -1; @@ -24940,7 +24928,7 @@ cond = stack_pointer[-1]; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsTrue(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); + RECORD_JUMP_TAKEN(); JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); diff --git a/Python/optimizer.c b/Python/optimizer.c index c916ffd0e84e39..b4d3429f7321fc 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -671,9 +671,7 @@ _PyJIT_translate_single_bytecode_to_trace( case POP_JUMP_IF_TRUE: { RESERVE(1); - int counter = this_instr[1].cache; - int bitcount = counter & 1; - int jump_likely = bitcount; + int jump_likely = jump_taken; uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_likely]; _Py_CODEUNIT *next_instr = target_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; _Py_CODEUNIT *false_target = next_instr + oparg; From 2c603cc2974dc18ac31cc2175b59e88591fafa26 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 24 Sep 2025 12:18:01 +0100 Subject: [PATCH 025/190] fix branch prediction for real --- Python/bytecodes.c | 2 -- Python/generated_cases.c.h | 4 ---- Python/optimizer.c | 11 ++++++----- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index c30e314e653e82..d8e987708eba7b 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3059,7 +3059,6 @@ dummy_func( assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsFalse(cond); DEAD(cond); - RECORD_JUMP_TAKEN(); JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } @@ -3067,7 +3066,6 @@ dummy_func( assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsTrue(cond); DEAD(cond); - RECORD_JUMP_TAKEN(); JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 83f5fde87e6a88..1633559d72ac12 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -24799,7 +24799,6 @@ cond = stack_pointer[-1]; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsFalse(cond); - RECORD_JUMP_TAKEN(); JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -24848,7 +24847,6 @@ cond = b; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsTrue(cond); - RECORD_JUMP_TAKEN(); JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } stack_pointer += -1; @@ -24898,7 +24896,6 @@ cond = b; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsFalse(cond); - RECORD_JUMP_TAKEN(); JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } stack_pointer += -1; @@ -24928,7 +24925,6 @@ cond = stack_pointer[-1]; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsTrue(cond); - RECORD_JUMP_TAKEN(); JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); diff --git a/Python/optimizer.c b/Python/optimizer.c index b4d3429f7321fc..c8a0a3cb74ed58 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -617,7 +617,8 @@ _PyJIT_translate_single_bytecode_to_trace( const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; // Strange control-flow, unsupported opcode, etc. - if (jump_taken || opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO) { + if (jump_taken || + opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO) { unsupported: // Rewind to previous instruction and replace with _EXIT_TRACE. _PyUOpInstruction *curr = &trace[trace_length-1]; @@ -671,11 +672,11 @@ _PyJIT_translate_single_bytecode_to_trace( case POP_JUMP_IF_TRUE: { RESERVE(1); - int jump_likely = jump_taken; + _Py_CODEUNIT *computed_next_instr = target_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; + _Py_CODEUNIT *computed_jump_instr = computed_next_instr + oparg; + int jump_likely = computed_jump_instr == next_instr; uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_likely]; - _Py_CODEUNIT *next_instr = target_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; - _Py_CODEUNIT *false_target = next_instr + oparg; - ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(false_target, old_code)); + ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(jump_likely ? computed_next_instr : computed_jump_instr, old_code)); break; } case JUMP_BACKWARD_JIT: From 02f1fb404e2e9d3e8a6ac58403442ab294fb8813 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 24 Sep 2025 12:22:17 +0100 Subject: [PATCH 026/190] fix non-sstandard C --- Python/optimizer.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/Python/optimizer.c b/Python/optimizer.c index c8a0a3cb74ed58..382a3150793ef3 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -620,15 +620,17 @@ _PyJIT_translate_single_bytecode_to_trace( if (jump_taken || opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO) { unsupported: - // Rewind to previous instruction and replace with _EXIT_TRACE. - _PyUOpInstruction *curr = &trace[trace_length-1]; - while (curr->opcode != _SET_IP && trace_length > 1) { - trace_length--; - curr = &trace[trace_length-1]; - } - assert(curr->opcode == _SET_IP || trace_length == 1); - curr->opcode = _EXIT_TRACE; - goto done; + { + // Rewind to previous instruction and replace with _EXIT_TRACE. + _PyUOpInstruction *curr = &trace[trace_length-1]; + while (curr->opcode != _SET_IP && trace_length > 1) { + trace_length--; + curr = &trace[trace_length-1]; + } + assert(curr->opcode == _SET_IP || trace_length == 1); + curr->opcode = _EXIT_TRACE; + goto done; + } } RESERVE_RAW(expansion->nuops + needs_guard_ip + 3, "uop and various checks"); From dc414a3158b11f49f40f79978a1ffa189a597f96 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 24 Sep 2025 19:21:20 +0100 Subject: [PATCH 027/190] Track from JUMP_BACKWARD rather than FOR_ITER --- Python/bytecodes.c | 2 +- Python/generated_cases.c.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index d8e987708eba7b..770805a98d61ef 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2978,7 +2978,7 @@ dummy_func( DISPATCH(); } } - _PyJIT_InitializeTracing(tstate, frame, next_instr, STACK_LEVEL(), 0, NULL); + _PyJIT_InitializeTracing(tstate, frame, this_instr, STACK_LEVEL(), 0, NULL); ENTER_TRACING(); // Don't add the JUMP_BACKWARD_JIT instruction to the trace. DISPATCH(); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 1633559d72ac12..e190f651fa4c84 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -8400,7 +8400,7 @@ DISPATCH(); } } - _PyJIT_InitializeTracing(tstate, frame, next_instr, STACK_LEVEL(), 0, NULL); + _PyJIT_InitializeTracing(tstate, frame, this_instr, STACK_LEVEL(), 0, NULL); ENTER_TRACING(); DISPATCH(); } @@ -22048,7 +22048,7 @@ TRACING_DISPATCH(); } } - _PyJIT_InitializeTracing(tstate, frame, next_instr, STACK_LEVEL(), 0, NULL); + _PyJIT_InitializeTracing(tstate, frame, this_instr, STACK_LEVEL(), 0, NULL); ENTER_TRACING(); TRACING_DISPATCH(); } From 0ffc2dd39fbf8a6c8e44c5355fa5289ba3da3a65 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 24 Sep 2025 20:34:23 +0100 Subject: [PATCH 028/190] Fix bug where code/func get freed halfway --- Include/internal/pycore_interp_structs.h | 4 ++-- Include/internal/pycore_optimizer.h | 2 ++ Python/ceval_macros.h | 9 ++++----- Python/generated_cases.c.h | 2 ++ Python/optimizer.c | 22 ++++++++++++++++++---- 5 files changed, 28 insertions(+), 11 deletions(-) diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 9671325362d5c3..6296f09ab995e1 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -951,8 +951,8 @@ struct _is { _Py_CODEUNIT *jit_tracer_initial_instr; int jit_tracer_initial_stack_depth; int jit_tracer_initial_chain_depth; - PyCodeObject *jit_tracer_initial_code; // Borrowed - PyFunctionObject *jit_tracer_initial_func; // Borrowed + PyCodeObject *jit_tracer_initial_code; // Strong + PyFunctionObject *jit_tracer_initial_func; // Strong struct _PyExitData *jit_tracer_previous_exit; bool jit_completed_loop; bool jit; diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 79b1931d070467..c7c304943bf87e 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -372,6 +372,8 @@ _PyJIT_translate_single_bytecode_to_trace( void _PyJIT_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit); + +void _PyJIT_FinalizeTracing(PyThreadState *tstate); #ifdef __cplusplus } #endif diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index b15b6788205915..313b062bcc22d8 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -141,7 +141,10 @@ # define LEAVE_TRACING() DISPATCH_TABLE_VAR = DISPATCH_TABLE; # define BAIL_TRACING_NO_DISPATCH() \ LEAVE_TRACING(); \ + _PyFrame_SetStackPointer(frame, stack_pointer); \ int _err = _PyOptimizer_Optimize(frame, tstate); \ + _PyJIT_FinalizeTracing(tstate); \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ if (_err < 0) { \ JUMP_TO_LABEL(error); \ } @@ -149,13 +152,11 @@ BAIL_TRACING_NO_DISPATCH() \ DISPATCH(); # define RECORD_TRACE() do { \ - frame->instr_ptr = next_instr; \ if (add_to_code_trace(tstate, frame, old_code, old_func, this_instr, next_instr, opcode, oparg, _jump_taken)) { \ BAIL_TRACING(); \ } \ } while (0); # define RECORD_TRACE_NO_DISPATCH() do { \ - frame->instr_ptr = next_instr; \ if (add_to_code_trace(tstate, frame, old_code, old_func, this_instr, next_instr, opcode, oparg, _jump_taken)) { \ BAIL_TRACING_NO_DISPATCH(); \ } \ @@ -413,6 +414,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer) #define TIER1_TO_TIER2(EXECUTOR) \ do { \ + LEAVE_TRACING(); \ OPT_STAT_INC(traces_executed); \ next_instr = _Py_jit_entry((EXECUTOR), frame, stack_pointer, tstate); \ frame = tstate->current_frame; \ @@ -428,9 +430,6 @@ do { \ assert(tstate->interp->jit_tracer_code_curr_size == 2); \ ENTER_TRACING(); \ } \ - else { \ - LEAVE_TRACING(); \ - } \ DISPATCH(); \ } while (0) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index e190f651fa4c84..9e855fb5afd573 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -27372,6 +27372,7 @@ JUMP_TO_LABEL(error); LABEL(error) { + assert(!IS_JIT_TRACING()); #ifdef NDEBUG if (!_PyErr_Occurred(tstate)) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -27401,6 +27402,7 @@ JUMP_TO_LABEL(error); LABEL(exception_unwind) { + assert(!IS_JIT_TRACING()); int offset = INSTR_OFFSET()-1; int level, handler, lasti; int handled = get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti); diff --git a/Python/optimizer.c b/Python/optimizer.c index 382a3150793ef3..65bc861a1deed9 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -129,13 +129,18 @@ _PyOptimizer_Optimize( // side-exit chains. We can only insert the executor into the bytecode if // this is true, since a deopt won't infinitely re-enter the executor: bool progress_needed = (chain_depth % MAX_CHAIN_DEPTH) == 0; - PyCodeObject *code = (PyCodeObject *)tstate->interp->jit_tracer_initial_func->func_code; - assert(PyCode_Check(code)); + PyCodeObject *code = (PyCodeObject *)tstate->interp->jit_tracer_initial_code; _Py_CODEUNIT *start = tstate->interp->jit_tracer_initial_instr; if (progress_needed && !has_space_for_executor(code, start)) { interp->compiling = false; return 0; } + // We are the only one still holding a reference to this code object that + // is practically dead. + if (_PyObject_IsUniquelyReferenced(code) || _PyObject_IsUniquelyReferenced(tstate->interp->jit_tracer_initial_func)) { + interp->compiling = false; + return 0; + } _PyExecutorObject *executor; int err = uop_optimize(frame, tstate, &executor, progress_needed); if (err <= 0) { @@ -840,8 +845,8 @@ _PyJIT_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_ tstate->interp->jit_tracer_code_curr_size = 2; tstate->interp->jit_tracer_code_max_size = UOP_MAX_TRACE_LENGTH; tstate->interp->jit_tracer_initial_instr = next_instr; - tstate->interp->jit_tracer_initial_code = code; - tstate->interp->jit_tracer_initial_func = _PyFrame_GetFunction(frame); + tstate->interp->jit_tracer_initial_code = (PyCodeObject *)Py_NewRef(code); + tstate->interp->jit_tracer_initial_func = (PyFunctionObject *)Py_NewRef(_PyFrame_GetFunction(frame)); tstate->interp->jit_tracer_previous_exit = exit; memset(&tstate->interp->jit_tracer_dependencies.bits, 0, sizeof(tstate->interp->jit_tracer_dependencies.bits)); tstate->interp->jit_completed_loop = false; @@ -849,6 +854,15 @@ _PyJIT_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_ tstate->interp->jit_tracer_initial_chain_depth = chain_depth; } +void +_PyJIT_FinalizeTracing(PyThreadState *tstate) +{ + Py_CLEAR(tstate->interp->jit_tracer_initial_code); + Py_CLEAR(tstate->interp->jit_tracer_initial_func); + tstate->interp->jit_tracer_code_curr_size = 2; +} + + #undef RESERVE #undef RESERVE_RAW #undef INSTR_IP From 2032b9c2f7311f966a580f27ff5f9614427393a3 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 9 Oct 2025 15:14:16 +0100 Subject: [PATCH 029/190] add back replaced, move jit tracing env var to --- Include/internal/pycore_interp_structs.h | 1 + Include/internal/pycore_opcode_metadata.h | 2 +- Include/internal/pycore_uop_metadata.h | 4 --- Python/bytecodes.c | 2 +- Python/ceval_macros.h | 23 ++++++-------- Python/executor_cases.c.h | 37 +---------------------- Python/generated_cases.c.h | 2 -- Python/optimizer_cases.c.h | 9 +----- 8 files changed, 15 insertions(+), 65 deletions(-) diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 6296f09ab995e1..af715518c07796 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -944,6 +944,7 @@ struct _is { struct callable_cache callable_cache; PyObject *common_consts[NUM_COMMON_CONSTANTS]; // JIT tracing state + bool jit_is_tracing; int jit_tracer_code_max_size; int jit_tracer_code_curr_size; _PyBloomFilter jit_tracer_dependencies; diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 8f25a9b6336bfd..b96033eadf90b6 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1394,7 +1394,7 @@ _PyOpcode_macro_expansion[256] = { [FORMAT_WITH_SPEC] = { .nuops = 1, .uops = { { _FORMAT_WITH_SPEC, OPARG_SIMPLE, 0 } } }, [FOR_ITER] = { .nuops = 1, .uops = { { _FOR_ITER, OPARG_REPLACED, 0 } } }, [FOR_ITER_GEN] = { .nuops = 3, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _FOR_ITER_GEN_FRAME, OPARG_SIMPLE, 1 }, { _PUSH_FRAME, OPARG_SIMPLE, 1 } } }, - [FOR_ITER_LIST] = { .nuops = 3, .uops = { { _ITER_CHECK_LIST, OPARG_SIMPLE, 1 }, { _ITER_JUMP_LIST, OPARG_REPLACED, 1 }, { _ITER_NEXT_LIST, OPARG_SIMPLE, 1 } } }, + [FOR_ITER_LIST] = { .nuops = 3, .uops = { { _ITER_CHECK_LIST, OPARG_SIMPLE, 1 }, { _ITER_JUMP_LIST, OPARG_REPLACED, 1 }, { _ITER_NEXT_LIST, OPARG_REPLACED, 1 } } }, [FOR_ITER_RANGE] = { .nuops = 3, .uops = { { _ITER_CHECK_RANGE, OPARG_SIMPLE, 1 }, { _ITER_JUMP_RANGE, OPARG_REPLACED, 1 }, { _ITER_NEXT_RANGE, OPARG_SIMPLE, 1 } } }, [FOR_ITER_TUPLE] = { .nuops = 3, .uops = { { _ITER_CHECK_TUPLE, OPARG_SIMPLE, 1 }, { _ITER_JUMP_TUPLE, OPARG_REPLACED, 1 }, { _ITER_NEXT_TUPLE, OPARG_SIMPLE, 1 } } }, [GET_AITER] = { .nuops = 1, .uops = { { _GET_AITER, OPARG_SIMPLE, 0 } } }, diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 1b599091f781fd..79f0b7a68d1f44 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -217,7 +217,6 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_FOR_ITER_TIER_TWO] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_ITER_CHECK_LIST] = HAS_EXIT_FLAG, [_GUARD_NOT_EXHAUSTED_LIST] = HAS_EXIT_FLAG, - [_ITER_NEXT_LIST] = HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_ITER_NEXT_LIST_TIER_TWO] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_ITER_CHECK_TUPLE] = HAS_EXIT_FLAG, [_GUARD_NOT_EXHAUSTED_TUPLE] = HAS_EXIT_FLAG, @@ -522,7 +521,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_ITER_CHECK_LIST] = "_ITER_CHECK_LIST", [_ITER_CHECK_RANGE] = "_ITER_CHECK_RANGE", [_ITER_CHECK_TUPLE] = "_ITER_CHECK_TUPLE", - [_ITER_NEXT_LIST] = "_ITER_NEXT_LIST", [_ITER_NEXT_LIST_TIER_TWO] = "_ITER_NEXT_LIST_TIER_TWO", [_ITER_NEXT_RANGE] = "_ITER_NEXT_RANGE", [_ITER_NEXT_TUPLE] = "_ITER_NEXT_TUPLE", @@ -1073,8 +1071,6 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _GUARD_NOT_EXHAUSTED_LIST: return 0; - case _ITER_NEXT_LIST: - return 0; case _ITER_NEXT_LIST_TIER_TWO: return 0; case _ITER_CHECK_TUPLE: diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 770805a98d61ef..64b56655de296e 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3310,7 +3310,7 @@ dummy_func( #endif } - op(_ITER_NEXT_LIST, (iter, null_or_index -- iter, null_or_index, next)) { + replaced op(_ITER_NEXT_LIST, (iter, null_or_index -- iter, null_or_index, next)) { PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); assert(PyList_CheckExact(list_o)); #ifdef Py_GIL_DISABLED diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 313b062bcc22d8..d76609054c80c3 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -136,9 +136,15 @@ JUMP_TO_LABEL(label); #if _Py_TAIL_CALL_INTERP || USE_COMPUTED_GOTOS -# define IS_JIT_TRACING() (DISPATCH_TABLE_VAR == TRACING_DISPATCH_TABLE) -# define ENTER_TRACING() DISPATCH_TABLE_VAR = TRACING_DISPATCH_TABLE; -# define LEAVE_TRACING() DISPATCH_TABLE_VAR = DISPATCH_TABLE; +# define IS_JIT_TRACING() (tstate->interp->jit_is_tracing) +# define ENTER_TRACING() \ + assert(!IS_JIT_TRACING()); \ + DISPATCH_TABLE_VAR = TRACING_DISPATCH_TABLE; \ + tstate->interp->jit_is_tracing = true; +# define LEAVE_TRACING() \ + assert(IS_JIT_TRACING()); \ + DISPATCH_TABLE_VAR = DISPATCH_TABLE; \ + tstate->interp->jit_is_tracing = false; # define BAIL_TRACING_NO_DISPATCH() \ LEAVE_TRACING(); \ _PyFrame_SetStackPointer(frame, stack_pointer); \ @@ -148,14 +154,6 @@ if (_err < 0) { \ JUMP_TO_LABEL(error); \ } -# define BAIL_TRACING() \ - BAIL_TRACING_NO_DISPATCH() \ - DISPATCH(); -# define RECORD_TRACE() do { \ - if (add_to_code_trace(tstate, frame, old_code, old_func, this_instr, next_instr, opcode, oparg, _jump_taken)) { \ - BAIL_TRACING(); \ - } \ - } while (0); # define RECORD_TRACE_NO_DISPATCH() do { \ if (add_to_code_trace(tstate, frame, old_code, old_func, this_instr, next_instr, opcode, oparg, _jump_taken)) { \ BAIL_TRACING_NO_DISPATCH(); \ @@ -224,7 +222,7 @@ do { \ #define TRACING_DISPATCH() \ { \ assert(frame->stackpointer == NULL); \ - RECORD_TRACE(); \ + RECORD_TRACE_NO_DISPATCH(); \ NEXTOPARG(); \ PRE_DISPATCH_GOTO(); \ DISPATCH_GOTO(); \ @@ -414,7 +412,6 @@ _PyFrame_SetStackPointer(frame, stack_pointer) #define TIER1_TO_TIER2(EXECUTOR) \ do { \ - LEAVE_TRACING(); \ OPT_STAT_INC(traces_executed); \ next_instr = _Py_jit_entry((EXECUTOR), frame, stack_pointer, tstate); \ frame = tstate->current_frame; \ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index b25fb61bd5209d..e9b607639f334c 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4463,42 +4463,7 @@ break; } - case _ITER_NEXT_LIST: { - _PyStackRef null_or_index; - _PyStackRef iter; - _PyStackRef next; - oparg = CURRENT_OPARG(); - null_or_index = stack_pointer[-1]; - iter = stack_pointer[-2]; - PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); - assert(PyList_CheckExact(list_o)); - #ifdef Py_GIL_DISABLED - assert(_Py_IsOwnedByCurrentThread(list_o) || - _PyObject_GC_IS_SHARED(list_o)); - STAT_INC(FOR_ITER, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (result < 0) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - if (result == 0) { - null_or_index = PyStackRef_TagInt(-1); - stack_pointer[-1] = null_or_index; - TIER2_JUMPBY(oparg + 1); - break; - } - #else - next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index))); - #endif - null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); - stack_pointer[-1] = null_or_index; - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - break; - } + /* _ITER_NEXT_LIST is not a viable micro-op for tier 2 because it is replaced */ case _ITER_NEXT_LIST_TIER_TWO: { _PyStackRef null_or_index; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 9e855fb5afd573..e190f651fa4c84 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -27372,7 +27372,6 @@ JUMP_TO_LABEL(error); LABEL(error) { - assert(!IS_JIT_TRACING()); #ifdef NDEBUG if (!_PyErr_Occurred(tstate)) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -27402,7 +27401,6 @@ JUMP_TO_LABEL(error); LABEL(exception_unwind) { - assert(!IS_JIT_TRACING()); int offset = INSTR_OFFSET()-1; int level, handler, lasti; int handled = get_exception_handler(_PyFrame_GetCode(frame), offset, &level, &handler, &lasti); diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 74091012568ce2..ef76374d244f38 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -2223,14 +2223,7 @@ break; } - case _ITER_NEXT_LIST: { - JitOptRef next; - next = sym_new_not_null(ctx); - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - break; - } + /* _ITER_NEXT_LIST is not a viable micro-op for tier 2 */ case _ITER_NEXT_LIST_TIER_TWO: { JitOptRef next; From 299a06858e73bcd7381446ebd3584082b0daeff0 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 9 Oct 2025 18:18:55 +0100 Subject: [PATCH 030/190] Handle recursive tracing and CALL_ALLOC_AND_ENTER_INIT --- Python/bytecodes.c | 5 ++++- Python/ceval_macros.h | 7 +++++-- Python/executor_cases.c.h | 5 ++++- Python/optimizer.c | 10 +++++++++- 4 files changed, 22 insertions(+), 5 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 64b56655de296e..0f9ec19d38f9fe 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5268,6 +5268,7 @@ dummy_func( tier2 op(_EXIT_TRACE, (exit_p/4 --)) { _PyExitData *exit = (_PyExitData *)exit_p; + assert(!exit->is_dynamic); #if defined(Py_DEBUG) && !defined(_Py_JIT) _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); @@ -5281,7 +5282,6 @@ dummy_func( } #endif tstate->jit_exit = exit; - assert(!exit->is_dynamic); TIER2_TO_TIER2(exit->executor); } @@ -5441,6 +5441,9 @@ dummy_func( TIER2_TO_TIER2(exit->executor); } else { + if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) { + GOTO_TIER_ONE(target, 0); + } if (!backoff_counter_triggers(temperature)) { exit->temperature = advance_backoff_counter(temperature); GOTO_TIER_ONE(target, 0); diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index d76609054c80c3..7167b8f4f63b23 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -138,13 +138,15 @@ #if _Py_TAIL_CALL_INTERP || USE_COMPUTED_GOTOS # define IS_JIT_TRACING() (tstate->interp->jit_is_tracing) # define ENTER_TRACING() \ - assert(!IS_JIT_TRACING()); \ DISPATCH_TABLE_VAR = TRACING_DISPATCH_TABLE; \ tstate->interp->jit_is_tracing = true; # define LEAVE_TRACING() \ assert(IS_JIT_TRACING()); \ DISPATCH_TABLE_VAR = DISPATCH_TABLE; \ tstate->interp->jit_is_tracing = false; +// This handles recursive tracing over C calls. +# define RELOAD_TRACING() \ + DISPATCH_TABLE_VAR = tstate->interp->jit_is_tracing ? TRACING_DISPATCH_TABLE : DISPATCH_TABLE; # define BAIL_TRACING_NO_DISPATCH() \ LEAVE_TRACING(); \ _PyFrame_SetStackPointer(frame, stack_pointer); \ @@ -155,7 +157,7 @@ JUMP_TO_LABEL(error); \ } # define RECORD_TRACE_NO_DISPATCH() do { \ - if (add_to_code_trace(tstate, frame, old_code, old_func, this_instr, next_instr, opcode, oparg, _jump_taken)) { \ + if (tstate->interp->jit_is_tracing && add_to_code_trace(tstate, frame, old_code, old_func, this_instr, next_instr, opcode, oparg, _jump_taken)) { \ BAIL_TRACING_NO_DISPATCH(); \ } \ } while (0); @@ -222,6 +224,7 @@ do { \ #define TRACING_DISPATCH() \ { \ assert(frame->stackpointer == NULL); \ + RELOAD_TRACING(); \ RECORD_TRACE_NO_DISPATCH(); \ NEXTOPARG(); \ PRE_DISPATCH_GOTO(); \ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index e9b607639f334c..73eb2692d99933 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7128,6 +7128,7 @@ case _EXIT_TRACE: { PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); _PyExitData *exit = (_PyExitData *)exit_p; + assert(!exit->is_dynamic); #if defined(Py_DEBUG) && !defined(_Py_JIT) _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); @@ -7143,7 +7144,6 @@ } #endif tstate->jit_exit = exit; - assert(!exit->is_dynamic); TIER2_TO_TIER2(exit->executor); break; } @@ -7498,6 +7498,9 @@ TIER2_TO_TIER2(exit->executor); } else { + if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) { + GOTO_TIER_ONE(target, 0); + } if (!backoff_counter_triggers(temperature)) { exit->temperature = advance_backoff_counter(temperature); GOTO_TIER_ONE(target, 0); diff --git a/Python/optimizer.c b/Python/optimizer.c index 65bc861a1deed9..9e5ec472caa43a 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -605,6 +605,10 @@ _PyJIT_translate_single_bytecode_to_trace( return 1; } + if (opcode == JUMP_FORWARD) { + return 1; + } + if (opcode == ENTER_EXECUTOR) { ADD_TO_TRACE(_CHECK_VALIDITY, 0, 0, target); ADD_TO_TRACE(_SET_IP, 0, (uintptr_t)target_instr, target); @@ -623,7 +627,10 @@ _PyJIT_translate_single_bytecode_to_trace( // Strange control-flow, unsupported opcode, etc. if (jump_taken || - opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO) { + opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO || + frame->owner >= FRAME_OWNED_BY_INTERPRETER || + // This can be supported, but requires a tracing shim frame. + opcode == CALL_ALLOC_AND_ENTER_INIT) { unsupported: { // Rewind to previous instruction and replace with _EXIT_TRACE. @@ -814,6 +821,7 @@ _PyJIT_translate_single_bytecode_to_trace( if (!is_terminator(&tstate->interp->jit_tracer_code_buffer[trace_length-1])) { // Undo the last few instructions. trace_length = tstate->interp->jit_tracer_code_curr_size; + max_length = tstate->interp->jit_tracer_code_max_size; // We previously reversed one. max_length += 1; ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); From 8e0fb214060c3c2486d88b0ed26356f14dd9bfe5 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 9 Oct 2025 19:39:45 +0100 Subject: [PATCH 031/190] Fix recursive tracing and dynamic exits --- Python/bytecodes.c | 34 ++++++++++++++++++++++++++++------ Python/ceval.c | 24 +++++++++++++++--------- Python/ceval_macros.h | 11 ++++++++--- Python/executor_cases.c.h | 32 ++++++++++++++++++++++++++------ Python/optimizer.c | 8 +++++++- 5 files changed, 84 insertions(+), 25 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 0f9ec19d38f9fe..15dbab0bef70f9 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5429,8 +5429,7 @@ dummy_func( tier2 op(_COLD_EXIT, ( -- )) { _PyExitData *exit = tstate->jit_exit; assert(exit != NULL); - bool is_dynamic = exit->is_dynamic; - _Py_CODEUNIT *target = is_dynamic ? frame->instr_ptr : (_PyFrame_GetBytecode(frame) + exit->target); + _Py_CODEUNIT *target = (_PyFrame_GetBytecode(frame) + exit->target); _Py_BackoffCounter temperature = exit->temperature; if (target->op.code == ENTER_EXECUTOR) { PyCodeObject *code = _PyFrame_GetCode(frame); @@ -5451,7 +5450,7 @@ dummy_func( exit->temperature = initial_temperature_backoff_counter(); _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); assert(tstate->current_executor == (PyObject *)previous_executor); - int chain_depth = is_dynamic ? 0 : current_executor->vm_data.chain_depth + 1; + int chain_depth = previous_executor->vm_data.chain_depth + 1; _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), chain_depth, exit); GOTO_TIER_ONE(target, 1); } @@ -5461,9 +5460,13 @@ dummy_func( EXIT_IF(frame->instr_ptr != (_Py_CODEUNIT *)ip); } + // Note: this is different than _COLD_EXIT/_EXIT_TRACE, as it may lead to multiple executors + // from a single exit! tier2 op(_DYNAMIC_EXIT, (exit_p/4 --)) { _Py_CODEUNIT *target = frame->instr_ptr; _PyExitData *exit = (_PyExitData *)exit_p; + _Py_BackoffCounter temperature = exit->temperature; + tstate->jit_exit = exit; #if defined(Py_DEBUG) && !defined(_Py_JIT) OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); if (frame->lltrace >= 2) { @@ -5475,9 +5478,28 @@ dummy_func( _PyOpcode_OpName[target->op.code]); } #endif - assert(exit->is_dynamic); - tstate->jit_exit = exit; - TIER2_TO_TIER2(exit->executor); + if (target->op.code == ENTER_EXECUTOR) { + PyCodeObject *code = _PyFrame_GetCode(frame); + _PyExecutorObject *executor = code->co_executors->executors[target->op.arg]; + Py_INCREF(executor); + assert(tstate->jit_exit == exit); + exit->executor = executor; + TIER2_TO_TIER2(exit->executor); + } + else { + if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) { + GOTO_TIER_ONE(target, 0); + } + if (!backoff_counter_triggers(temperature)) { + exit->temperature = advance_backoff_counter(temperature); + GOTO_TIER_ONE(target, 0); + } + exit->temperature = initial_temperature_backoff_counter(); + _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); + assert(tstate->current_executor == (PyObject *)previous_executor); + _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), 0, exit); + GOTO_TIER_ONE(target, 1); + } } label(pop_2_error) { diff --git a/Python/ceval.c b/Python/ceval.c index a6219e5c26d670..bf1616b04bf50b 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1049,6 +1049,12 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int return NULL; } + // We came in already tracing, this means a recursive trace in the form of + // Python -> C -> Python happened. We want to abandon the outer trace then. + if (IS_JIT_TRACING()) { + _PyJIT_FinalizeTracing(tstate); + tstate->interp->jit_is_tracing = false; + } /* Local "register" variables. * These are cached values from the frame and code object. */ _Py_CODEUNIT *next_instr; @@ -1194,15 +1200,15 @@ _PyTier2Interpreter( for (;;) { uopcode = next_uop->opcode; #ifdef Py_DEBUG - if (frame->lltrace >= 4) { - if (next_uop->opcode != _YIELD_VALUE && - next_uop->opcode != _FOR_ITER_GEN_FRAME && - next_uop->opcode != _PUSH_FRAME && - next_uop->opcode != _PY_FRAME_KW && - next_uop->opcode != _SAVE_RETURN_OFFSET && - next_uop->opcode != _SAVE_RETURN_OFFSET) { - dump_stack(frame, stack_pointer); - } + if (frame->lltrace >= 2) { + // if (next_uop->opcode != _YIELD_VALUE && + // next_uop->opcode != _FOR_ITER_GEN_FRAME && + // next_uop->opcode != _PUSH_FRAME && + // next_uop->opcode != _PY_FRAME_KW && + // next_uop->opcode != _SAVE_RETURN_OFFSET && + // next_uop->opcode != _SAVE_RETURN_OFFSET) { + // dump_stack(frame, stack_pointer); + // } if (next_uop->opcode == _START_EXECUTOR) { printf("%4d uop: ", 0); } diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 7167b8f4f63b23..edacf4418b97fd 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -137,12 +137,16 @@ #if _Py_TAIL_CALL_INTERP || USE_COMPUTED_GOTOS # define IS_JIT_TRACING() (tstate->interp->jit_is_tracing) +# define SET_TRACING() \ + DISPATCH_TABLE_VAR = TRACING_DISPATCH_TABLE; # define ENTER_TRACING() \ - DISPATCH_TABLE_VAR = TRACING_DISPATCH_TABLE; \ + SET_TRACING(); \ tstate->interp->jit_is_tracing = true; +# define UNSET_TRACING() \ + DISPATCH_TABLE_VAR = DISPATCH_TABLE; # define LEAVE_TRACING() \ assert(IS_JIT_TRACING()); \ - DISPATCH_TABLE_VAR = DISPATCH_TABLE; \ + UNSET_TRACING(); \ tstate->interp->jit_is_tracing = false; // This handles recursive tracing over C calls. # define RELOAD_TRACING() \ @@ -157,7 +161,7 @@ JUMP_TO_LABEL(error); \ } # define RECORD_TRACE_NO_DISPATCH() do { \ - if (tstate->interp->jit_is_tracing && add_to_code_trace(tstate, frame, old_code, old_func, this_instr, next_instr, opcode, oparg, _jump_taken)) { \ + if (DISPATCH_TABLE_VAR == TRACING_DISPATCH_TABLE && add_to_code_trace(tstate, frame, old_code, old_func, this_instr, next_instr, opcode, oparg, _jump_taken)) { \ BAIL_TRACING_NO_DISPATCH(); \ } \ } while (0); @@ -218,6 +222,7 @@ do { \ } while (0) #define TRACING_DISPATCH_INLINED(NEW_FRAME) \ + RELOAD_TRACING(); \ RECORD_TRACE_NO_DISPATCH(); \ DISPATCH_INLINED(NEW_FRAME); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 73eb2692d99933..86c47b5601048e 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7486,8 +7486,7 @@ case _COLD_EXIT: { _PyExitData *exit = tstate->jit_exit; assert(exit != NULL); - bool is_dynamic = exit->is_dynamic; - _Py_CODEUNIT *target = is_dynamic ? frame->instr_ptr : (_PyFrame_GetBytecode(frame) + exit->target); + _Py_CODEUNIT *target = (_PyFrame_GetBytecode(frame) + exit->target); _Py_BackoffCounter temperature = exit->temperature; if (target->op.code == ENTER_EXECUTOR) { PyCodeObject *code = _PyFrame_GetCode(frame); @@ -7508,7 +7507,7 @@ exit->temperature = initial_temperature_backoff_counter(); _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); assert(tstate->current_executor == (PyObject *)previous_executor); - int chain_depth = is_dynamic ? 0 : current_executor->vm_data.chain_depth + 1; + int chain_depth = previous_executor->vm_data.chain_depth + 1; _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), chain_depth, exit); GOTO_TIER_ONE(target, 1); } @@ -7528,6 +7527,8 @@ PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); _Py_CODEUNIT *target = frame->instr_ptr; _PyExitData *exit = (_PyExitData *)exit_p; + _Py_BackoffCounter temperature = exit->temperature; + tstate->jit_exit = exit; #if defined(Py_DEBUG) && !defined(_Py_JIT) OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); if (frame->lltrace >= 2) { @@ -7541,9 +7542,28 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } #endif - assert(exit->is_dynamic); - tstate->jit_exit = exit; - TIER2_TO_TIER2(exit->executor); + if (target->op.code == ENTER_EXECUTOR) { + PyCodeObject *code = _PyFrame_GetCode(frame); + _PyExecutorObject *executor = code->co_executors->executors[target->op.arg]; + Py_INCREF(executor); + assert(tstate->jit_exit == exit); + exit->executor = executor; + TIER2_TO_TIER2(exit->executor); + } + else { + if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) { + GOTO_TIER_ONE(target, 0); + } + if (!backoff_counter_triggers(temperature)) { + exit->temperature = advance_backoff_counter(temperature); + GOTO_TIER_ONE(target, 0); + } + exit->temperature = initial_temperature_backoff_counter(); + _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); + assert(tstate->current_executor == (PyObject *)previous_executor); + _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), 0, exit); + GOTO_TIER_ONE(target, 1); + } break; } diff --git a/Python/optimizer.c b/Python/optimizer.c index 9e5ec472caa43a..03cd070c163e49 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -131,6 +131,11 @@ _PyOptimizer_Optimize( bool progress_needed = (chain_depth % MAX_CHAIN_DEPTH) == 0; PyCodeObject *code = (PyCodeObject *)tstate->interp->jit_tracer_initial_code; _Py_CODEUNIT *start = tstate->interp->jit_tracer_initial_instr; + // A recursive trace might've cleared the values. In that case, bail. + if (code == NULL) { + interp->compiling = false; + return 0; + } if (progress_needed && !has_space_for_executor(code, start)) { interp->compiling = false; return 0; @@ -868,6 +873,7 @@ _PyJIT_FinalizeTracing(PyThreadState *tstate) Py_CLEAR(tstate->interp->jit_tracer_initial_code); Py_CLEAR(tstate->interp->jit_tracer_initial_func); tstate->interp->jit_tracer_code_curr_size = 2; + tstate->interp->jit_tracer_code_max_size = UOP_MAX_TRACE_LENGTH - 1; } @@ -1197,7 +1203,7 @@ uop_optimize( int curr_stackentries = tstate->interp->jit_tracer_initial_stack_depth; int length = interp->jit_tracer_code_curr_size; // Trace too short, don't bother. - if (length <= 4) { + if (length <= 5) { return 0; } assert(length > 0); From 95eee89121e0db3bf230f58c67c079cfaf2ada76 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 9 Oct 2025 20:54:43 +0100 Subject: [PATCH 032/190] Fix handling of EXTENDED_ARG --- Python/bytecodes.c | 9 +++------ Python/ceval_macros.h | 4 +++- Python/executor_cases.c.h | 9 +++------ Python/optimizer.c | 14 +++----------- 4 files changed, 12 insertions(+), 24 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 15dbab0bef70f9..812b323c137b2b 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5466,7 +5466,6 @@ dummy_func( _Py_CODEUNIT *target = frame->instr_ptr; _PyExitData *exit = (_PyExitData *)exit_p; _Py_BackoffCounter temperature = exit->temperature; - tstate->jit_exit = exit; #if defined(Py_DEBUG) && !defined(_Py_JIT) OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); if (frame->lltrace >= 2) { @@ -5481,10 +5480,8 @@ dummy_func( if (target->op.code == ENTER_EXECUTOR) { PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = code->co_executors->executors[target->op.arg]; - Py_INCREF(executor); - assert(tstate->jit_exit == exit); - exit->executor = executor; - TIER2_TO_TIER2(exit->executor); + tstate->jit_exit = NULL; + TIER2_TO_TIER2(executor); } else { if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) { @@ -5497,7 +5494,7 @@ dummy_func( exit->temperature = initial_temperature_backoff_counter(); _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); assert(tstate->current_executor == (PyObject *)previous_executor); - _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), 0, exit); + _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), 0, NULL); GOTO_TIER_ONE(target, 1); } } diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index edacf4418b97fd..8085fb3f192806 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -133,6 +133,8 @@ #define TRACING_JUMP_TO_LABEL(label) \ RECORD_JUMP_TAKEN() \ RECORD_TRACE_NO_DISPATCH() \ + assert(!IS_JIT_TRACING()); \ + RELOAD_TRACING(); \ JUMP_TO_LABEL(label); #if _Py_TAIL_CALL_INTERP || USE_COMPUTED_GOTOS @@ -161,7 +163,7 @@ JUMP_TO_LABEL(error); \ } # define RECORD_TRACE_NO_DISPATCH() do { \ - if (DISPATCH_TABLE_VAR == TRACING_DISPATCH_TABLE && add_to_code_trace(tstate, frame, old_code, old_func, this_instr, next_instr, opcode, oparg, _jump_taken)) { \ + if (IS_JIT_TRACING() && add_to_code_trace(tstate, frame, old_code, old_func, this_instr, next_instr, opcode, oparg, _jump_taken)) { \ BAIL_TRACING_NO_DISPATCH(); \ } \ } while (0); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 86c47b5601048e..58c57ea6b96fa8 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7528,7 +7528,6 @@ _Py_CODEUNIT *target = frame->instr_ptr; _PyExitData *exit = (_PyExitData *)exit_p; _Py_BackoffCounter temperature = exit->temperature; - tstate->jit_exit = exit; #if defined(Py_DEBUG) && !defined(_Py_JIT) OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); if (frame->lltrace >= 2) { @@ -7545,10 +7544,8 @@ if (target->op.code == ENTER_EXECUTOR) { PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = code->co_executors->executors[target->op.arg]; - Py_INCREF(executor); - assert(tstate->jit_exit == exit); - exit->executor = executor; - TIER2_TO_TIER2(exit->executor); + tstate->jit_exit = NULL; + TIER2_TO_TIER2(executor); } else { if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) { @@ -7561,7 +7558,7 @@ exit->temperature = initial_temperature_backoff_counter(); _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); assert(tstate->current_executor == (PyObject *)previous_executor); - _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), 0, exit); + _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), 0, NULL); GOTO_TIER_ONE(target, 1); } break; diff --git a/Python/optimizer.c b/Python/optimizer.c index 03cd070c163e49..a8df28cd2a86f4 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -468,7 +468,6 @@ is_for_iter_test[MAX_UOP_ID + 1] = { [_GUARD_NOT_EXHAUSTED_RANGE] = 1, [_GUARD_NOT_EXHAUSTED_LIST] = 1, [_GUARD_NOT_EXHAUSTED_TUPLE] = 1, - [_FOR_ITER_TIER_TWO] = 1, }; static const uint16_t @@ -586,18 +585,11 @@ _PyJIT_translate_single_bytecode_to_trace( DPRINTF(2, "%d: %s(%d)\n", target, _PyOpcode_OpName[opcode], oparg); - // One for possible _DEOPT, one because _CHECK_VALIDITY itself might _DEOPT - max_length -= 2; if ((uint16_t)oparg != (uint64_t)oparg) { - // Back up over EXTENDED_ARGs - _PyUOpInstruction *curr = &trace[trace_length-1]; - while (oparg > 0) { - oparg >>= 8; - trace_length--; - } - goto full; + goto unsupported; } - + // One for possible _DEOPT, one because _CHECK_VALIDITY itself might _DEOPT + max_length -= 2; if (opcode == EXTENDED_ARG) { return 1; From 6936a38478b4ef576352b27d0674bb4ecd0cb9e9 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 16 Oct 2025 21:59:01 +0100 Subject: [PATCH 033/190] Just punt on large opargs for now --- Python/bytecodes.c | 6 ++++-- Python/executor_cases.c.h | 5 +++-- Python/optimizer.c | 8 +++++--- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 812b323c137b2b..5ab80565d6d486 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5407,9 +5407,11 @@ dummy_func( } tier2 op(_ERROR_POP_N, (target/2 --)) { - assert(target != 0); assert(oparg == 0); - frame->instr_ptr = _PyFrame_GetBytecode(frame) + target; + _Py_CODEUNIT *current_instr = _PyFrame_GetBytecode(frame) + target; + _Py_CODEUNIT *next_instr = current_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[current_instr->op.code]]; + // gh-140104: The exception handler expects frame->instr_ptr to be pointing to next_instr, not this_instr! + frame->instr_ptr = next_instr; SYNC_SP(); GOTO_TIER_ONE(NULL, 0); } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 58c57ea6b96fa8..f13baadd3bc76f 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7459,9 +7459,10 @@ case _ERROR_POP_N: { oparg = CURRENT_OPARG(); uint32_t target = (uint32_t)CURRENT_OPERAND0(); - assert(target != 0); assert(oparg == 0); - frame->instr_ptr = _PyFrame_GetBytecode(frame) + target; + _Py_CODEUNIT *current_instr = _PyFrame_GetBytecode(frame) + target; + _Py_CODEUNIT *next_instr = current_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[current_instr->op.code]]; + frame->instr_ptr = next_instr; GOTO_TIER_ONE(NULL, 0); break; } diff --git a/Python/optimizer.c b/Python/optimizer.c index a8df28cd2a86f4..e4b29124d17a14 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -585,11 +585,10 @@ _PyJIT_translate_single_bytecode_to_trace( DPRINTF(2, "%d: %s(%d)\n", target, _PyOpcode_OpName[opcode], oparg); - if ((uint16_t)oparg != (uint64_t)oparg) { + // TODO support EXTENDED_ARG + if (oparg > 255) { goto unsupported; } - // One for possible _DEOPT, one because _CHECK_VALIDITY itself might _DEOPT - max_length -= 2; if (opcode == EXTENDED_ARG) { return 1; @@ -606,6 +605,9 @@ _PyJIT_translate_single_bytecode_to_trace( return 1; } + // One for possible _DEOPT, one because _CHECK_VALIDITY itself might _DEOPT + max_length -= 2; + if (opcode == ENTER_EXECUTOR) { ADD_TO_TRACE(_CHECK_VALIDITY, 0, 0, target); ADD_TO_TRACE(_SET_IP, 0, (uintptr_t)target_instr, target); From a274451e052225781879991d833893eb805e63ec Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 16 Oct 2025 22:01:51 +0100 Subject: [PATCH 034/190] cleanup a little --- Python/optimizer.c | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/Python/optimizer.c b/Python/optimizer.c index e4b29124d17a14..532881aef92fd6 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -585,14 +585,6 @@ _PyJIT_translate_single_bytecode_to_trace( DPRINTF(2, "%d: %s(%d)\n", target, _PyOpcode_OpName[opcode], oparg); - // TODO support EXTENDED_ARG - if (oparg > 255) { - goto unsupported; - } - - if (opcode == EXTENDED_ARG) { - return 1; - } if (opcode == NOP) { return 1; } @@ -626,6 +618,9 @@ _PyJIT_translate_single_bytecode_to_trace( // Strange control-flow, unsupported opcode, etc. if (jump_taken || + // TODO handle extended args. + oparg > 255 || + opcode == EXTENDED_ARG || opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO || frame->owner >= FRAME_OWNED_BY_INTERPRETER || // This can be supported, but requires a tracing shim frame. From f55129ee4281ac93296a0b246bc48f0f6d9d2579 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 17 Oct 2025 00:24:54 +0100 Subject: [PATCH 035/190] fix recursive tracing --- Include/internal/pycore_interp_structs.h | 4 +- Python/ceval.c | 35 ++++++----- Python/ceval_macros.h | 19 +----- Python/optimizer.c | 74 +++++++++++++----------- 4 files changed, 65 insertions(+), 67 deletions(-) diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index af715518c07796..e3264035ed9b4e 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -944,7 +944,6 @@ struct _is { struct callable_cache callable_cache; PyObject *common_consts[NUM_COMMON_CONSTANTS]; // JIT tracing state - bool jit_is_tracing; int jit_tracer_code_max_size; int jit_tracer_code_curr_size; _PyBloomFilter jit_tracer_dependencies; @@ -955,7 +954,8 @@ struct _is { PyCodeObject *jit_tracer_initial_code; // Strong PyFunctionObject *jit_tracer_initial_func; // Strong struct _PyExitData *jit_tracer_previous_exit; - bool jit_completed_loop; + _PyInterpreterFrame *jit_tracer_current_frame; + // End Jit tracing state bool jit; bool compiling; struct _PyUOpInstruction *jit_uop_buffer; diff --git a/Python/ceval.c b/Python/ceval.c index bf1616b04bf50b..c08e5830744eac 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1049,12 +1049,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int return NULL; } - // We came in already tracing, this means a recursive trace in the form of - // Python -> C -> Python happened. We want to abandon the outer trace then. - if (IS_JIT_TRACING()) { - _PyJIT_FinalizeTracing(tstate); - tstate->interp->jit_is_tracing = false; - } /* Local "register" variables. * These are cached values from the frame and code object. */ _Py_CODEUNIT *next_instr; @@ -1201,14 +1195,27 @@ _PyTier2Interpreter( uopcode = next_uop->opcode; #ifdef Py_DEBUG if (frame->lltrace >= 2) { - // if (next_uop->opcode != _YIELD_VALUE && - // next_uop->opcode != _FOR_ITER_GEN_FRAME && - // next_uop->opcode != _PUSH_FRAME && - // next_uop->opcode != _PY_FRAME_KW && - // next_uop->opcode != _SAVE_RETURN_OFFSET && - // next_uop->opcode != _SAVE_RETURN_OFFSET) { - // dump_stack(frame, stack_pointer); - // } + if (next_uop->opcode != _YIELD_VALUE && + next_uop->opcode != _FOR_ITER_GEN_FRAME && + next_uop->opcode != _PUSH_FRAME && + next_uop->opcode != _PY_FRAME_KW && + next_uop->opcode != _SAVE_RETURN_OFFSET && + next_uop->opcode != _SAVE_RETURN_OFFSET) { + if (next_uop->opcode != _START_EXECUTOR) { + if (next_uop->format == UOP_FORMAT_TARGET) { + _Py_CODEUNIT *aim = _PyFrame_GetBytecode(frame) + next_uop->target; + printf(" aim=[%s]\n", _PyOpcode_OpName[aim->op.code]); + } + else if (next_uop->format == UOP_FORMAT_JUMP) { + _PyUOpInstruction *aim_uop = current_executor->trace + next_uop->jump_target; + if (aim_uop->format == UOP_FORMAT_TARGET) { + _Py_CODEUNIT *aim = _PyFrame_GetBytecode(frame) + aim_uop->target; + printf(" aim=[%s]\n", _PyOpcode_OpName[aim->op.code]); + } + } + } + dump_stack(frame, stack_pointer); + } if (next_uop->opcode == _START_EXECUTOR) { printf("%4d uop: ", 0); } diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 8085fb3f192806..ff038026f9770a 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -134,25 +134,14 @@ RECORD_JUMP_TAKEN() \ RECORD_TRACE_NO_DISPATCH() \ assert(!IS_JIT_TRACING()); \ - RELOAD_TRACING(); \ JUMP_TO_LABEL(label); #if _Py_TAIL_CALL_INTERP || USE_COMPUTED_GOTOS -# define IS_JIT_TRACING() (tstate->interp->jit_is_tracing) -# define SET_TRACING() \ - DISPATCH_TABLE_VAR = TRACING_DISPATCH_TABLE; +# define IS_JIT_TRACING() (DISPATCH_TABLE_VAR == TRACING_DISPATCH_TABLE) # define ENTER_TRACING() \ - SET_TRACING(); \ - tstate->interp->jit_is_tracing = true; -# define UNSET_TRACING() \ - DISPATCH_TABLE_VAR = DISPATCH_TABLE; + DISPATCH_TABLE_VAR = TRACING_DISPATCH_TABLE; # define LEAVE_TRACING() \ - assert(IS_JIT_TRACING()); \ - UNSET_TRACING(); \ - tstate->interp->jit_is_tracing = false; -// This handles recursive tracing over C calls. -# define RELOAD_TRACING() \ - DISPATCH_TABLE_VAR = tstate->interp->jit_is_tracing ? TRACING_DISPATCH_TABLE : DISPATCH_TABLE; + DISPATCH_TABLE_VAR = DISPATCH_TABLE; # define BAIL_TRACING_NO_DISPATCH() \ LEAVE_TRACING(); \ _PyFrame_SetStackPointer(frame, stack_pointer); \ @@ -224,14 +213,12 @@ do { \ } while (0) #define TRACING_DISPATCH_INLINED(NEW_FRAME) \ - RELOAD_TRACING(); \ RECORD_TRACE_NO_DISPATCH(); \ DISPATCH_INLINED(NEW_FRAME); #define TRACING_DISPATCH() \ { \ assert(frame->stackpointer == NULL); \ - RELOAD_TRACING(); \ RECORD_TRACE_NO_DISPATCH(); \ NEXTOPARG(); \ PRE_DISPATCH_GOTO(); \ diff --git a/Python/optimizer.c b/Python/optimizer.c index 532881aef92fd6..ed0e92bceb9c5f 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -562,7 +562,8 @@ _PyJIT_translate_single_bytecode_to_trace( if (Py_IsNone((PyObject *)func)) { func = NULL; } - int is_first_instr = tstate->interp->jit_tracer_initial_instr == this_instr ; + + int is_first_instr = tstate->interp->jit_tracer_initial_instr == this_instr; bool progress_needed = (tstate->interp->jit_tracer_initial_chain_depth % MAX_CHAIN_DEPTH) == 0 && is_first_instr;; _PyBloomFilter *dependencies = &tstate->interp->jit_tracer_dependencies; _Py_BloomFilter_Add(dependencies, old_code); @@ -583,7 +584,39 @@ _PyJIT_translate_single_bytecode_to_trace( target = INSTR_IP(target_instr, old_code); - DPRINTF(2, "%d: %s(%d)\n", target, _PyOpcode_OpName[opcode], oparg); + bool needs_guard_ip = _PyOpcode_NeedsGuardIp[opcode] && + !(opcode == FOR_ITER_RANGE || opcode == FOR_ITER_LIST || opcode == FOR_ITER_TUPLE) && + !(opcode == JUMP_BACKWARD_NO_INTERRUPT || opcode == JUMP_BACKWARD || opcode == JUMP_BACKWARD_JIT) && + !(opcode == POP_JUMP_IF_TRUE || opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_NONE || opcode == POP_JUMP_IF_NOT_NONE); + + // Strange control-flow, unsupported opcode, etc. + if (jump_taken || + // This happens when a recursive call happens that we can't trace. Such as Python -> C -> Python calls + // If we haven't guarded the IP, then it's untraceable. + (frame != tstate->interp->jit_tracer_current_frame && !needs_guard_ip) || + // TODO handle extended args. + oparg > 255 || opcode == EXTENDED_ARG || + opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO || + frame->owner >= FRAME_OWNED_BY_INTERPRETER || + // This can be supported, but requires a tracing shim frame. + opcode == CALL_ALLOC_AND_ENTER_INIT) { + unsupported: + { + // Rewind to previous instruction and replace with _EXIT_TRACE. + _PyUOpInstruction *curr = &trace[trace_length-1]; + while (curr->opcode != _SET_IP && trace_length > 1) { + trace_length--; + curr = &trace[trace_length-1]; + } + assert(curr->opcode == _SET_IP || trace_length == 1); + curr->opcode = _EXIT_TRACE; + goto done; + } + } + + tstate->interp->jit_tracer_current_frame = frame; + + DPRINTF(2, "%p %d: %s(%d)\n", old_code, target, _PyOpcode_OpName[opcode], oparg); if (opcode == NOP) { return 1; @@ -601,47 +634,18 @@ _PyJIT_translate_single_bytecode_to_trace( max_length -= 2; if (opcode == ENTER_EXECUTOR) { - ADD_TO_TRACE(_CHECK_VALIDITY, 0, 0, target); - ADD_TO_TRACE(_SET_IP, 0, (uintptr_t)target_instr, target); - ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); goto full; } - bool needs_guard_ip = _PyOpcode_NeedsGuardIp[opcode] && - !(opcode == FOR_ITER_RANGE || opcode == FOR_ITER_LIST || opcode == FOR_ITER_TUPLE) && - !(opcode == JUMP_BACKWARD_NO_INTERRUPT || opcode == JUMP_BACKWARD || opcode == JUMP_BACKWARD_JIT) && - !(opcode == POP_JUMP_IF_TRUE || opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_NONE || opcode == POP_JUMP_IF_NOT_NONE); - - assert(opcode != ENTER_EXECUTOR && opcode != EXTENDED_ARG); - const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; - // Strange control-flow, unsupported opcode, etc. - if (jump_taken || - // TODO handle extended args. - oparg > 255 || - opcode == EXTENDED_ARG || - opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO || - frame->owner >= FRAME_OWNED_BY_INTERPRETER || - // This can be supported, but requires a tracing shim frame. - opcode == CALL_ALLOC_AND_ENTER_INIT) { - unsupported: - { - // Rewind to previous instruction and replace with _EXIT_TRACE. - _PyUOpInstruction *curr = &trace[trace_length-1]; - while (curr->opcode != _SET_IP && trace_length > 1) { - trace_length--; - curr = &trace[trace_length-1]; - } - assert(curr->opcode == _SET_IP || trace_length == 1); - curr->opcode = _EXIT_TRACE; - goto done; - } - } RESERVE_RAW(expansion->nuops + needs_guard_ip + 3, "uop and various checks"); ADD_TO_TRACE(_CHECK_VALIDITY, 0, 0, target); + assert(opcode != ENTER_EXECUTOR && opcode != EXTENDED_ARG); + assert(!_PyErr_Occurred(tstate)); + if (!OPCODE_HAS_NO_SAVE_IP(opcode)) { ADD_TO_TRACE(_SET_IP, 0, (uintptr_t)target_instr, target); } @@ -851,9 +855,9 @@ _PyJIT_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_ tstate->interp->jit_tracer_initial_func = (PyFunctionObject *)Py_NewRef(_PyFrame_GetFunction(frame)); tstate->interp->jit_tracer_previous_exit = exit; memset(&tstate->interp->jit_tracer_dependencies.bits, 0, sizeof(tstate->interp->jit_tracer_dependencies.bits)); - tstate->interp->jit_completed_loop = false; tstate->interp->jit_tracer_initial_stack_depth = curr_stackdepth; tstate->interp->jit_tracer_initial_chain_depth = chain_depth; + tstate->interp->jit_tracer_current_frame = frame; } void From 71bd27be7229f5e42aadb2c0309c7e34c713a5fd Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 17 Oct 2025 01:23:33 +0100 Subject: [PATCH 036/190] comment out debugging --- Python/ceval.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index c08e5830744eac..035a22615ca961 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1201,19 +1201,19 @@ _PyTier2Interpreter( next_uop->opcode != _PY_FRAME_KW && next_uop->opcode != _SAVE_RETURN_OFFSET && next_uop->opcode != _SAVE_RETURN_OFFSET) { - if (next_uop->opcode != _START_EXECUTOR) { - if (next_uop->format == UOP_FORMAT_TARGET) { - _Py_CODEUNIT *aim = _PyFrame_GetBytecode(frame) + next_uop->target; - printf(" aim=[%s]\n", _PyOpcode_OpName[aim->op.code]); - } - else if (next_uop->format == UOP_FORMAT_JUMP) { - _PyUOpInstruction *aim_uop = current_executor->trace + next_uop->jump_target; - if (aim_uop->format == UOP_FORMAT_TARGET) { - _Py_CODEUNIT *aim = _PyFrame_GetBytecode(frame) + aim_uop->target; - printf(" aim=[%s]\n", _PyOpcode_OpName[aim->op.code]); - } - } - } + // if (next_uop->opcode != _START_EXECUTOR) { + // if (next_uop->format == UOP_FORMAT_TARGET) { + // _Py_CODEUNIT *aim = _PyFrame_GetBytecode(frame) + next_uop->target; + // printf(" aim=[%s]\n", _PyOpcode_OpName[aim->op.code]); + // } + // else if (next_uop->format == UOP_FORMAT_JUMP) { + // _PyUOpInstruction *aim_uop = current_executor->trace + next_uop->jump_target; + // if (aim_uop->format == UOP_FORMAT_TARGET) { + // _Py_CODEUNIT *aim = _PyFrame_GetBytecode(frame) + aim_uop->target; + // printf(" aim=[%s]\n", _PyOpcode_OpName[aim->op.code]); + // } + // } + // } dump_stack(frame, stack_pointer); } if (next_uop->opcode == _START_EXECUTOR) { From e834c8817adc925559c62cd75a5d771b5642c21d Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 17 Oct 2025 01:57:11 +0100 Subject: [PATCH 037/190] Delete out.txt --- out.txt | 1089 ------------------------------------------------------- 1 file changed, 1089 deletions(-) delete mode 100644 out.txt diff --git a/out.txt b/out.txt deleted file mode 100644 index 225b73b0f38765..00000000000000 --- a/out.txt +++ /dev/null @@ -1,1089 +0,0 @@ -0 ADD_TO_BYTECODE_TRACE: FOR_ITER_GEN 3 -2 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -7 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5565a -12 ADD_TO_BYTECODE_TRACE: RESUME_CHECK 2 -13 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5565c -18 ADD_TO_BYTECODE_TRACE: JUMP_BACKWARD_NO_INTERRUPT 5 -19 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -24 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55654 -29 ADD_TO_BYTECODE_TRACE: SEND_GEN 3 -31 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -36 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5565a -41 ADD_TO_BYTECODE_TRACE: RESUME_CHECK 2 -42 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5565c -47 ADD_TO_BYTECODE_TRACE: JUMP_BACKWARD_NO_INTERRUPT 5 -48 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -53 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55654 -58 ADD_TO_BYTECODE_TRACE: SEND_GEN 3 -60 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -65 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5565a -70 ADD_TO_BYTECODE_TRACE: RESUME_CHECK 2 -71 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5565c -76 ADD_TO_BYTECODE_TRACE: JUMP_BACKWARD_NO_INTERRUPT 5 -77 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -82 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55654 -87 ADD_TO_BYTECODE_TRACE: SEND_GEN 3 -89 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -94 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5565a -99 ADD_TO_BYTECODE_TRACE: RESUME_CHECK 2 -100 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5565c -105 ADD_TO_BYTECODE_TRACE: JUMP_BACKWARD_NO_INTERRUPT 5 -106 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -111 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55654 -116 ADD_TO_BYTECODE_TRACE: SEND_GEN 3 -118 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -123 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556c2 -128 ADD_TO_BYTECODE_TRACE: RESUME_CHECK 2 -129 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556c4 -134 ADD_TO_BYTECODE_TRACE: JUMP_BACKWARD_NO_INTERRUPT 5 -135 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -140 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556bc -145 ADD_TO_BYTECODE_TRACE: SEND_GEN 3 -147 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -152 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5565a -157 ADD_TO_BYTECODE_TRACE: RESUME_CHECK 2 -158 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5565c -163 ADD_TO_BYTECODE_TRACE: JUMP_BACKWARD_NO_INTERRUPT 5 -164 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -169 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55654 -174 ADD_TO_BYTECODE_TRACE: SEND_GEN 3 -176 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -181 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556c2 -186 ADD_TO_BYTECODE_TRACE: RESUME_CHECK 2 -187 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556c4 -192 ADD_TO_BYTECODE_TRACE: JUMP_BACKWARD_NO_INTERRUPT 5 -193 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -198 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556bc -203 ADD_TO_BYTECODE_TRACE: SEND_GEN 3 -205 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -210 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5565a -215 ADD_TO_BYTECODE_TRACE: RESUME_CHECK 2 -216 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5565c -221 ADD_TO_BYTECODE_TRACE: JUMP_BACKWARD_NO_INTERRUPT 5 -222 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -227 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55654 -232 ADD_TO_BYTECODE_TRACE: SEND_GEN 3 -234 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -239 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5565a -244 ADD_TO_BYTECODE_TRACE: RESUME_CHECK 2 -245 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5565c -250 ADD_TO_BYTECODE_TRACE: JUMP_BACKWARD_NO_INTERRUPT 5 -251 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -256 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55654 -261 ADD_TO_BYTECODE_TRACE: SEND_GEN 3 -263 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -268 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556c2 -273 ADD_TO_BYTECODE_TRACE: RESUME_CHECK 2 -274 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556c4 -279 ADD_TO_BYTECODE_TRACE: JUMP_BACKWARD_NO_INTERRUPT 5 -280 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -285 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556bc -290 ADD_TO_BYTECODE_TRACE: SEND_GEN 3 -292 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -297 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556c2 -302 ADD_TO_BYTECODE_TRACE: RESUME_CHECK 2 -303 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556c4 -308 ADD_TO_BYTECODE_TRACE: JUMP_BACKWARD_NO_INTERRUPT 5 -309 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -314 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556bc -319 ADD_TO_BYTECODE_TRACE: SEND_GEN 3 -321 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -326 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556c2 -331 ADD_TO_BYTECODE_TRACE: RESUME_CHECK 2 -332 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556c4 -337 ADD_TO_BYTECODE_TRACE: JUMP_BACKWARD_NO_INTERRUPT 5 -338 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -343 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556bc -348 ADD_TO_BYTECODE_TRACE: SEND_GEN 3 -350 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -355 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556c2 -360 ADD_TO_BYTECODE_TRACE: RESUME_CHECK 2 -361 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556c4 -366 ADD_TO_BYTECODE_TRACE: JUMP_BACKWARD_NO_INTERRUPT 5 -367 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -372 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556bc -377 ADD_TO_BYTECODE_TRACE: SEND_GEN 3 -379 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -384 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5565a -389 ADD_TO_BYTECODE_TRACE: RESUME_CHECK 2 -390 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5565c -395 ADD_TO_BYTECODE_TRACE: JUMP_BACKWARD_NO_INTERRUPT 5 -396 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -401 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55654 -406 ADD_TO_BYTECODE_TRACE: SEND_GEN 3 -408 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -413 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556c2 -418 ADD_TO_BYTECODE_TRACE: RESUME_CHECK 2 -419 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556c4 -424 ADD_TO_BYTECODE_TRACE: JUMP_BACKWARD_NO_INTERRUPT 5 -425 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -430 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556bc -435 ADD_TO_BYTECODE_TRACE: SEND_GEN 3 -437 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -442 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5565a -447 ADD_TO_BYTECODE_TRACE: RESUME_CHECK 2 -448 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5565c -453 ADD_TO_BYTECODE_TRACE: JUMP_BACKWARD_NO_INTERRUPT 5 -454 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -459 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55654 -464 ADD_TO_BYTECODE_TRACE: SEND_GEN 3 -466 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -471 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5567a -476 ADD_TO_BYTECODE_TRACE: RESUME_CHECK 5 -477 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5567c -482 ADD_TO_BYTECODE_TRACE: POP_TOP 0 -483 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5567e -488 ADD_TO_BYTECODE_TRACE: LOAD_FAST_BORROW 0 -489 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55680 -494 ADD_TO_BYTECODE_TRACE: LOAD_ATTR_INSTANCE_VALUE 4 -504 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55694 -509 ADD_TO_BYTECODE_TRACE: TO_BOOL_NONE 0 -513 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5569c -518 ADD_TO_BYTECODE_TRACE: POP_JUMP_IF_FALSE 23 -520 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -525 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556ce -530 ADD_TO_BYTECODE_TRACE: LOAD_CONST 0 -531 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556d0 -536 ADD_TO_BYTECODE_TRACE: RETURN_VALUE 0 -537 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -542 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5565e -547 ADD_TO_BYTECODE_TRACE: END_SEND 0 -548 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55660 -553 ADD_TO_BYTECODE_TRACE: POP_TOP 0 -554 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55662 -559 ADD_TO_BYTECODE_TRACE: LOAD_FAST_BORROW 0 -560 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55664 -565 ADD_TO_BYTECODE_TRACE: LOAD_ATTR_INSTANCE_VALUE 2 -575 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55678 -580 ADD_TO_BYTECODE_TRACE: YIELD_VALUE 0 -581 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -586 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556c0 -591 ADD_TO_BYTECODE_TRACE: YIELD_VALUE 1 -592 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -597 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55658 -602 ADD_TO_BYTECODE_TRACE: YIELD_VALUE 1 -603 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -608 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556c0 -613 ADD_TO_BYTECODE_TRACE: YIELD_VALUE 1 -614 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -619 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556c0 -624 ADD_TO_BYTECODE_TRACE: YIELD_VALUE 1 -625 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -630 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556c0 -635 ADD_TO_BYTECODE_TRACE: YIELD_VALUE 1 -636 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -641 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556c0 -646 ADD_TO_BYTECODE_TRACE: YIELD_VALUE 1 -647 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -652 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55658 -657 ADD_TO_BYTECODE_TRACE: YIELD_VALUE 1 -658 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -663 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55658 -668 ADD_TO_BYTECODE_TRACE: YIELD_VALUE 1 -669 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -674 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556c0 -679 ADD_TO_BYTECODE_TRACE: YIELD_VALUE 1 -680 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -685 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55658 -690 ADD_TO_BYTECODE_TRACE: YIELD_VALUE 1 -691 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -696 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db556c0 -701 ADD_TO_BYTECODE_TRACE: YIELD_VALUE 1 -702 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -707 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55658 -712 ADD_TO_BYTECODE_TRACE: YIELD_VALUE 1 -713 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -718 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55658 -723 ADD_TO_BYTECODE_TRACE: YIELD_VALUE 1 -724 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -729 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55658 -734 ADD_TO_BYTECODE_TRACE: YIELD_VALUE 1 -735 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -740 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db55658 -745 ADD_TO_BYTECODE_TRACE: YIELD_VALUE 1 -746 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -751 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5630e -756 ADD_TO_BYTECODE_TRACE: STORE_FAST 3 -757 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db56310 -762 ADD_TO_BYTECODE_TRACE: JUMP_BACKWARD_JIT 5 -764 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -769 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5630a -FOR_ITER_GEN(3) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -RESUME_CHECK(2) -TIER1_SET_IP(0) -JUMP_BACKWARD_NO_INTERRUPT(5) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -SEND_GEN(3) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -RESUME_CHECK(2) -TIER1_SET_IP(0) -JUMP_BACKWARD_NO_INTERRUPT(5) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -SEND_GEN(3) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -RESUME_CHECK(2) -TIER1_SET_IP(0) -JUMP_BACKWARD_NO_INTERRUPT(5) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -SEND_GEN(3) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -RESUME_CHECK(2) -TIER1_SET_IP(0) -JUMP_BACKWARD_NO_INTERRUPT(5) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -SEND_GEN(3) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -RESUME_CHECK(2) -TIER1_SET_IP(0) -JUMP_BACKWARD_NO_INTERRUPT(5) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -SEND_GEN(3) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -RESUME_CHECK(2) -TIER1_SET_IP(0) -JUMP_BACKWARD_NO_INTERRUPT(5) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -SEND_GEN(3) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -RESUME_CHECK(2) -TIER1_SET_IP(0) -JUMP_BACKWARD_NO_INTERRUPT(5) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -SEND_GEN(3) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -RESUME_CHECK(2) -TIER1_SET_IP(0) -JUMP_BACKWARD_NO_INTERRUPT(5) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -SEND_GEN(3) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -RESUME_CHECK(2) -TIER1_SET_IP(0) -JUMP_BACKWARD_NO_INTERRUPT(5) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -SEND_GEN(3) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -RESUME_CHECK(2) -TIER1_SET_IP(0) -JUMP_BACKWARD_NO_INTERRUPT(5) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -SEND_GEN(3) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -RESUME_CHECK(2) -TIER1_SET_IP(0) -JUMP_BACKWARD_NO_INTERRUPT(5) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -SEND_GEN(3) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -RESUME_CHECK(2) -TIER1_SET_IP(0) -JUMP_BACKWARD_NO_INTERRUPT(5) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -SEND_GEN(3) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -RESUME_CHECK(2) -TIER1_SET_IP(0) -JUMP_BACKWARD_NO_INTERRUPT(5) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -SEND_GEN(3) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -RESUME_CHECK(2) -TIER1_SET_IP(0) -JUMP_BACKWARD_NO_INTERRUPT(5) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -SEND_GEN(3) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -RESUME_CHECK(2) -TIER1_SET_IP(0) -JUMP_BACKWARD_NO_INTERRUPT(5) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -SEND_GEN(3) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -RESUME_CHECK(2) -TIER1_SET_IP(0) -JUMP_BACKWARD_NO_INTERRUPT(5) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -SEND_GEN(3) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -RESUME_CHECK(5) -TIER1_SET_IP(0) -POP_TOP(0) -TIER1_SET_IP(0) -LOAD_FAST_BORROW(0) -TIER1_SET_IP(0) -LOAD_ATTR_INSTANCE_VALUE(4) -TIER1_SET_IP(0) -TO_BOOL_NONE(0) -TIER1_SET_IP(0) -POP_JUMP_IF_FALSE(23) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -LOAD_CONST(0) -TIER1_SET_IP(0) -RETURN_VALUE(0) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -END_SEND(0) -TIER1_SET_IP(0) -POP_TOP(0) -TIER1_SET_IP(0) -LOAD_FAST_BORROW(0) -TIER1_SET_IP(0) -LOAD_ATTR_INSTANCE_VALUE(2) -TIER1_SET_IP(0) -YIELD_VALUE(0) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -YIELD_VALUE(1) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -YIELD_VALUE(1) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -YIELD_VALUE(1) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -YIELD_VALUE(1) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -YIELD_VALUE(1) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -YIELD_VALUE(1) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -YIELD_VALUE(1) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -YIELD_VALUE(1) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -YIELD_VALUE(1) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -YIELD_VALUE(1) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -YIELD_VALUE(1) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -YIELD_VALUE(1) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -YIELD_VALUE(1) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -YIELD_VALUE(1) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -YIELD_VALUE(1) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -STORE_FAST(3) -TIER1_SET_IP(0) -JUMP_BACKWARD_JIT(5) -TIER1_GUARD_IP(0) -TIER1_SET_IP(0) -Optimizing bench_generators (/home/ken/Documents/GitHub/cpython/../bm_generators.py:36) at byte offset 186 - 1 ADD_TO_TRACE: _START_EXECUTOR (0, target=93, operand0=0x71448db5630a, operand1=0) - 2 ADD_TO_TRACE: _MAKE_WARM (0, target=0, operand0=0, operand1=0) -93: FOR_ITER_GEN(3) - 3 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=93, operand0=0, operand1=0) - 4 ADD_TO_TRACE: _SET_IP (0, target=93, operand0=0x71448db5630a, operand1=0) - 5 ADD_TO_TRACE: _FOR_ITER (3, target=93, operand0=0, operand1=0, error_target=0) -93: TIER1_GUARD_IP(0) - 6 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=93, operand0=0, operand1=0) - 7 ADD_TO_TRACE: _GUARD_IP (0, target=37, operand0=0x71448db5565a, operand1=0) -37: RESUME_CHECK(2) - 8 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=37, operand0=0, operand1=0) - 9 ADD_TO_TRACE: _SET_IP (0, target=37, operand0=0x71448db5565a, operand1=0) - 10 ADD_TO_TRACE: _RESUME_CHECK (2, target=37, operand0=0, operand1=0) -37: TIER1_SET_IP(0) - 11 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=37, operand0=0, operand1=0) - 12 ADD_TO_TRACE: _SET_IP (0, target=37, operand0=0x71448db5565a, operand1=0) -38: JUMP_BACKWARD_NO_INTERRUPT(5) - 13 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=38, operand0=0, operand1=0) - 14 ADD_TO_TRACE: _SET_IP (0, target=38, operand0=0x71448db5565c, operand1=0) - 15 ADD_TO_TRACE: _CHECK_PERIODIC (0, target=38, operand0=0, operand1=0, error_target=0) -38: TIER1_GUARD_IP(0) - 16 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=38, operand0=0, operand1=0) - 17 ADD_TO_TRACE: _GUARD_IP (0, target=34, operand0=0x71448db55654, operand1=0) -34: SEND_GEN(3) - 18 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=34, operand0=0, operand1=0) - 19 ADD_TO_TRACE: _SET_IP (0, target=34, operand0=0x71448db55654, operand1=0) - 20 ADD_TO_TRACE: _CHECK_PEP_523 (3, target=34, operand0=0, operand1=0) - 21 ADD_TO_TRACE: _SEND_GEN_FRAME (3, target=34, operand0=0, operand1=0) - 22 ADD_TO_TRACE: _PUSH_FRAME (3, target=34, operand0=0, operand1=0) -34: TIER1_GUARD_IP(0) - 23 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=34, operand0=0, operand1=0) - 24 ADD_TO_TRACE: _GUARD_IP (0, target=37, operand0=0x71448db5565a, operand1=0) -37: RESUME_CHECK(2) - 25 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=37, operand0=0, operand1=0) - 26 ADD_TO_TRACE: _SET_IP (0, target=37, operand0=0x71448db5565a, operand1=0) - 27 ADD_TO_TRACE: _RESUME_CHECK (2, target=37, operand0=0, operand1=0) -37: TIER1_SET_IP(0) - 28 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=37, operand0=0, operand1=0) - 29 ADD_TO_TRACE: _SET_IP (0, target=37, operand0=0x71448db5565a, operand1=0) -38: JUMP_BACKWARD_NO_INTERRUPT(5) - 30 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=38, operand0=0, operand1=0) - 31 ADD_TO_TRACE: _SET_IP (0, target=38, operand0=0x71448db5565c, operand1=0) - 32 ADD_TO_TRACE: _CHECK_PERIODIC (0, target=38, operand0=0, operand1=0, error_target=0) -38: TIER1_GUARD_IP(0) - 33 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=38, operand0=0, operand1=0) - 34 ADD_TO_TRACE: _GUARD_IP (0, target=34, operand0=0x71448db55654, operand1=0) -34: SEND_GEN(3) - 35 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=34, operand0=0, operand1=0) - 36 ADD_TO_TRACE: _SET_IP (0, target=34, operand0=0x71448db55654, operand1=0) - 37 ADD_TO_TRACE: _CHECK_PEP_523 (3, target=34, operand0=0, operand1=0) - 38 ADD_TO_TRACE: _SEND_GEN_FRAME (3, target=34, operand0=0, operand1=0) - 39 ADD_TO_TRACE: _PUSH_FRAME (3, target=34, operand0=0, operand1=0) -34: TIER1_GUARD_IP(0) - 40 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=34, operand0=0, operand1=0) - 41 ADD_TO_TRACE: _GUARD_IP (0, target=37, operand0=0x71448db5565a, operand1=0) -37: RESUME_CHECK(2) - 42 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=37, operand0=0, operand1=0) - 43 ADD_TO_TRACE: _SET_IP (0, target=37, operand0=0x71448db5565a, operand1=0) - 44 ADD_TO_TRACE: _RESUME_CHECK (2, target=37, operand0=0, operand1=0) -37: TIER1_SET_IP(0) - 45 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=37, operand0=0, operand1=0) - 46 ADD_TO_TRACE: _SET_IP (0, target=37, operand0=0x71448db5565a, operand1=0) -38: JUMP_BACKWARD_NO_INTERRUPT(5) - 47 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=38, operand0=0, operand1=0) - 48 ADD_TO_TRACE: _SET_IP (0, target=38, operand0=0x71448db5565c, operand1=0) - 49 ADD_TO_TRACE: _CHECK_PERIODIC (0, target=38, operand0=0, operand1=0, error_target=0) -38: TIER1_GUARD_IP(0) - 50 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=38, operand0=0, operand1=0) - 51 ADD_TO_TRACE: _GUARD_IP (0, target=34, operand0=0x71448db55654, operand1=0) -34: SEND_GEN(3) - 52 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=34, operand0=0, operand1=0) - 53 ADD_TO_TRACE: _SET_IP (0, target=34, operand0=0x71448db55654, operand1=0) - 54 ADD_TO_TRACE: _CHECK_PEP_523 (3, target=34, operand0=0, operand1=0) - 55 ADD_TO_TRACE: _SEND_GEN_FRAME (3, target=34, operand0=0, operand1=0) - 56 ADD_TO_TRACE: _PUSH_FRAME (3, target=34, operand0=0, operand1=0) -34: TIER1_GUARD_IP(0) - 57 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=34, operand0=0, operand1=0) - 58 ADD_TO_TRACE: _GUARD_IP (0, target=37, operand0=0x71448db5565a, operand1=0) -37: RESUME_CHECK(2) - 59 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=37, operand0=0, operand1=0) - 60 ADD_TO_TRACE: _SET_IP (0, target=37, operand0=0x71448db5565a, operand1=0) - 61 ADD_TO_TRACE: _RESUME_CHECK (2, target=37, operand0=0, operand1=0) -37: TIER1_SET_IP(0) - 62 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=37, operand0=0, operand1=0) - 63 ADD_TO_TRACE: _SET_IP (0, target=37, operand0=0x71448db5565a, operand1=0) -38: JUMP_BACKWARD_NO_INTERRUPT(5) - 64 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=38, operand0=0, operand1=0) - 65 ADD_TO_TRACE: _SET_IP (0, target=38, operand0=0x71448db5565c, operand1=0) - 66 ADD_TO_TRACE: _CHECK_PERIODIC (0, target=38, operand0=0, operand1=0, error_target=0) -38: TIER1_GUARD_IP(0) - 67 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=38, operand0=0, operand1=0) - 68 ADD_TO_TRACE: _GUARD_IP (0, target=34, operand0=0x71448db55654, operand1=0) -34: SEND_GEN(3) - 69 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=34, operand0=0, operand1=0) - 70 ADD_TO_TRACE: _SET_IP (0, target=34, operand0=0x71448db55654, operand1=0) - 71 ADD_TO_TRACE: _CHECK_PEP_523 (3, target=34, operand0=0, operand1=0) - 72 ADD_TO_TRACE: _SEND_GEN_FRAME (3, target=34, operand0=0, operand1=0) - 73 ADD_TO_TRACE: _PUSH_FRAME (3, target=34, operand0=0, operand1=0) -34: TIER1_GUARD_IP(0) - 74 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=34, operand0=0, operand1=0) - 75 ADD_TO_TRACE: _GUARD_IP (0, target=89, operand0=0x71448db556c2, operand1=0) -89: RESUME_CHECK(2) - 76 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=89, operand0=0, operand1=0) - 77 ADD_TO_TRACE: _SET_IP (0, target=89, operand0=0x71448db556c2, operand1=0) - 78 ADD_TO_TRACE: _RESUME_CHECK (2, target=89, operand0=0, operand1=0) -89: TIER1_SET_IP(0) - 79 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=89, operand0=0, operand1=0) - 80 ADD_TO_TRACE: _SET_IP (0, target=89, operand0=0x71448db556c2, operand1=0) -90: JUMP_BACKWARD_NO_INTERRUPT(5) - 81 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=90, operand0=0, operand1=0) - 82 ADD_TO_TRACE: _SET_IP (0, target=90, operand0=0x71448db556c4, operand1=0) - 83 ADD_TO_TRACE: _CHECK_PERIODIC (0, target=90, operand0=0, operand1=0, error_target=0) -90: TIER1_GUARD_IP(0) - 84 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=90, operand0=0, operand1=0) - 85 ADD_TO_TRACE: _GUARD_IP (0, target=86, operand0=0x71448db556bc, operand1=0) -86: SEND_GEN(3) - 86 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=86, operand0=0, operand1=0) - 87 ADD_TO_TRACE: _SET_IP (0, target=86, operand0=0x71448db556bc, operand1=0) - 88 ADD_TO_TRACE: _CHECK_PEP_523 (3, target=86, operand0=0, operand1=0) - 89 ADD_TO_TRACE: _SEND_GEN_FRAME (3, target=86, operand0=0, operand1=0) - 90 ADD_TO_TRACE: _PUSH_FRAME (3, target=86, operand0=0, operand1=0) -86: TIER1_GUARD_IP(0) - 91 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=86, operand0=0, operand1=0) - 92 ADD_TO_TRACE: _GUARD_IP (0, target=37, operand0=0x71448db5565a, operand1=0) -37: RESUME_CHECK(2) - 93 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=37, operand0=0, operand1=0) - 94 ADD_TO_TRACE: _SET_IP (0, target=37, operand0=0x71448db5565a, operand1=0) - 95 ADD_TO_TRACE: _RESUME_CHECK (2, target=37, operand0=0, operand1=0) -37: TIER1_SET_IP(0) - 96 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=37, operand0=0, operand1=0) - 97 ADD_TO_TRACE: _SET_IP (0, target=37, operand0=0x71448db5565a, operand1=0) -38: JUMP_BACKWARD_NO_INTERRUPT(5) - 98 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=38, operand0=0, operand1=0) - 99 ADD_TO_TRACE: _SET_IP (0, target=38, operand0=0x71448db5565c, operand1=0) - 100 ADD_TO_TRACE: _CHECK_PERIODIC (0, target=38, operand0=0, operand1=0, error_target=0) -38: TIER1_GUARD_IP(0) - 101 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=38, operand0=0, operand1=0) - 102 ADD_TO_TRACE: _GUARD_IP (0, target=34, operand0=0x71448db55654, operand1=0) -34: SEND_GEN(3) - 103 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=34, operand0=0, operand1=0) - 104 ADD_TO_TRACE: _SET_IP (0, target=34, operand0=0x71448db55654, operand1=0) - 105 ADD_TO_TRACE: _CHECK_PEP_523 (3, target=34, operand0=0, operand1=0) - 106 ADD_TO_TRACE: _SEND_GEN_FRAME (3, target=34, operand0=0, operand1=0) - 107 ADD_TO_TRACE: _PUSH_FRAME (3, target=34, operand0=0, operand1=0) -34: TIER1_GUARD_IP(0) - 108 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=34, operand0=0, operand1=0) - 109 ADD_TO_TRACE: _GUARD_IP (0, target=89, operand0=0x71448db556c2, operand1=0) -89: RESUME_CHECK(2) - 110 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=89, operand0=0, operand1=0) - 111 ADD_TO_TRACE: _SET_IP (0, target=89, operand0=0x71448db556c2, operand1=0) - 112 ADD_TO_TRACE: _RESUME_CHECK (2, target=89, operand0=0, operand1=0) -89: TIER1_SET_IP(0) - 113 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=89, operand0=0, operand1=0) - 114 ADD_TO_TRACE: _SET_IP (0, target=89, operand0=0x71448db556c2, operand1=0) -90: JUMP_BACKWARD_NO_INTERRUPT(5) - 115 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=90, operand0=0, operand1=0) - 116 ADD_TO_TRACE: _SET_IP (0, target=90, operand0=0x71448db556c4, operand1=0) - 117 ADD_TO_TRACE: _CHECK_PERIODIC (0, target=90, operand0=0, operand1=0, error_target=0) -90: TIER1_GUARD_IP(0) - 118 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=90, operand0=0, operand1=0) - 119 ADD_TO_TRACE: _GUARD_IP (0, target=86, operand0=0x71448db556bc, operand1=0) -86: SEND_GEN(3) - 120 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=86, operand0=0, operand1=0) - 121 ADD_TO_TRACE: _SET_IP (0, target=86, operand0=0x71448db556bc, operand1=0) - 122 ADD_TO_TRACE: _CHECK_PEP_523 (3, target=86, operand0=0, operand1=0) - 123 ADD_TO_TRACE: _SEND_GEN_FRAME (3, target=86, operand0=0, operand1=0) - 124 ADD_TO_TRACE: _PUSH_FRAME (3, target=86, operand0=0, operand1=0) -86: TIER1_GUARD_IP(0) - 125 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=86, operand0=0, operand1=0) - 126 ADD_TO_TRACE: _GUARD_IP (0, target=37, operand0=0x71448db5565a, operand1=0) -37: RESUME_CHECK(2) - 127 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=37, operand0=0, operand1=0) - 128 ADD_TO_TRACE: _SET_IP (0, target=37, operand0=0x71448db5565a, operand1=0) - 129 ADD_TO_TRACE: _RESUME_CHECK (2, target=37, operand0=0, operand1=0) -37: TIER1_SET_IP(0) - 130 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=37, operand0=0, operand1=0) - 131 ADD_TO_TRACE: _SET_IP (0, target=37, operand0=0x71448db5565a, operand1=0) -38: JUMP_BACKWARD_NO_INTERRUPT(5) - 132 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=38, operand0=0, operand1=0) - 133 ADD_TO_TRACE: _SET_IP (0, target=38, operand0=0x71448db5565c, operand1=0) - 134 ADD_TO_TRACE: _CHECK_PERIODIC (0, target=38, operand0=0, operand1=0, error_target=0) -38: TIER1_GUARD_IP(0) - 135 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=38, operand0=0, operand1=0) - 136 ADD_TO_TRACE: _GUARD_IP (0, target=34, operand0=0x71448db55654, operand1=0) -34: SEND_GEN(3) - 137 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=34, operand0=0, operand1=0) - 138 ADD_TO_TRACE: _SET_IP (0, target=34, operand0=0x71448db55654, operand1=0) - 139 ADD_TO_TRACE: _CHECK_PEP_523 (3, target=34, operand0=0, operand1=0) - 140 ADD_TO_TRACE: _SEND_GEN_FRAME (3, target=34, operand0=0, operand1=0) - 141 ADD_TO_TRACE: _PUSH_FRAME (3, target=34, operand0=0, operand1=0) -34: TIER1_GUARD_IP(0) - 142 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=34, operand0=0, operand1=0) - 143 ADD_TO_TRACE: _GUARD_IP (0, target=37, operand0=0x71448db5565a, operand1=0) -37: RESUME_CHECK(2) - 144 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=37, operand0=0, operand1=0) - 145 ADD_TO_TRACE: _SET_IP (0, target=37, operand0=0x71448db5565a, operand1=0) - 146 ADD_TO_TRACE: _RESUME_CHECK (2, target=37, operand0=0, operand1=0) -37: TIER1_SET_IP(0) - 147 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=37, operand0=0, operand1=0) - 148 ADD_TO_TRACE: _SET_IP (0, target=37, operand0=0x71448db5565a, operand1=0) -38: JUMP_BACKWARD_NO_INTERRUPT(5) - 149 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=38, operand0=0, operand1=0) - 150 ADD_TO_TRACE: _SET_IP (0, target=38, operand0=0x71448db5565c, operand1=0) - 151 ADD_TO_TRACE: _CHECK_PERIODIC (0, target=38, operand0=0, operand1=0, error_target=0) -38: TIER1_GUARD_IP(0) - 152 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=38, operand0=0, operand1=0) - 153 ADD_TO_TRACE: _GUARD_IP (0, target=34, operand0=0x71448db55654, operand1=0) -34: SEND_GEN(3) - 154 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=34, operand0=0, operand1=0) - 155 ADD_TO_TRACE: _SET_IP (0, target=34, operand0=0x71448db55654, operand1=0) - 156 ADD_TO_TRACE: _CHECK_PEP_523 (3, target=34, operand0=0, operand1=0) - 157 ADD_TO_TRACE: _SEND_GEN_FRAME (3, target=34, operand0=0, operand1=0) - 158 ADD_TO_TRACE: _PUSH_FRAME (3, target=34, operand0=0, operand1=0) -34: TIER1_GUARD_IP(0) - 159 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=34, operand0=0, operand1=0) - 160 ADD_TO_TRACE: _GUARD_IP (0, target=89, operand0=0x71448db556c2, operand1=0) -89: RESUME_CHECK(2) - 161 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=89, operand0=0, operand1=0) - 162 ADD_TO_TRACE: _SET_IP (0, target=89, operand0=0x71448db556c2, operand1=0) - 163 ADD_TO_TRACE: _RESUME_CHECK (2, target=89, operand0=0, operand1=0) -89: TIER1_SET_IP(0) - 164 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=89, operand0=0, operand1=0) - 165 ADD_TO_TRACE: _SET_IP (0, target=89, operand0=0x71448db556c2, operand1=0) -90: JUMP_BACKWARD_NO_INTERRUPT(5) - 166 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=90, operand0=0, operand1=0) - 167 ADD_TO_TRACE: _SET_IP (0, target=90, operand0=0x71448db556c4, operand1=0) - 168 ADD_TO_TRACE: _CHECK_PERIODIC (0, target=90, operand0=0, operand1=0, error_target=0) -90: TIER1_GUARD_IP(0) - 169 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=90, operand0=0, operand1=0) - 170 ADD_TO_TRACE: _GUARD_IP (0, target=86, operand0=0x71448db556bc, operand1=0) -86: SEND_GEN(3) - 171 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=86, operand0=0, operand1=0) - 172 ADD_TO_TRACE: _SET_IP (0, target=86, operand0=0x71448db556bc, operand1=0) - 173 ADD_TO_TRACE: _CHECK_PEP_523 (3, target=86, operand0=0, operand1=0) - 174 ADD_TO_TRACE: _SEND_GEN_FRAME (3, target=86, operand0=0, operand1=0) - 175 ADD_TO_TRACE: _PUSH_FRAME (3, target=86, operand0=0, operand1=0) -86: TIER1_GUARD_IP(0) - 176 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=86, operand0=0, operand1=0) - 177 ADD_TO_TRACE: _GUARD_IP (0, target=89, operand0=0x71448db556c2, operand1=0) -89: RESUME_CHECK(2) - 178 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=89, operand0=0, operand1=0) - 179 ADD_TO_TRACE: _SET_IP (0, target=89, operand0=0x71448db556c2, operand1=0) - 180 ADD_TO_TRACE: _RESUME_CHECK (2, target=89, operand0=0, operand1=0) -89: TIER1_SET_IP(0) - 181 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=89, operand0=0, operand1=0) - 182 ADD_TO_TRACE: _SET_IP (0, target=89, operand0=0x71448db556c2, operand1=0) -90: JUMP_BACKWARD_NO_INTERRUPT(5) - 183 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=90, operand0=0, operand1=0) - 184 ADD_TO_TRACE: _SET_IP (0, target=90, operand0=0x71448db556c4, operand1=0) - 185 ADD_TO_TRACE: _CHECK_PERIODIC (0, target=90, operand0=0, operand1=0, error_target=0) -90: TIER1_GUARD_IP(0) - 186 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=90, operand0=0, operand1=0) - 187 ADD_TO_TRACE: _GUARD_IP (0, target=86, operand0=0x71448db556bc, operand1=0) -86: SEND_GEN(3) - 188 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=86, operand0=0, operand1=0) - 189 ADD_TO_TRACE: _SET_IP (0, target=86, operand0=0x71448db556bc, operand1=0) - 190 ADD_TO_TRACE: _CHECK_PEP_523 (3, target=86, operand0=0, operand1=0) - 191 ADD_TO_TRACE: _SEND_GEN_FRAME (3, target=86, operand0=0, operand1=0) - 192 ADD_TO_TRACE: _PUSH_FRAME (3, target=86, operand0=0, operand1=0) -86: TIER1_GUARD_IP(0) - 193 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=86, operand0=0, operand1=0) - 194 ADD_TO_TRACE: _GUARD_IP (0, target=89, operand0=0x71448db556c2, operand1=0) -89: RESUME_CHECK(2) - 195 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=89, operand0=0, operand1=0) - 196 ADD_TO_TRACE: _SET_IP (0, target=89, operand0=0x71448db556c2, operand1=0) - 197 ADD_TO_TRACE: _RESUME_CHECK (2, target=89, operand0=0, operand1=0) -89: TIER1_SET_IP(0) - 198 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=89, operand0=0, operand1=0) - 199 ADD_TO_TRACE: _SET_IP (0, target=89, operand0=0x71448db556c2, operand1=0) -90: JUMP_BACKWARD_NO_INTERRUPT(5) - 200 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=90, operand0=0, operand1=0) - 201 ADD_TO_TRACE: _SET_IP (0, target=90, operand0=0x71448db556c4, operand1=0) - 202 ADD_TO_TRACE: _CHECK_PERIODIC (0, target=90, operand0=0, operand1=0, error_target=0) -90: TIER1_GUARD_IP(0) - 203 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=90, operand0=0, operand1=0) - 204 ADD_TO_TRACE: _GUARD_IP (0, target=86, operand0=0x71448db556bc, operand1=0) -86: SEND_GEN(3) - 205 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=86, operand0=0, operand1=0) - 206 ADD_TO_TRACE: _SET_IP (0, target=86, operand0=0x71448db556bc, operand1=0) - 207 ADD_TO_TRACE: _CHECK_PEP_523 (3, target=86, operand0=0, operand1=0) - 208 ADD_TO_TRACE: _SEND_GEN_FRAME (3, target=86, operand0=0, operand1=0) - 209 ADD_TO_TRACE: _PUSH_FRAME (3, target=86, operand0=0, operand1=0) -86: TIER1_GUARD_IP(0) - 210 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=86, operand0=0, operand1=0) - 211 ADD_TO_TRACE: _GUARD_IP (0, target=89, operand0=0x71448db556c2, operand1=0) -89: RESUME_CHECK(2) - 212 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=89, operand0=0, operand1=0) - 213 ADD_TO_TRACE: _SET_IP (0, target=89, operand0=0x71448db556c2, operand1=0) - 214 ADD_TO_TRACE: _RESUME_CHECK (2, target=89, operand0=0, operand1=0) -89: TIER1_SET_IP(0) - 215 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=89, operand0=0, operand1=0) - 216 ADD_TO_TRACE: _SET_IP (0, target=89, operand0=0x71448db556c2, operand1=0) -90: JUMP_BACKWARD_NO_INTERRUPT(5) - 217 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=90, operand0=0, operand1=0) - 218 ADD_TO_TRACE: _SET_IP (0, target=90, operand0=0x71448db556c4, operand1=0) - 219 ADD_TO_TRACE: _CHECK_PERIODIC (0, target=90, operand0=0, operand1=0, error_target=0) -90: TIER1_GUARD_IP(0) - 220 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=90, operand0=0, operand1=0) - 221 ADD_TO_TRACE: _GUARD_IP (0, target=86, operand0=0x71448db556bc, operand1=0) -86: SEND_GEN(3) - 222 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=86, operand0=0, operand1=0) - 223 ADD_TO_TRACE: _SET_IP (0, target=86, operand0=0x71448db556bc, operand1=0) - 224 ADD_TO_TRACE: _CHECK_PEP_523 (3, target=86, operand0=0, operand1=0) - 225 ADD_TO_TRACE: _SEND_GEN_FRAME (3, target=86, operand0=0, operand1=0) - 226 ADD_TO_TRACE: _PUSH_FRAME (3, target=86, operand0=0, operand1=0) -86: TIER1_GUARD_IP(0) - 227 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=86, operand0=0, operand1=0) - 228 ADD_TO_TRACE: _GUARD_IP (0, target=37, operand0=0x71448db5565a, operand1=0) -37: RESUME_CHECK(2) - 229 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=37, operand0=0, operand1=0) - 230 ADD_TO_TRACE: _SET_IP (0, target=37, operand0=0x71448db5565a, operand1=0) - 231 ADD_TO_TRACE: _RESUME_CHECK (2, target=37, operand0=0, operand1=0) -37: TIER1_SET_IP(0) - 232 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=37, operand0=0, operand1=0) - 233 ADD_TO_TRACE: _SET_IP (0, target=37, operand0=0x71448db5565a, operand1=0) -38: JUMP_BACKWARD_NO_INTERRUPT(5) - 234 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=38, operand0=0, operand1=0) - 235 ADD_TO_TRACE: _SET_IP (0, target=38, operand0=0x71448db5565c, operand1=0) - 236 ADD_TO_TRACE: _CHECK_PERIODIC (0, target=38, operand0=0, operand1=0, error_target=0) -38: TIER1_GUARD_IP(0) - 237 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=38, operand0=0, operand1=0) - 238 ADD_TO_TRACE: _GUARD_IP (0, target=34, operand0=0x71448db55654, operand1=0) -34: SEND_GEN(3) - 239 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=34, operand0=0, operand1=0) - 240 ADD_TO_TRACE: _SET_IP (0, target=34, operand0=0x71448db55654, operand1=0) - 241 ADD_TO_TRACE: _CHECK_PEP_523 (3, target=34, operand0=0, operand1=0) - 242 ADD_TO_TRACE: _SEND_GEN_FRAME (3, target=34, operand0=0, operand1=0) - 243 ADD_TO_TRACE: _PUSH_FRAME (3, target=34, operand0=0, operand1=0) -34: TIER1_GUARD_IP(0) - 244 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=34, operand0=0, operand1=0) - 245 ADD_TO_TRACE: _GUARD_IP (0, target=89, operand0=0x71448db556c2, operand1=0) -89: RESUME_CHECK(2) - 246 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=89, operand0=0, operand1=0) - 247 ADD_TO_TRACE: _SET_IP (0, target=89, operand0=0x71448db556c2, operand1=0) - 248 ADD_TO_TRACE: _RESUME_CHECK (2, target=89, operand0=0, operand1=0) -89: TIER1_SET_IP(0) - 249 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=89, operand0=0, operand1=0) - 250 ADD_TO_TRACE: _SET_IP (0, target=89, operand0=0x71448db556c2, operand1=0) -90: JUMP_BACKWARD_NO_INTERRUPT(5) - 251 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=90, operand0=0, operand1=0) - 252 ADD_TO_TRACE: _SET_IP (0, target=90, operand0=0x71448db556c4, operand1=0) - 253 ADD_TO_TRACE: _CHECK_PERIODIC (0, target=90, operand0=0, operand1=0, error_target=0) -90: TIER1_GUARD_IP(0) - 254 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=90, operand0=0, operand1=0) - 255 ADD_TO_TRACE: _GUARD_IP (0, target=86, operand0=0x71448db556bc, operand1=0) -86: SEND_GEN(3) - 256 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=86, operand0=0, operand1=0) - 257 ADD_TO_TRACE: _SET_IP (0, target=86, operand0=0x71448db556bc, operand1=0) - 258 ADD_TO_TRACE: _CHECK_PEP_523 (3, target=86, operand0=0, operand1=0) - 259 ADD_TO_TRACE: _SEND_GEN_FRAME (3, target=86, operand0=0, operand1=0) - 260 ADD_TO_TRACE: _PUSH_FRAME (3, target=86, operand0=0, operand1=0) -86: TIER1_GUARD_IP(0) - 261 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=86, operand0=0, operand1=0) - 262 ADD_TO_TRACE: _GUARD_IP (0, target=37, operand0=0x71448db5565a, operand1=0) -37: RESUME_CHECK(2) - 263 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=37, operand0=0, operand1=0) - 264 ADD_TO_TRACE: _SET_IP (0, target=37, operand0=0x71448db5565a, operand1=0) - 265 ADD_TO_TRACE: _RESUME_CHECK (2, target=37, operand0=0, operand1=0) -37: TIER1_SET_IP(0) - 266 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=37, operand0=0, operand1=0) - 267 ADD_TO_TRACE: _SET_IP (0, target=37, operand0=0x71448db5565a, operand1=0) -38: JUMP_BACKWARD_NO_INTERRUPT(5) - 268 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=38, operand0=0, operand1=0) - 269 ADD_TO_TRACE: _SET_IP (0, target=38, operand0=0x71448db5565c, operand1=0) - 270 ADD_TO_TRACE: _CHECK_PERIODIC (0, target=38, operand0=0, operand1=0, error_target=0) -38: TIER1_GUARD_IP(0) - 271 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=38, operand0=0, operand1=0) - 272 ADD_TO_TRACE: _GUARD_IP (0, target=34, operand0=0x71448db55654, operand1=0) -34: SEND_GEN(3) - 273 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=34, operand0=0, operand1=0) - 274 ADD_TO_TRACE: _SET_IP (0, target=34, operand0=0x71448db55654, operand1=0) - 275 ADD_TO_TRACE: _CHECK_PEP_523 (3, target=34, operand0=0, operand1=0) - 276 ADD_TO_TRACE: _SEND_GEN_FRAME (3, target=34, operand0=0, operand1=0) - 277 ADD_TO_TRACE: _PUSH_FRAME (3, target=34, operand0=0, operand1=0) -34: TIER1_GUARD_IP(0) - 278 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=34, operand0=0, operand1=0) - 279 ADD_TO_TRACE: _GUARD_IP (0, target=53, operand0=0x71448db5567a, operand1=0) -53: RESUME_CHECK(5) - 280 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=53, operand0=0, operand1=0) - 281 ADD_TO_TRACE: _SET_IP (0, target=53, operand0=0x71448db5567a, operand1=0) - 282 ADD_TO_TRACE: _RESUME_CHECK (5, target=53, operand0=0, operand1=0) -53: TIER1_SET_IP(0) - 283 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=53, operand0=0, operand1=0) - 284 ADD_TO_TRACE: _SET_IP (0, target=53, operand0=0x71448db5567a, operand1=0) -54: POP_TOP(0) - 285 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=54, operand0=0, operand1=0) - 286 ADD_TO_TRACE: _SET_IP (0, target=54, operand0=0x71448db5567c, operand1=0) - 287 ADD_TO_TRACE: _POP_TOP (0, target=54, operand0=0, operand1=0) -54: TIER1_SET_IP(0) - 288 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=54, operand0=0, operand1=0) - 289 ADD_TO_TRACE: _SET_IP (0, target=54, operand0=0x71448db5567c, operand1=0) -55: LOAD_FAST_BORROW(0) - 290 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=55, operand0=0, operand1=0) - 291 ADD_TO_TRACE: _SET_IP (0, target=55, operand0=0x71448db5567e, operand1=0) - 292 ADD_TO_TRACE: _LOAD_FAST_BORROW (0, target=55, operand0=0, operand1=0) -55: TIER1_SET_IP(0) - 293 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=55, operand0=0, operand1=0) - 294 ADD_TO_TRACE: _SET_IP (0, target=55, operand0=0x71448db5567e, operand1=0) -56: LOAD_ATTR_INSTANCE_VALUE(4) - 295 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=56, operand0=0, operand1=0) - 296 ADD_TO_TRACE: _SET_IP (0, target=56, operand0=0x71448db55680, operand1=0) - 297 ADD_TO_TRACE: _GUARD_TYPE_VERSION (4, target=56, operand0=0x2004a, operand1=0) - 298 ADD_TO_TRACE: _CHECK_MANAGED_OBJECT_HAS_VALUES (4, target=56, operand0=0, operand1=0) - 299 ADD_TO_TRACE: _LOAD_ATTR_INSTANCE_VALUE (4, target=56, operand0=0x20, operand1=0) - 300 ADD_TO_TRACE: _PUSH_NULL_CONDITIONAL (4, target=56, operand0=0, operand1=0) -56: TIER1_SET_IP(0) - 301 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=56, operand0=0, operand1=0) - 302 ADD_TO_TRACE: _SET_IP (0, target=56, operand0=0x71448db55680, operand1=0) -66: TO_BOOL_NONE(0) - 303 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=66, operand0=0, operand1=0) - 304 ADD_TO_TRACE: _SET_IP (0, target=66, operand0=0x71448db55694, operand1=0) - 305 ADD_TO_TRACE: _TO_BOOL_NONE (0, target=66, operand0=0, operand1=0) -66: TIER1_SET_IP(0) - 306 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=66, operand0=0, operand1=0) - 307 ADD_TO_TRACE: _SET_IP (0, target=66, operand0=0x71448db55694, operand1=0) -70: POP_JUMP_IF_FALSE(23) - 308 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=70, operand0=0, operand1=0) - 309 ADD_TO_TRACE: _SET_IP (0, target=70, operand0=0x71448db5569c, operand1=0) - 310 ADD_TO_TRACE: _POP_JUMP_IF_FALSE (23, target=70, operand0=0, operand1=0) -70: TIER1_GUARD_IP(0) - 311 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=70, operand0=0, operand1=0) - 312 ADD_TO_TRACE: _GUARD_IP (0, target=95, operand0=0x71448db556ce, operand1=0) -95: LOAD_CONST(0) - 313 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=95, operand0=0, operand1=0) - 314 ADD_TO_TRACE: _SET_IP (0, target=95, operand0=0x71448db556ce, operand1=0) - 315 ADD_TO_TRACE: _LOAD_CONST (0, target=95, operand0=0, operand1=0) -95: TIER1_SET_IP(0) - 316 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=95, operand0=0, operand1=0) - 317 ADD_TO_TRACE: _SET_IP (0, target=95, operand0=0x71448db556ce, operand1=0) -96: RETURN_VALUE(0) - 318 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=96, operand0=0, operand1=0) - 319 ADD_TO_TRACE: _SET_IP (0, target=96, operand0=0x71448db556d0, operand1=0) - 320 ADD_TO_TRACE: _RETURN_VALUE (0, target=96, operand0=0, operand1=0) -96: TIER1_GUARD_IP(0) - 321 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=96, operand0=0, operand1=0) - 322 ADD_TO_TRACE: _GUARD_IP (0, target=39, operand0=0x71448db5565e, operand1=0) -39: END_SEND(0) - 323 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=39, operand0=0, operand1=0) - 324 ADD_TO_TRACE: _SET_IP (0, target=39, operand0=0x71448db5565e, operand1=0) - 325 ADD_TO_TRACE: _END_SEND (0, target=39, operand0=0, operand1=0) -39: TIER1_SET_IP(0) - 326 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=39, operand0=0, operand1=0) - 327 ADD_TO_TRACE: _SET_IP (0, target=39, operand0=0x71448db5565e, operand1=0) -40: POP_TOP(0) - 328 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=40, operand0=0, operand1=0) - 329 ADD_TO_TRACE: _SET_IP (0, target=40, operand0=0x71448db55660, operand1=0) - 330 ADD_TO_TRACE: _POP_TOP (0, target=40, operand0=0, operand1=0) -40: TIER1_SET_IP(0) - 331 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=40, operand0=0, operand1=0) - 332 ADD_TO_TRACE: _SET_IP (0, target=40, operand0=0x71448db55660, operand1=0) -41: LOAD_FAST_BORROW(0) - 333 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=41, operand0=0, operand1=0) - 334 ADD_TO_TRACE: _SET_IP (0, target=41, operand0=0x71448db55662, operand1=0) - 335 ADD_TO_TRACE: _LOAD_FAST_BORROW (0, target=41, operand0=0, operand1=0) -41: TIER1_SET_IP(0) - 336 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=41, operand0=0, operand1=0) - 337 ADD_TO_TRACE: _SET_IP (0, target=41, operand0=0x71448db55662, operand1=0) -42: LOAD_ATTR_INSTANCE_VALUE(2) - 338 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=42, operand0=0, operand1=0) - 339 ADD_TO_TRACE: _SET_IP (0, target=42, operand0=0x71448db55664, operand1=0) - 340 ADD_TO_TRACE: _GUARD_TYPE_VERSION (2, target=42, operand0=0x2004a, operand1=0) - 341 ADD_TO_TRACE: _CHECK_MANAGED_OBJECT_HAS_VALUES (2, target=42, operand0=0, operand1=0) - 342 ADD_TO_TRACE: _LOAD_ATTR_INSTANCE_VALUE (2, target=42, operand0=0x28, operand1=0) - 343 ADD_TO_TRACE: _PUSH_NULL_CONDITIONAL (2, target=42, operand0=0, operand1=0) -42: TIER1_SET_IP(0) - 344 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=42, operand0=0, operand1=0) - 345 ADD_TO_TRACE: _SET_IP (0, target=42, operand0=0x71448db55664, operand1=0) -52: YIELD_VALUE(0) - 346 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=52, operand0=0, operand1=0) - 347 ADD_TO_TRACE: _SET_IP (0, target=52, operand0=0x71448db55678, operand1=0) - 348 ADD_TO_TRACE: _YIELD_VALUE (0, target=52, operand0=0, operand1=0) -52: TIER1_GUARD_IP(0) - 349 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=52, operand0=0, operand1=0) - 350 ADD_TO_TRACE: _GUARD_IP (0, target=88, operand0=0x71448db556c0, operand1=0) -88: YIELD_VALUE(1) - 351 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=88, operand0=0, operand1=0) - 352 ADD_TO_TRACE: _SET_IP (0, target=88, operand0=0x71448db556c0, operand1=0) - 353 ADD_TO_TRACE: _YIELD_VALUE (1, target=88, operand0=0, operand1=0) -88: TIER1_GUARD_IP(0) - 354 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=88, operand0=0, operand1=0) - 355 ADD_TO_TRACE: _GUARD_IP (0, target=36, operand0=0x71448db55658, operand1=0) -36: YIELD_VALUE(1) - 356 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=36, operand0=0, operand1=0) - 357 ADD_TO_TRACE: _SET_IP (0, target=36, operand0=0x71448db55658, operand1=0) - 358 ADD_TO_TRACE: _YIELD_VALUE (1, target=36, operand0=0, operand1=0) -36: TIER1_GUARD_IP(0) - 359 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=36, operand0=0, operand1=0) - 360 ADD_TO_TRACE: _GUARD_IP (0, target=88, operand0=0x71448db556c0, operand1=0) -88: YIELD_VALUE(1) - 361 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=88, operand0=0, operand1=0) - 362 ADD_TO_TRACE: _SET_IP (0, target=88, operand0=0x71448db556c0, operand1=0) - 363 ADD_TO_TRACE: _YIELD_VALUE (1, target=88, operand0=0, operand1=0) -88: TIER1_GUARD_IP(0) - 364 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=88, operand0=0, operand1=0) - 365 ADD_TO_TRACE: _GUARD_IP (0, target=88, operand0=0x71448db556c0, operand1=0) -88: YIELD_VALUE(1) - 366 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=88, operand0=0, operand1=0) - 367 ADD_TO_TRACE: _SET_IP (0, target=88, operand0=0x71448db556c0, operand1=0) - 368 ADD_TO_TRACE: _YIELD_VALUE (1, target=88, operand0=0, operand1=0) -88: TIER1_GUARD_IP(0) - 369 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=88, operand0=0, operand1=0) - 370 ADD_TO_TRACE: _GUARD_IP (0, target=88, operand0=0x71448db556c0, operand1=0) -88: YIELD_VALUE(1) - 371 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=88, operand0=0, operand1=0) - 372 ADD_TO_TRACE: _SET_IP (0, target=88, operand0=0x71448db556c0, operand1=0) - 373 ADD_TO_TRACE: _YIELD_VALUE (1, target=88, operand0=0, operand1=0) -88: TIER1_GUARD_IP(0) - 374 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=88, operand0=0, operand1=0) - 375 ADD_TO_TRACE: _GUARD_IP (0, target=88, operand0=0x71448db556c0, operand1=0) -88: YIELD_VALUE(1) - 376 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=88, operand0=0, operand1=0) - 377 ADD_TO_TRACE: _SET_IP (0, target=88, operand0=0x71448db556c0, operand1=0) - 378 ADD_TO_TRACE: _YIELD_VALUE (1, target=88, operand0=0, operand1=0) -88: TIER1_GUARD_IP(0) - 379 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=88, operand0=0, operand1=0) - 380 ADD_TO_TRACE: _GUARD_IP (0, target=36, operand0=0x71448db55658, operand1=0) -36: YIELD_VALUE(1) - 381 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=36, operand0=0, operand1=0) - 382 ADD_TO_TRACE: _SET_IP (0, target=36, operand0=0x71448db55658, operand1=0) - 383 ADD_TO_TRACE: _YIELD_VALUE (1, target=36, operand0=0, operand1=0) -36: TIER1_GUARD_IP(0) - 384 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=36, operand0=0, operand1=0) - 385 ADD_TO_TRACE: _GUARD_IP (0, target=36, operand0=0x71448db55658, operand1=0) -36: YIELD_VALUE(1) - 386 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=36, operand0=0, operand1=0) - 387 ADD_TO_TRACE: _SET_IP (0, target=36, operand0=0x71448db55658, operand1=0) - 388 ADD_TO_TRACE: _YIELD_VALUE (1, target=36, operand0=0, operand1=0) -36: TIER1_GUARD_IP(0) - 389 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=36, operand0=0, operand1=0) - 390 ADD_TO_TRACE: _GUARD_IP (0, target=88, operand0=0x71448db556c0, operand1=0) -88: YIELD_VALUE(1) - 391 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=88, operand0=0, operand1=0) - 392 ADD_TO_TRACE: _SET_IP (0, target=88, operand0=0x71448db556c0, operand1=0) - 393 ADD_TO_TRACE: _YIELD_VALUE (1, target=88, operand0=0, operand1=0) -88: TIER1_GUARD_IP(0) - 394 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=88, operand0=0, operand1=0) - 395 ADD_TO_TRACE: _GUARD_IP (0, target=36, operand0=0x71448db55658, operand1=0) -36: YIELD_VALUE(1) - 396 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=36, operand0=0, operand1=0) - 397 ADD_TO_TRACE: _SET_IP (0, target=36, operand0=0x71448db55658, operand1=0) - 398 ADD_TO_TRACE: _YIELD_VALUE (1, target=36, operand0=0, operand1=0) -36: TIER1_GUARD_IP(0) - 399 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=36, operand0=0, operand1=0) - 400 ADD_TO_TRACE: _GUARD_IP (0, target=88, operand0=0x71448db556c0, operand1=0) -88: YIELD_VALUE(1) - 401 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=88, operand0=0, operand1=0) - 402 ADD_TO_TRACE: _SET_IP (0, target=88, operand0=0x71448db556c0, operand1=0) - 403 ADD_TO_TRACE: _YIELD_VALUE (1, target=88, operand0=0, operand1=0) -88: TIER1_GUARD_IP(0) - 404 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=88, operand0=0, operand1=0) - 405 ADD_TO_TRACE: _GUARD_IP (0, target=36, operand0=0x71448db55658, operand1=0) -36: YIELD_VALUE(1) - 406 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=36, operand0=0, operand1=0) - 407 ADD_TO_TRACE: _SET_IP (0, target=36, operand0=0x71448db55658, operand1=0) - 408 ADD_TO_TRACE: _YIELD_VALUE (1, target=36, operand0=0, operand1=0) -36: TIER1_GUARD_IP(0) - 409 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=36, operand0=0, operand1=0) - 410 ADD_TO_TRACE: _GUARD_IP (0, target=36, operand0=0x71448db55658, operand1=0) -36: YIELD_VALUE(1) - 411 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=36, operand0=0, operand1=0) - 412 ADD_TO_TRACE: _SET_IP (0, target=36, operand0=0x71448db55658, operand1=0) - 413 ADD_TO_TRACE: _YIELD_VALUE (1, target=36, operand0=0, operand1=0) -36: TIER1_GUARD_IP(0) - 414 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=36, operand0=0, operand1=0) - 415 ADD_TO_TRACE: _GUARD_IP (0, target=36, operand0=0x71448db55658, operand1=0) -36: YIELD_VALUE(1) - 416 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=36, operand0=0, operand1=0) - 417 ADD_TO_TRACE: _SET_IP (0, target=36, operand0=0x71448db55658, operand1=0) - 418 ADD_TO_TRACE: _YIELD_VALUE (1, target=36, operand0=0, operand1=0) -36: TIER1_GUARD_IP(0) - 419 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=36, operand0=0, operand1=0) - 420 ADD_TO_TRACE: _GUARD_IP (0, target=36, operand0=0x71448db55658, operand1=0) -36: YIELD_VALUE(1) - 421 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=36, operand0=0, operand1=0) - 422 ADD_TO_TRACE: _SET_IP (0, target=36, operand0=0x71448db55658, operand1=0) - 423 ADD_TO_TRACE: _YIELD_VALUE (1, target=36, operand0=0, operand1=0) -36: TIER1_GUARD_IP(0) - 424 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=36, operand0=0, operand1=0) - 425 ADD_TO_TRACE: _GUARD_IP (0, target=95, operand0=0x71448db5630e, operand1=0) -95: STORE_FAST(3) - 426 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=95, operand0=0, operand1=0) - 427 ADD_TO_TRACE: _SET_IP (0, target=95, operand0=0x71448db5630e, operand1=0) - 428 ADD_TO_TRACE: _STORE_FAST (3, target=95, operand0=0, operand1=0) -95: TIER1_SET_IP(0) - 429 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=95, operand0=0, operand1=0) - 430 ADD_TO_TRACE: _SET_IP (0, target=95, operand0=0x71448db5630e, operand1=0) -96: JUMP_BACKWARD_JIT(5) - 431 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=96, operand0=0, operand1=0) - 432 ADD_TO_TRACE: _SET_IP (0, target=96, operand0=0x71448db56310, operand1=0) - 433 ADD_TO_TRACE: _CHECK_PERIODIC (0, target=96, operand0=0, operand1=0, error_target=0) -96: TIER1_GUARD_IP(0) - 434 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=96, operand0=0, operand1=0) - 435 ADD_TO_TRACE: _GUARD_IP (0, target=93, operand0=0x71448db5630a, operand1=0) - 436 ADD_TO_TRACE: _JUMP_TO_TOP (0, target=0, operand0=0, operand1=0) -Created a proto-trace for bench_generators (/home/ken/Documents/GitHub/cpython/../bm_generators.py:36) at byte offset 186 -- length 436 - locals=[, , , ] - stack=[, , , ] - 0 uop: _START_EXECUTOR (0, jump_target=238, operand0=0x64d708ad28e0, operand1=0) - locals=[, , , ] - stack=[, , , ] - 1 uop: _MAKE_WARM (0, target=0, operand0=0, operand1=0) - locals=[, , , ] - stack=[, , , ] - 2 uop: _SET_IP (0, target=93, operand0=0x71448db5630a, operand1=0) - locals=[, , , ] - stack=[, , , ] - 3 uop: _FOR_ITER (3, jump_target=0, operand0=0, operand1=0, error_target=239) - locals=[, , , ] - stack=[, , , , ] - 4 uop: _CHECK_VALIDITY (0, jump_target=238, operand0=0, operand1=0) - locals=[, , , ] - stack=[, , , , ] - 5 uop: _GUARD_IP (0, target=37, operand0=0x71448db5565a, operand1=0) -0 ADD_TO_BYTECODE_TRACE: STORE_FAST 3 -1 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db56310 -6 ADD_TO_BYTECODE_TRACE: JUMP_BACKWARD_JIT 5 -8 ADD_TO_BYTECODE_TRACE: TIER1_GUARD_IP 0 -13 ADD_TO_BYTECODE_TRACE: TIER1_SET_IP 0x71448db5630a -STORE_FAST(3) -TIER1_SET_IP(3) -JUMP_BACKWARD_JIT(5) -TIER1_GUARD_IP(86) -TIER1_SET_IP(0) -Optimizing bench_generators (/home/ken/Documents/GitHub/cpython/../bm_generators.py:36) at byte offset 186 - 1 ADD_TO_TRACE: _START_EXECUTOR (0, target=93, operand0=0x71448db5630a, operand1=0) - 2 ADD_TO_TRACE: _MAKE_WARM (0, target=0, operand0=0, operand1=0) -93: STORE_FAST(3) - 3 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=93, operand0=0, operand1=0) - 4 ADD_TO_TRACE: _SET_IP (0, target=93, operand0=0x71448db5630a, operand1=0) - 5 ADD_TO_TRACE: _STORE_FAST (3, target=93, operand0=0, operand1=0) -93: TIER1_SET_IP(3) - 6 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=93, operand0=0, operand1=0) - 7 ADD_TO_TRACE: _SET_IP (0, target=93, operand0=0x71448db5630a, operand1=0) -96: JUMP_BACKWARD_JIT(5) - 8 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=96, operand0=0, operand1=0) - 9 ADD_TO_TRACE: _SET_IP (0, target=96, operand0=0x71448db56310, operand1=0) - 10 ADD_TO_TRACE: _CHECK_PERIODIC (0, target=96, operand0=0, operand1=0, error_target=0) -96: TIER1_GUARD_IP(86) - 11 ADD_TO_TRACE: _CHECK_VALIDITY (0, target=96, operand0=0, operand1=0) - 12 ADD_TO_TRACE: _GUARD_IP (0, target=93, operand0=0x71448db5630a, operand1=0) - 13 ADD_TO_TRACE: _JUMP_TO_TOP (0, target=0, operand0=0, operand1=0) -Created a proto-trace for bench_generators (/home/ken/Documents/GitHub/cpython/../bm_generators.py:36) at byte offset 186 -- length 13 - locals=[, , , ] - stack=[, , , ] - 0 uop: _START_EXECUTOR (0, jump_target=10, operand0=0x71448db814f0, operand1=0) - locals=[, , , ] - stack=[, , , ] - 1 uop: _MAKE_WARM (0, target=0, operand0=0, operand1=0) - locals=[, , , ] - stack=[, , , ] - 2 uop: _SET_IP (0, target=93, operand0=0x71448db5630a, operand1=0) - locals=[, , , ] - stack=[, , , ] - 3 uop: _STORE_FAST_3 (3, target=93, operand0=0, operand1=0) - locals=[, , , ] - stack=[, , ] - 4 uop: _CHECK_VALIDITY (0, jump_target=10, operand0=0, operand1=0) - locals=[, , , ] - stack=[, , ] - 5 uop: _SET_IP (0, target=96, operand0=0x71448db56310, operand1=0) - locals=[, , , ] - stack=[, , ] - 6 uop: _CHECK_PERIODIC (0, jump_target=0, operand0=0, operand1=0, error_target=11) - locals=[, , , ] - stack=[, , ] - 7 uop: _CHECK_VALIDITY (0, jump_target=12, operand0=0, operand1=0) - locals=[, , , ] - stack=[, , ] From cae8f10ab4cda8f3204cd8b6f5a3f28de2719777 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 17 Oct 2025 11:02:31 +0100 Subject: [PATCH 038/190] patch the graphviz dump --- Python/optimizer.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Python/optimizer.c b/Python/optimizer.c index ed0e92bceb9c5f..67368fefa3452e 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1662,6 +1662,7 @@ executor_to_gv(_PyExecutorObject *executor, FILE *out) fprintf(out, "]\n\n"); /* Write all the outgoing edges */ + _PyExecutorObject *cold = _PyExecutor_GetColdExecutor(); for (uint32_t i = 0; i < executor->code_size; i++) { _PyUOpInstruction const *inst = &executor->trace[i]; uint16_t flags = _PyUop_Flags[inst->opcode]; @@ -1675,7 +1676,7 @@ executor_to_gv(_PyExecutorObject *executor, FILE *out) assert(exit_inst->opcode == _EXIT_TRACE || exit_inst->opcode == _DYNAMIC_EXIT); exit = (_PyExitData *)exit_inst->operand0; } - if (exit != NULL && exit->executor != NULL) { + if (exit != NULL && exit->executor != cold) { fprintf(out, "executor_%p:i%d -> executor_%p:start\n", executor, i, exit->executor); } if (inst->opcode == _EXIT_TRACE || inst->opcode == _JUMP_TO_TOP) { From 39bc819822244b95fd97376cc3e36b348ec40b83 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 17 Oct 2025 17:06:46 +0100 Subject: [PATCH 039/190] fix bug with predicted stuff --- Python/generated_cases.c.h | 4178 +++++++++++----------- Tools/cases_generator/tier1_generator.py | 12 +- 2 files changed, 2095 insertions(+), 2095 deletions(-) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index e190f651fa4c84..476d433e617ea7 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -23,18 +23,18 @@ int opcode = BINARY_OP; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP); PREDICTED_BINARY_OP:; _Py_CODEUNIT* const this_instr = next_instr - 6; (void)this_instr; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; _PyStackRef lhs; _PyStackRef rhs; _PyStackRef res; @@ -92,17 +92,17 @@ int opcode = BINARY_OP_ADD_FLOAT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef left; @@ -156,17 +156,17 @@ int opcode = BINARY_OP_ADD_INT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_ADD_INT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_ADD_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef left; @@ -222,17 +222,17 @@ int opcode = BINARY_OP_ADD_UNICODE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; @@ -288,17 +288,17 @@ int opcode = BINARY_OP_EXTEND; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_EXTEND); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_EXTEND); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; _PyStackRef right; @@ -358,17 +358,17 @@ int opcode = BINARY_OP_INPLACE_ADD_UNICODE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; @@ -447,17 +447,17 @@ int opcode = BINARY_OP_MULTIPLY_FLOAT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef left; @@ -511,17 +511,17 @@ int opcode = BINARY_OP_MULTIPLY_INT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef left; @@ -577,17 +577,17 @@ int opcode = BINARY_OP_SUBSCR_DICT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_DICT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBSCR_DICT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef nos; _PyStackRef dict_st; @@ -649,17 +649,17 @@ int opcode = BINARY_OP_SUBSCR_GETITEM; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_GETITEM); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBSCR_GETITEM); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef container; _PyStackRef getitem; @@ -739,17 +739,17 @@ int opcode = BINARY_OP_SUBSCR_LIST_INT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_INT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; @@ -835,17 +835,17 @@ int opcode = BINARY_OP_SUBSCR_LIST_SLICE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_SLICE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_SLICE); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef tos; _PyStackRef nos; @@ -913,17 +913,17 @@ int opcode = BINARY_OP_SUBSCR_STR_INT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_STR_INT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBSCR_STR_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; @@ -997,17 +997,17 @@ int opcode = BINARY_OP_SUBSCR_TUPLE_INT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_TUPLE_INT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBSCR_TUPLE_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; @@ -1076,17 +1076,17 @@ int opcode = BINARY_OP_SUBTRACT_FLOAT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef left; @@ -1140,17 +1140,17 @@ int opcode = BINARY_OP_SUBTRACT_INT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef left; @@ -1206,15 +1206,15 @@ int opcode = BINARY_SLICE; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BINARY_SLICE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BINARY_SLICE); _PyStackRef container; _PyStackRef start; _PyStackRef stop; @@ -1268,15 +1268,15 @@ int opcode = BUILD_INTERPOLATION; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_INTERPOLATION); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_INTERPOLATION); _PyStackRef value; _PyStackRef str; _PyStackRef *format; @@ -1332,15 +1332,15 @@ int opcode = BUILD_LIST; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_LIST); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_LIST); _PyStackRef *values; _PyStackRef list; values = &stack_pointer[-oparg]; @@ -1362,15 +1362,15 @@ int opcode = BUILD_MAP; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_MAP); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_MAP); _PyStackRef *values; _PyStackRef map; values = &stack_pointer[-oparg*2]; @@ -1420,15 +1420,15 @@ int opcode = BUILD_SET; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_SET); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_SET); _PyStackRef *values; _PyStackRef set; values = &stack_pointer[-oparg]; @@ -1483,15 +1483,15 @@ int opcode = BUILD_SLICE; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_SLICE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_SLICE); _PyStackRef *args; _PyStackRef slice; args = &stack_pointer[-oparg]; @@ -1524,15 +1524,15 @@ int opcode = BUILD_STRING; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_STRING); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_STRING); _PyStackRef *pieces; _PyStackRef str; pieces = &stack_pointer[-oparg]; @@ -1577,15 +1577,15 @@ int opcode = BUILD_TEMPLATE; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_TEMPLATE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_TEMPLATE); _PyStackRef strings; _PyStackRef interpolations; _PyStackRef template; @@ -1621,15 +1621,15 @@ int opcode = BUILD_TUPLE; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_TUPLE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_TUPLE); _PyStackRef *values; _PyStackRef tup; values = &stack_pointer[-oparg]; @@ -1649,15 +1649,15 @@ int opcode = CACHE; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CACHE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CACHE); assert(0 && "Executing a cache."); Py_FatalError("Executing a cache."); DISPATCH(); @@ -1668,18 +1668,18 @@ int opcode = CALL; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL); PREDICTED_CALL:; _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; opcode = CALL; _PyStackRef callable; _PyStackRef self_or_null; @@ -1852,17 +1852,17 @@ int opcode = CALL_ALLOC_AND_ENTER_INIT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -1979,17 +1979,17 @@ int opcode = CALL_BOUND_METHOD_EXACT_ARGS; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef null; @@ -2128,17 +2128,17 @@ int opcode = CALL_BOUND_METHOD_GENERAL; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef null; @@ -2262,17 +2262,17 @@ int opcode = CALL_BUILTIN_CLASS; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_CLASS); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_CLASS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -2373,17 +2373,17 @@ int opcode = CALL_BUILTIN_FAST; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_FAST); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -2488,17 +2488,17 @@ int opcode = CALL_BUILTIN_FAST_WITH_KEYWORDS; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -2603,17 +2603,17 @@ int opcode = CALL_BUILTIN_O; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_O); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_O); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -2693,17 +2693,17 @@ int opcode = CALL_FUNCTION_EX; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_FUNCTION_EX); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_FUNCTION_EX); opcode = CALL_FUNCTION_EX; _PyStackRef func; _PyStackRef callargs; @@ -2866,15 +2866,15 @@ int opcode = CALL_INTRINSIC_1; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_INTRINSIC_1); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_INTRINSIC_1); _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; @@ -2902,15 +2902,15 @@ int opcode = CALL_INTRINSIC_2; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_INTRINSIC_2); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_INTRINSIC_2); _PyStackRef value2_st; _PyStackRef value1_st; _PyStackRef res; @@ -2947,17 +2947,17 @@ int opcode = CALL_ISINSTANCE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_ISINSTANCE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_ISINSTANCE); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef null; _PyStackRef callable; @@ -3029,18 +3029,18 @@ int opcode = CALL_KW; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW); PREDICTED_CALL_KW:; _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; opcode = CALL_KW; _PyStackRef callable; _PyStackRef self_or_null; @@ -3217,17 +3217,17 @@ int opcode = CALL_KW_BOUND_METHOD; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef null; @@ -3353,17 +3353,17 @@ int opcode = CALL_KW_NON_PY; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_NON_PY); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_KW_NON_PY); opcode = CALL_KW_NON_PY; static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); _PyStackRef callable; @@ -3486,17 +3486,17 @@ int opcode = CALL_KW_PY; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_PY); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_KW_PY); static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -3602,17 +3602,17 @@ int opcode = CALL_LEN; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_LEN); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_LEN); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef null; _PyStackRef callable; @@ -3680,17 +3680,17 @@ int opcode = CALL_LIST_APPEND; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_LIST_APPEND); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_LIST_APPEND); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef nos; @@ -3774,17 +3774,17 @@ int opcode = CALL_METHOD_DESCRIPTOR_FAST; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -3901,17 +3901,17 @@ int opcode = CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -4030,17 +4030,17 @@ int opcode = CALL_METHOD_DESCRIPTOR_NOARGS; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -4129,17 +4129,17 @@ int opcode = CALL_METHOD_DESCRIPTOR_O; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -4241,17 +4241,17 @@ int opcode = CALL_NON_PY_GENERAL; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_NON_PY_GENERAL); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_NON_PY_GENERAL); opcode = CALL_NON_PY_GENERAL; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; @@ -4362,17 +4362,17 @@ int opcode = CALL_PY_EXACT_ARGS; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -4481,17 +4481,17 @@ int opcode = CALL_PY_GENERAL; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_GENERAL); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_PY_GENERAL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -4587,17 +4587,17 @@ int opcode = CALL_STR_1; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_STR_1); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_STR_1); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef null; _PyStackRef callable; @@ -4665,17 +4665,17 @@ int opcode = CALL_TUPLE_1; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_TUPLE_1); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_TUPLE_1); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef null; _PyStackRef callable; @@ -4743,17 +4743,17 @@ int opcode = CALL_TYPE_1; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_TYPE_1); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_TYPE_1); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef null; _PyStackRef callable; @@ -4804,15 +4804,15 @@ int opcode = CHECK_EG_MATCH; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CHECK_EG_MATCH); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CHECK_EG_MATCH); _PyStackRef exc_value_st; _PyStackRef match_type_st; _PyStackRef rest; @@ -4881,15 +4881,15 @@ int opcode = CHECK_EXC_MATCH; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CHECK_EXC_MATCH); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CHECK_EXC_MATCH); _PyStackRef left; _PyStackRef right; _PyStackRef b; @@ -4924,17 +4924,17 @@ int opcode = CLEANUP_THROW; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CLEANUP_THROW); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CLEANUP_THROW); _PyStackRef sub_iter; _PyStackRef last_sent_val; _PyStackRef exc_value_st; @@ -4989,18 +4989,18 @@ int opcode = COMPARE_OP; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP); PREDICTED_COMPARE_OP:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; _PyStackRef left; _PyStackRef right; _PyStackRef res; @@ -5068,17 +5068,17 @@ int opcode = COMPARE_OP_FLOAT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_FLOAT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_FLOAT); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); _PyStackRef value; _PyStackRef left; @@ -5129,17 +5129,17 @@ int opcode = COMPARE_OP_INT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_INT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_INT); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); _PyStackRef value; _PyStackRef left; @@ -5194,17 +5194,17 @@ int opcode = COMPARE_OP_STR; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_STR); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_STR); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; @@ -5259,18 +5259,18 @@ int opcode = CONTAINS_OP; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP); PREDICTED_CONTAINS_OP:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; _PyStackRef right; _PyStackRef left; _PyStackRef b; @@ -5325,17 +5325,17 @@ int opcode = CONTAINS_OP_DICT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP_DICT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(CONTAINS_OP_DICT); static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); _PyStackRef tos; _PyStackRef left; @@ -5389,17 +5389,17 @@ int opcode = CONTAINS_OP_SET; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP_SET); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(CONTAINS_OP_SET); static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); _PyStackRef tos; _PyStackRef left; @@ -5453,15 +5453,15 @@ int opcode = CONVERT_VALUE; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CONVERT_VALUE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CONVERT_VALUE); _PyStackRef value; _PyStackRef result; value = stack_pointer[-1]; @@ -5491,15 +5491,15 @@ int opcode = COPY; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(COPY); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(COPY); _PyStackRef bottom; _PyStackRef top; bottom = stack_pointer[-1 - (oparg-1)]; @@ -5515,15 +5515,15 @@ int opcode = COPY_FREE_VARS; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(COPY_FREE_VARS); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(COPY_FREE_VARS); PyCodeObject *co = _PyFrame_GetCode(frame); assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -5542,15 +5542,15 @@ int opcode = DELETE_ATTR; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_ATTR); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_ATTR); _PyStackRef owner; owner = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -5573,15 +5573,15 @@ int opcode = DELETE_DEREF; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_DEREF); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_DEREF); PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); if (oldobj == NULL) { @@ -5601,15 +5601,15 @@ int opcode = DELETE_FAST; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_FAST); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_FAST); _PyStackRef v = GETLOCAL(oparg); if (PyStackRef_IsNull(v)) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -5633,15 +5633,15 @@ int opcode = DELETE_GLOBAL; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_GLOBAL); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_GLOBAL); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyDict_Pop(GLOBALS(), name, NULL); @@ -5664,15 +5664,15 @@ int opcode = DELETE_NAME; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_NAME); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_NAME); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *ns = LOCALS(); int err; @@ -5702,15 +5702,15 @@ int opcode = DELETE_SUBSCR; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_SUBSCR); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_SUBSCR); _PyStackRef container; _PyStackRef sub; sub = stack_pointer[-1]; @@ -5740,15 +5740,15 @@ int opcode = DICT_MERGE; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DICT_MERGE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DICT_MERGE); _PyStackRef callable; _PyStackRef dict; _PyStackRef update; @@ -5785,15 +5785,15 @@ int opcode = DICT_UPDATE; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DICT_UPDATE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DICT_UPDATE); _PyStackRef dict; _PyStackRef update; update = stack_pointer[-1]; @@ -5834,17 +5834,17 @@ int opcode = END_ASYNC_FOR; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(END_ASYNC_FOR); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(END_ASYNC_FOR); _PyStackRef awaitable_st; _PyStackRef exc_st; exc_st = stack_pointer[-1]; @@ -5885,14 +5885,14 @@ int opcode = END_FOR; (void)(opcode); #endif + next_instr += 1; + INSTRUCTION_STATS(END_FOR); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - next_instr += 1; - INSTRUCTION_STATS(END_FOR); _PyStackRef value; value = stack_pointer[-1]; stack_pointer += -1; @@ -5908,15 +5908,15 @@ int opcode = END_SEND; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(END_SEND); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(END_SEND); _PyStackRef receiver; _PyStackRef value; _PyStackRef val; @@ -5937,17 +5937,17 @@ int opcode = ENTER_EXECUTOR; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(ENTER_EXECUTOR); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(ENTER_EXECUTOR); opcode = ENTER_EXECUTOR; #ifdef _Py_TIER2 PyCodeObject *code = _PyFrame_GetCode(frame); @@ -5982,15 +5982,15 @@ int opcode = EXIT_INIT_CHECK; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(EXIT_INIT_CHECK); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(EXIT_INIT_CHECK); _PyStackRef should_be_none; should_be_none = stack_pointer[-1]; if (!PyStackRef_IsNone(should_be_none)) { @@ -6011,15 +6011,15 @@ int opcode = EXTENDED_ARG; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(EXTENDED_ARG); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(EXTENDED_ARG); opcode = EXTENDED_ARG; assert(oparg); opcode = next_instr->op.code; @@ -6033,15 +6033,15 @@ int opcode = FORMAT_SIMPLE; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(FORMAT_SIMPLE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(FORMAT_SIMPLE); _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; @@ -6075,15 +6075,15 @@ int opcode = FORMAT_WITH_SPEC; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(FORMAT_WITH_SPEC); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(FORMAT_WITH_SPEC); _PyStackRef value; _PyStackRef fmt_spec; _PyStackRef res; @@ -6117,18 +6117,18 @@ int opcode = FOR_ITER; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER); PREDICTED_FOR_ITER:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; _PyStackRef iter; _PyStackRef null_or_index; _PyStackRef next; @@ -6177,17 +6177,17 @@ int opcode = FOR_ITER_GEN; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_GEN); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_GEN); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; _PyStackRef gen_frame; @@ -6254,17 +6254,17 @@ int opcode = FOR_ITER_LIST; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_LIST); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_LIST); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; _PyStackRef null_or_index; @@ -6344,17 +6344,17 @@ int opcode = FOR_ITER_RANGE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_RANGE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_RANGE); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; _PyStackRef next; @@ -6417,17 +6417,17 @@ int opcode = FOR_ITER_TUPLE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_TUPLE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_TUPLE); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; _PyStackRef null_or_index; @@ -6479,15 +6479,15 @@ int opcode = GET_AITER; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_AITER); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_AITER); _PyStackRef obj; _PyStackRef iter; obj = stack_pointer[-1]; @@ -6546,15 +6546,15 @@ int opcode = GET_ANEXT; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_ANEXT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_ANEXT); _PyStackRef aiter; _PyStackRef awaitable; aiter = stack_pointer[-1]; @@ -6576,15 +6576,15 @@ int opcode = GET_AWAITABLE; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_AWAITABLE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_AWAITABLE); _PyStackRef iterable; _PyStackRef iter; iterable = stack_pointer[-1]; @@ -6611,15 +6611,15 @@ int opcode = GET_ITER; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_ITER); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_ITER); _PyStackRef iterable; _PyStackRef iter; _PyStackRef index_or_null; @@ -6663,15 +6663,15 @@ int opcode = GET_LEN; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_LEN); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_LEN); _PyStackRef obj; _PyStackRef len; obj = stack_pointer[-1]; @@ -6697,15 +6697,15 @@ int opcode = GET_YIELD_FROM_ITER; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_YIELD_FROM_ITER); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_YIELD_FROM_ITER); _PyStackRef iterable; _PyStackRef iter; iterable = stack_pointer[-1]; @@ -6748,15 +6748,15 @@ int opcode = IMPORT_FROM; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IMPORT_FROM); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IMPORT_FROM); _PyStackRef from; _PyStackRef res; from = stack_pointer[-1]; @@ -6779,15 +6779,15 @@ int opcode = IMPORT_NAME; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IMPORT_NAME); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IMPORT_NAME); _PyStackRef level; _PyStackRef fromlist; _PyStackRef res; @@ -6824,17 +6824,17 @@ int opcode = INSTRUMENTED_CALL; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(INSTRUMENTED_CALL); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(INSTRUMENTED_CALL); opcode = INSTRUMENTED_CALL; _PyStackRef callable; _PyStackRef self_or_null; @@ -7018,17 +7018,17 @@ int opcode = INSTRUMENTED_CALL_FUNCTION_EX; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); opcode = INSTRUMENTED_CALL_FUNCTION_EX; _PyStackRef func; _PyStackRef callargs; @@ -7191,17 +7191,17 @@ int opcode = INSTRUMENTED_CALL_KW; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); opcode = INSTRUMENTED_CALL_KW; _PyStackRef callable; _PyStackRef self_or_null; @@ -7383,17 +7383,17 @@ int opcode = INSTRUMENTED_END_ASYNC_FOR; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_ASYNC_FOR); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_END_ASYNC_FOR); _PyStackRef awaitable_st; _PyStackRef exc_st; // _MONITOR_END_ASYNC_FOR @@ -7442,16 +7442,16 @@ int opcode = INSTRUMENTED_END_FOR; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_FOR); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_END_FOR); _PyStackRef receiver; _PyStackRef value; value = stack_pointer[-1]; @@ -7477,17 +7477,17 @@ int opcode = INSTRUMENTED_END_SEND; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_SEND); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_END_SEND); _PyStackRef receiver; _PyStackRef value; _PyStackRef val; @@ -7517,17 +7517,17 @@ int opcode = INSTRUMENTED_FOR_ITER; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); _PyStackRef iter; _PyStackRef null_or_index; _PyStackRef next; @@ -7559,17 +7559,17 @@ int opcode = INSTRUMENTED_INSTRUCTION; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); opcode = INSTRUMENTED_INSTRUCTION; _PyFrame_SetStackPointer(frame, stack_pointer); int next_opcode = _Py_call_instrumentation_instruction( @@ -7592,17 +7592,17 @@ int opcode = INSTRUMENTED_JUMP_BACKWARD; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); /* Skip 1 cache entry */ // _CHECK_PERIODIC { @@ -7625,17 +7625,17 @@ int opcode = INSTRUMENTED_JUMP_FORWARD; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); DISPATCH(); } @@ -7645,18 +7645,18 @@ int opcode = INSTRUMENTED_LINE; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_LINE); + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; opcode = INSTRUMENTED_LINE; int original_opcode = 0; if (tstate->tracing) { @@ -7691,17 +7691,17 @@ int opcode = INSTRUMENTED_LOAD_SUPER_ATTR; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); opcode = INSTRUMENTED_LOAD_SUPER_ATTR; _PyStackRef global_super_st; _PyStackRef class_st; @@ -7817,18 +7817,18 @@ int opcode = INSTRUMENTED_NOT_TAKEN; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_NOT_TAKEN); + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; (void)this_instr; INSTRUMENTED_JUMP(prev_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); DISPATCH(); @@ -7839,18 +7839,18 @@ int opcode = INSTRUMENTED_POP_ITER; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_POP_ITER); + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; _PyStackRef iter; _PyStackRef index_or_null; index_or_null = stack_pointer[-1]; @@ -7870,17 +7870,17 @@ int opcode = INSTRUMENTED_POP_JUMP_IF_FALSE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); _PyStackRef cond; /* Skip 1 cache entry */ cond = stack_pointer[-1]; @@ -7900,17 +7900,17 @@ int opcode = INSTRUMENTED_POP_JUMP_IF_NONE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); _PyStackRef value; /* Skip 1 cache entry */ value = stack_pointer[-1]; @@ -7937,17 +7937,17 @@ int opcode = INSTRUMENTED_POP_JUMP_IF_NOT_NONE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); _PyStackRef value; /* Skip 1 cache entry */ value = stack_pointer[-1]; @@ -7972,17 +7972,17 @@ int opcode = INSTRUMENTED_POP_JUMP_IF_TRUE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); _PyStackRef cond; /* Skip 1 cache entry */ cond = stack_pointer[-1]; @@ -8002,17 +8002,17 @@ int opcode = INSTRUMENTED_RESUME; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RESUME); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_RESUME); // _LOAD_BYTECODE { #ifdef Py_GIL_DISABLED @@ -8088,17 +8088,17 @@ int opcode = INSTRUMENTED_RETURN_VALUE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); _PyStackRef val; _PyStackRef retval; _PyStackRef res; @@ -8146,17 +8146,17 @@ int opcode = INSTRUMENTED_YIELD_VALUE; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; _PyStackRef val; _PyStackRef retval; _PyStackRef value; @@ -8223,15 +8223,15 @@ int opcode = INTERPRETER_EXIT; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INTERPRETER_EXIT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INTERPRETER_EXIT); _PyStackRef retval; retval = stack_pointer[-1]; assert(frame->owner == FRAME_OWNED_BY_INTERPRETER); @@ -8264,15 +8264,15 @@ int opcode = IS_OP; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IS_OP); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IS_OP); _PyStackRef left; _PyStackRef right; _PyStackRef b; @@ -8303,18 +8303,18 @@ int opcode = JUMP_BACKWARD; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(JUMP_BACKWARD); PREDICTED_JUMP_BACKWARD:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; /* Skip 1 cache entry */ // _SPECIALIZE_JUMP_BACKWARD { @@ -8350,17 +8350,17 @@ int opcode = JUMP_BACKWARD_JIT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(JUMP_BACKWARD_JIT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(JUMP_BACKWARD_JIT); static_assert(1 == 1, "incorrect cache size"); /* Skip 1 cache entry */ // _CHECK_PERIODIC @@ -8417,15 +8417,15 @@ int opcode = JUMP_BACKWARD_NO_INTERRUPT; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); #if TIER_ONE assert(oparg <= INSTR_OFFSET()); #endif @@ -8438,15 +8438,15 @@ int opcode = JUMP_BACKWARD_NO_JIT; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(JUMP_BACKWARD_NO_JIT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(JUMP_BACKWARD_NO_JIT); static_assert(1 == 1, "incorrect cache size"); /* Skip 1 cache entry */ // _CHECK_PERIODIC @@ -8473,15 +8473,15 @@ int opcode = JUMP_FORWARD; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(JUMP_FORWARD); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(JUMP_FORWARD); JUMPBY(oparg); DISPATCH(); } @@ -8491,15 +8491,15 @@ int opcode = LIST_APPEND; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LIST_APPEND); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LIST_APPEND); _PyStackRef list; _PyStackRef v; v = stack_pointer[-1]; @@ -8519,15 +8519,15 @@ int opcode = LIST_EXTEND; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LIST_EXTEND); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LIST_EXTEND); _PyStackRef list_st; _PyStackRef iterable_st; iterable_st = stack_pointer[-1]; @@ -8572,18 +8572,18 @@ int opcode = LOAD_ATTR; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR); PREDICTED_LOAD_ATTR:; _Py_CODEUNIT* const this_instr = next_instr - 10; (void)this_instr; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; _PyStackRef owner; _PyStackRef *attr; _PyStackRef *self_or_null; @@ -8659,17 +8659,17 @@ int opcode = LOAD_ATTR_CLASS; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_CLASS); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_CLASS); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -8723,17 +8723,17 @@ int opcode = LOAD_ATTR_CLASS_WITH_METACLASS_CHECK; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -8797,17 +8797,17 @@ int opcode = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; /* Skip 1 cache entry */ @@ -8861,17 +8861,17 @@ int opcode = LOAD_ATTR_INSTANCE_VALUE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -8947,17 +8947,17 @@ int opcode = LOAD_ATTR_METHOD_LAZY_DICT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -9009,17 +9009,17 @@ int opcode = LOAD_ATTR_METHOD_NO_DICT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -9061,17 +9061,17 @@ int opcode = LOAD_ATTR_METHOD_WITH_VALUES; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -9134,17 +9134,17 @@ int opcode = LOAD_ATTR_MODULE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_MODULE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_MODULE); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -9214,17 +9214,17 @@ int opcode = LOAD_ATTR_NONDESCRIPTOR_NO_DICT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -9267,17 +9267,17 @@ int opcode = LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -9341,17 +9341,17 @@ int opcode = LOAD_ATTR_PROPERTY; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef new_frame; @@ -9441,17 +9441,17 @@ int opcode = LOAD_ATTR_SLOT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_SLOT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_SLOT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -9516,17 +9516,17 @@ int opcode = LOAD_ATTR_WITH_HINT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -9632,15 +9632,15 @@ int opcode = LOAD_BUILD_CLASS; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_BUILD_CLASS); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_BUILD_CLASS); _PyStackRef bc; PyObject *bc_o; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -9668,15 +9668,15 @@ int opcode = LOAD_COMMON_CONSTANT; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); _PyStackRef value; assert(oparg < NUM_COMMON_CONSTANTS); value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); @@ -9691,15 +9691,15 @@ int opcode = LOAD_CONST; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_CONST); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_CONST); _PyStackRef value; PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); value = PyStackRef_FromPyObjectBorrow(obj); @@ -9714,15 +9714,15 @@ int opcode = LOAD_DEREF; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_DEREF); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_DEREF); _PyStackRef value; PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -9748,15 +9748,15 @@ int opcode = LOAD_FAST; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST); _PyStackRef value; assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_DUP(GETLOCAL(oparg)); @@ -9771,15 +9771,15 @@ int opcode = LOAD_FAST_AND_CLEAR; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); _PyStackRef value; value = GETLOCAL(oparg); GETLOCAL(oparg) = PyStackRef_NULL; @@ -9794,15 +9794,15 @@ int opcode = LOAD_FAST_BORROW; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_BORROW); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_BORROW); _PyStackRef value; assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_Borrow(GETLOCAL(oparg)); @@ -9817,15 +9817,15 @@ int opcode = LOAD_FAST_BORROW_LOAD_FAST_BORROW; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_BORROW_LOAD_FAST_BORROW); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_BORROW_LOAD_FAST_BORROW); _PyStackRef value1; _PyStackRef value2; uint32_t oparg1 = oparg >> 4; @@ -9844,15 +9844,15 @@ int opcode = LOAD_FAST_CHECK; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_CHECK); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_CHECK); _PyStackRef value; _PyStackRef value_s = GETLOCAL(oparg); if (PyStackRef_IsNull(value_s)) { @@ -9876,15 +9876,15 @@ int opcode = LOAD_FAST_LOAD_FAST; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); _PyStackRef value1; _PyStackRef value2; uint32_t oparg1 = oparg >> 4; @@ -9903,15 +9903,15 @@ int opcode = LOAD_FROM_DICT_OR_DEREF; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); _PyStackRef class_dict_st; _PyStackRef value; class_dict_st = stack_pointer[-1]; @@ -9954,15 +9954,15 @@ int opcode = LOAD_FROM_DICT_OR_GLOBALS; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); _PyStackRef mod_or_class_dict; _PyStackRef v; mod_or_class_dict = stack_pointer[-1]; @@ -10035,18 +10035,18 @@ int opcode = LOAD_GLOBAL; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL); PREDICTED_LOAD_GLOBAL:; _Py_CODEUNIT* const this_instr = next_instr - 5; (void)this_instr; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; _PyStackRef *res; _PyStackRef *null; // _SPECIALIZE_LOAD_GLOBAL @@ -10097,17 +10097,17 @@ int opcode = LOAD_GLOBAL_BUILTIN; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); _PyStackRef res; _PyStackRef *null; @@ -10183,17 +10183,17 @@ int opcode = LOAD_GLOBAL_MODULE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); _PyStackRef res; _PyStackRef *null; @@ -10256,15 +10256,15 @@ int opcode = LOAD_LOCALS; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_LOCALS); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_LOCALS); _PyStackRef locals; PyObject *l = LOCALS(); if (l == NULL) { @@ -10286,15 +10286,15 @@ int opcode = LOAD_NAME; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_NAME); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_NAME); _PyStackRef v; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -10315,15 +10315,15 @@ int opcode = LOAD_SMALL_INT; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_SMALL_INT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_SMALL_INT); _PyStackRef value; assert(oparg < _PY_NSMALLPOSINTS); PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; @@ -10339,15 +10339,15 @@ int opcode = LOAD_SPECIAL; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_SPECIAL); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_SPECIAL); _PyStackRef self; _PyStackRef *method_and_self; // _INSERT_NULL @@ -10391,18 +10391,18 @@ int opcode = LOAD_SUPER_ATTR; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR); PREDICTED_LOAD_SUPER_ATTR:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; opcode = LOAD_SUPER_ATTR; _PyStackRef global_super_st; _PyStackRef class_st; @@ -10534,17 +10534,17 @@ int opcode = LOAD_SUPER_ATTR_ATTR; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); _PyStackRef global_super_st; _PyStackRef class_st; @@ -10602,17 +10602,17 @@ int opcode = LOAD_SUPER_ATTR_METHOD; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); _PyStackRef global_super_st; _PyStackRef class_st; @@ -10686,15 +10686,15 @@ int opcode = MAKE_CELL; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAKE_CELL); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MAKE_CELL); PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); PyObject *cell = PyCell_New(initial); if (cell == NULL) { @@ -10713,15 +10713,15 @@ int opcode = MAKE_FUNCTION; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAKE_FUNCTION); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MAKE_FUNCTION); _PyStackRef codeobj_st; _PyStackRef func; codeobj_st = stack_pointer[-1]; @@ -10752,15 +10752,15 @@ int opcode = MAP_ADD; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAP_ADD); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MAP_ADD); _PyStackRef dict_st; _PyStackRef key; _PyStackRef value; @@ -10789,15 +10789,15 @@ int opcode = MATCH_CLASS; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_CLASS); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_CLASS); _PyStackRef subject; _PyStackRef type; _PyStackRef names; @@ -10847,15 +10847,15 @@ int opcode = MATCH_KEYS; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_KEYS); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_KEYS); _PyStackRef subject; _PyStackRef keys; _PyStackRef values_or_none; @@ -10880,15 +10880,15 @@ int opcode = MATCH_MAPPING; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_MAPPING); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_MAPPING); _PyStackRef subject; _PyStackRef res; subject = stack_pointer[-1]; @@ -10905,15 +10905,15 @@ int opcode = MATCH_SEQUENCE; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_SEQUENCE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_SEQUENCE); _PyStackRef subject; _PyStackRef res; subject = stack_pointer[-1]; @@ -10930,15 +10930,15 @@ int opcode = NOP; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(NOP); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(NOP); DISPATCH(); } @@ -10947,15 +10947,15 @@ int opcode = NOT_TAKEN; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(NOT_TAKEN); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(NOT_TAKEN); DISPATCH(); } @@ -10964,15 +10964,15 @@ int opcode = POP_EXCEPT; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_EXCEPT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(POP_EXCEPT); _PyStackRef exc_value; exc_value = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; @@ -10991,15 +10991,15 @@ int opcode = POP_ITER; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_ITER); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(POP_ITER); _PyStackRef iter; _PyStackRef index_or_null; index_or_null = stack_pointer[-1]; @@ -11018,15 +11018,15 @@ int opcode = POP_JUMP_IF_FALSE; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_FALSE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_FALSE); _PyStackRef cond; /* Skip 1 cache entry */ cond = stack_pointer[-1]; @@ -11043,15 +11043,15 @@ int opcode = POP_JUMP_IF_NONE; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_NONE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_NONE); _PyStackRef value; _PyStackRef b; _PyStackRef cond; @@ -11089,15 +11089,15 @@ int opcode = POP_JUMP_IF_NOT_NONE; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); _PyStackRef value; _PyStackRef b; _PyStackRef cond; @@ -11135,15 +11135,15 @@ int opcode = POP_JUMP_IF_TRUE; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_TRUE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_TRUE); _PyStackRef cond; /* Skip 1 cache entry */ cond = stack_pointer[-1]; @@ -11160,15 +11160,15 @@ int opcode = POP_TOP; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_TOP); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(POP_TOP); _PyStackRef value; value = stack_pointer[-1]; stack_pointer += -1; @@ -11184,15 +11184,15 @@ int opcode = PUSH_EXC_INFO; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(PUSH_EXC_INFO); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(PUSH_EXC_INFO); _PyStackRef exc; _PyStackRef prev_exc; _PyStackRef new_exc; @@ -11219,15 +11219,15 @@ int opcode = PUSH_NULL; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(PUSH_NULL); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(PUSH_NULL); _PyStackRef res; res = PyStackRef_NULL; stack_pointer[0] = res; @@ -11241,17 +11241,17 @@ int opcode = RAISE_VARARGS; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RAISE_VARARGS); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RAISE_VARARGS); _PyStackRef *args; args = &stack_pointer[-oparg]; assert(oparg < 3); @@ -11276,17 +11276,17 @@ int opcode = RERAISE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RERAISE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RERAISE); _PyStackRef *values; _PyStackRef exc_st; exc_st = stack_pointer[-1]; @@ -11310,15 +11310,15 @@ int opcode = RESERVED; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESERVED); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RESERVED); assert(0 && "Executing RESERVED instruction."); Py_FatalError("Executing RESERVED instruction."); DISPATCH(); @@ -11329,18 +11329,18 @@ int opcode = RESUME; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESUME); PREDICTED_RESUME:; _Py_CODEUNIT* const this_instr = next_instr - 1; (void)this_instr; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; // _LOAD_BYTECODE { #ifdef Py_GIL_DISABLED @@ -11411,17 +11411,17 @@ int opcode = RESUME_CHECK; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESUME_CHECK); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RESUME_CHECK); static_assert(0 == 0, "incorrect cache size"); #if defined(__EMSCRIPTEN__) if (_Py_emscripten_signal_clock == 0) { @@ -11455,15 +11455,15 @@ int opcode = RETURN_GENERATOR; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_GENERATOR); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RETURN_GENERATOR); _PyStackRef res; assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -11503,15 +11503,15 @@ int opcode = RETURN_VALUE; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_VALUE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RETURN_VALUE); _PyStackRef retval; _PyStackRef res; retval = stack_pointer[-1]; @@ -11543,18 +11543,18 @@ int opcode = SEND; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(SEND); PREDICTED_SEND:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; _PyStackRef receiver; _PyStackRef v; _PyStackRef retval; @@ -11654,17 +11654,17 @@ int opcode = SEND_GEN; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND_GEN); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(SEND_GEN); static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); _PyStackRef receiver; _PyStackRef v; @@ -11729,15 +11729,15 @@ int opcode = SETUP_ANNOTATIONS; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SETUP_ANNOTATIONS); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SETUP_ANNOTATIONS); PyObject *ann_dict; if (LOCALS() == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -11781,15 +11781,15 @@ int opcode = SET_ADD; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_ADD); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SET_ADD); _PyStackRef set; _PyStackRef v; v = stack_pointer[-1]; @@ -11811,15 +11811,15 @@ int opcode = SET_FUNCTION_ATTRIBUTE; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); _PyStackRef attr_st; _PyStackRef func_in; _PyStackRef func_out; @@ -11845,15 +11845,15 @@ int opcode = SET_UPDATE; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_UPDATE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SET_UPDATE); _PyStackRef set; _PyStackRef iterable; iterable = stack_pointer[-1]; @@ -11878,18 +11878,18 @@ int opcode = STORE_ATTR; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR); PREDICTED_STORE_ATTR:; _Py_CODEUNIT* const this_instr = next_instr - 5; (void)this_instr; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; _PyStackRef owner; _PyStackRef v; // _SPECIALIZE_STORE_ATTR @@ -11941,17 +11941,17 @@ int opcode = STORE_ATTR_INSTANCE_VALUE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); _PyStackRef owner; _PyStackRef value; @@ -12023,17 +12023,17 @@ int opcode = STORE_ATTR_SLOT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_SLOT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_SLOT); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); _PyStackRef owner; _PyStackRef value; @@ -12080,17 +12080,17 @@ int opcode = STORE_ATTR_WITH_HINT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); _PyStackRef owner; _PyStackRef value; @@ -12174,15 +12174,15 @@ int opcode = STORE_DEREF; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_DEREF); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_DEREF); _PyStackRef v; v = stack_pointer[-1]; PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); @@ -12199,15 +12199,15 @@ int opcode = STORE_FAST; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST); _PyStackRef value; value = stack_pointer[-1]; _PyStackRef tmp = GETLOCAL(oparg); @@ -12225,15 +12225,15 @@ int opcode = STORE_FAST_LOAD_FAST; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); _PyStackRef value1; _PyStackRef value2; value1 = stack_pointer[-1]; @@ -12254,15 +12254,15 @@ int opcode = STORE_FAST_STORE_FAST; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST_STORE_FAST); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST_STORE_FAST); _PyStackRef value2; _PyStackRef value1; value1 = stack_pointer[-1]; @@ -12291,15 +12291,15 @@ int opcode = STORE_GLOBAL; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_GLOBAL); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_GLOBAL); _PyStackRef v; v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -12322,15 +12322,15 @@ int opcode = STORE_NAME; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_NAME); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_NAME); _PyStackRef v; v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -12374,15 +12374,15 @@ int opcode = STORE_SLICE; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_SLICE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_SLICE); _PyStackRef v; _PyStackRef container; _PyStackRef start; @@ -12440,18 +12440,18 @@ int opcode = STORE_SUBSCR; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR); PREDICTED_STORE_SUBSCR:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; _PyStackRef container; _PyStackRef sub; _PyStackRef v; @@ -12505,17 +12505,17 @@ int opcode = STORE_SUBSCR_DICT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_DICT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR_DICT); static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); _PyStackRef nos; _PyStackRef value; @@ -12562,17 +12562,17 @@ int opcode = STORE_SUBSCR_LIST_INT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; @@ -12649,15 +12649,15 @@ int opcode = SWAP; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SWAP); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SWAP); _PyStackRef bottom; _PyStackRef top; top = stack_pointer[-1]; @@ -12675,18 +12675,18 @@ int opcode = TO_BOOL; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL); PREDICTED_TO_BOOL:; _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; _PyStackRef value; _PyStackRef res; // _SPECIALIZE_TO_BOOL @@ -12733,17 +12733,17 @@ int opcode = TO_BOOL_ALWAYS_TRUE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef owner; _PyStackRef value; @@ -12782,17 +12782,17 @@ int opcode = TO_BOOL_BOOL; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_BOOL); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_BOOL); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef value; /* Skip 1 cache entry */ @@ -12812,17 +12812,17 @@ int opcode = TO_BOOL_INT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_INT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_INT); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef value; _PyStackRef res; @@ -12858,17 +12858,17 @@ int opcode = TO_BOOL_LIST; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_LIST); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_LIST); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef tos; _PyStackRef value; @@ -12907,17 +12907,17 @@ int opcode = TO_BOOL_NONE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_NONE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_NONE); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef value; _PyStackRef res; @@ -12940,17 +12940,17 @@ int opcode = TO_BOOL_STR; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_STR); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_STR); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef value; _PyStackRef res; @@ -12994,15 +12994,15 @@ int opcode = UNARY_INVERT; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_INVERT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_INVERT); _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; @@ -13029,15 +13029,15 @@ int opcode = UNARY_NEGATIVE; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_NEGATIVE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_NEGATIVE); _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; @@ -13064,15 +13064,15 @@ int opcode = UNARY_NOT; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_NOT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_NOT); _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; @@ -13088,15 +13088,15 @@ int opcode = UNPACK_EX; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNPACK_EX); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNPACK_EX); _PyStackRef seq; _PyStackRef *top; seq = stack_pointer[-1]; @@ -13121,18 +13121,18 @@ int opcode = UNPACK_SEQUENCE; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE); PREDICTED_UNPACK_SEQUENCE:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; _PyStackRef seq; _PyStackRef *top; // _SPECIALIZE_UNPACK_SEQUENCE @@ -13178,17 +13178,17 @@ int opcode = UNPACK_SEQUENCE_LIST; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); _PyStackRef tos; _PyStackRef seq; @@ -13243,17 +13243,17 @@ int opcode = UNPACK_SEQUENCE_TUPLE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); _PyStackRef tos; _PyStackRef seq; @@ -13299,17 +13299,17 @@ int opcode = UNPACK_SEQUENCE_TWO_TUPLE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); _PyStackRef tos; _PyStackRef seq; @@ -13356,15 +13356,15 @@ int opcode = WITH_EXCEPT_START; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(WITH_EXCEPT_START); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(WITH_EXCEPT_START); _PyStackRef exit_func; _PyStackRef exit_self; _PyStackRef lasti; @@ -13407,15 +13407,15 @@ int opcode = YIELD_VALUE; (void)(opcode); #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(YIELD_VALUE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(YIELD_VALUE); _PyStackRef retval; _PyStackRef value; retval = stack_pointer[-1]; @@ -13465,18 +13465,18 @@ int opcode = BINARY_OP; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP); PREDICTED_TRACING_BINARY_OP:; _Py_CODEUNIT* const this_instr = next_instr - 6; (void)this_instr; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; _PyStackRef lhs; _PyStackRef rhs; _PyStackRef res; @@ -13535,17 +13535,17 @@ int opcode = BINARY_OP_ADD_FLOAT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef left; @@ -13600,17 +13600,17 @@ int opcode = BINARY_OP_ADD_INT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_ADD_INT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_ADD_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef left; @@ -13667,17 +13667,17 @@ int opcode = BINARY_OP_ADD_UNICODE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; @@ -13734,17 +13734,17 @@ int opcode = BINARY_OP_EXTEND; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_EXTEND); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_EXTEND); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; _PyStackRef right; @@ -13805,17 +13805,17 @@ int opcode = BINARY_OP_INPLACE_ADD_UNICODE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; @@ -13895,17 +13895,17 @@ int opcode = BINARY_OP_MULTIPLY_FLOAT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef left; @@ -13960,17 +13960,17 @@ int opcode = BINARY_OP_MULTIPLY_INT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef left; @@ -14027,17 +14027,17 @@ int opcode = BINARY_OP_SUBSCR_DICT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_DICT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBSCR_DICT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef nos; _PyStackRef dict_st; @@ -14100,17 +14100,17 @@ int opcode = BINARY_OP_SUBSCR_GETITEM; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_GETITEM); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBSCR_GETITEM); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef container; _PyStackRef getitem; @@ -14191,17 +14191,17 @@ int opcode = BINARY_OP_SUBSCR_LIST_INT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_INT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; @@ -14288,17 +14288,17 @@ int opcode = BINARY_OP_SUBSCR_LIST_SLICE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_SLICE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_SLICE); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef tos; _PyStackRef nos; @@ -14367,17 +14367,17 @@ int opcode = BINARY_OP_SUBSCR_STR_INT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_STR_INT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBSCR_STR_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; @@ -14452,17 +14452,17 @@ int opcode = BINARY_OP_SUBSCR_TUPLE_INT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_TUPLE_INT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBSCR_TUPLE_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; @@ -14532,17 +14532,17 @@ int opcode = BINARY_OP_SUBTRACT_FLOAT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef left; @@ -14597,17 +14597,17 @@ int opcode = BINARY_OP_SUBTRACT_INT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef left; @@ -14664,17 +14664,17 @@ int opcode = BINARY_SLICE; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BINARY_SLICE); + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; _PyStackRef container; _PyStackRef start; _PyStackRef stop; @@ -14729,17 +14729,17 @@ int opcode = BUILD_INTERPOLATION; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_INTERPOLATION); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_INTERPOLATION); _PyStackRef value; _PyStackRef str; _PyStackRef *format; @@ -14796,17 +14796,17 @@ int opcode = BUILD_LIST; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_LIST); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_LIST); _PyStackRef *values; _PyStackRef list; values = &stack_pointer[-oparg]; @@ -14829,17 +14829,17 @@ int opcode = BUILD_MAP; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_MAP); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_MAP); _PyStackRef *values; _PyStackRef map; values = &stack_pointer[-oparg*2]; @@ -14890,17 +14890,17 @@ int opcode = BUILD_SET; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_SET); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_SET); _PyStackRef *values; _PyStackRef set; values = &stack_pointer[-oparg]; @@ -14956,17 +14956,17 @@ int opcode = BUILD_SLICE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_SLICE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_SLICE); _PyStackRef *args; _PyStackRef slice; args = &stack_pointer[-oparg]; @@ -15000,17 +15000,17 @@ int opcode = BUILD_STRING; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_STRING); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_STRING); _PyStackRef *pieces; _PyStackRef str; pieces = &stack_pointer[-oparg]; @@ -15056,17 +15056,17 @@ int opcode = BUILD_TEMPLATE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_TEMPLATE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_TEMPLATE); _PyStackRef strings; _PyStackRef interpolations; _PyStackRef template; @@ -15103,17 +15103,17 @@ int opcode = BUILD_TUPLE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_TUPLE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_TUPLE); _PyStackRef *values; _PyStackRef tup; values = &stack_pointer[-oparg]; @@ -15134,17 +15134,17 @@ int opcode = CACHE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CACHE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CACHE); assert(0 && "Executing a cache."); Py_FatalError("Executing a cache."); TRACING_DISPATCH(); @@ -15156,18 +15156,18 @@ int opcode = CALL; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL); PREDICTED_TRACING_CALL:; _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; opcode = CALL; _PyStackRef callable; _PyStackRef self_or_null; @@ -15341,17 +15341,17 @@ int opcode = CALL_ALLOC_AND_ENTER_INIT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -15469,17 +15469,17 @@ int opcode = CALL_BOUND_METHOD_EXACT_ARGS; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef null; @@ -15619,17 +15619,17 @@ int opcode = CALL_BOUND_METHOD_GENERAL; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef null; @@ -15754,17 +15754,17 @@ int opcode = CALL_BUILTIN_CLASS; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_CLASS); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_CLASS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -15866,17 +15866,17 @@ int opcode = CALL_BUILTIN_FAST; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_FAST); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -15982,17 +15982,17 @@ int opcode = CALL_BUILTIN_FAST_WITH_KEYWORDS; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -16098,17 +16098,17 @@ int opcode = CALL_BUILTIN_O; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_O); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_O); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -16189,17 +16189,17 @@ int opcode = CALL_FUNCTION_EX; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_FUNCTION_EX); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_FUNCTION_EX); opcode = CALL_FUNCTION_EX; _PyStackRef func; _PyStackRef callargs; @@ -16363,17 +16363,17 @@ int opcode = CALL_INTRINSIC_1; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_INTRINSIC_1); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_INTRINSIC_1); _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; @@ -16402,17 +16402,17 @@ int opcode = CALL_INTRINSIC_2; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_INTRINSIC_2); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_INTRINSIC_2); _PyStackRef value2_st; _PyStackRef value1_st; _PyStackRef res; @@ -16450,17 +16450,17 @@ int opcode = CALL_ISINSTANCE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_ISINSTANCE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_ISINSTANCE); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef null; _PyStackRef callable; @@ -16533,18 +16533,18 @@ int opcode = CALL_KW; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW); PREDICTED_TRACING_CALL_KW:; _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; opcode = CALL_KW; _PyStackRef callable; _PyStackRef self_or_null; @@ -16722,17 +16722,17 @@ int opcode = CALL_KW_BOUND_METHOD; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef null; @@ -16859,17 +16859,17 @@ int opcode = CALL_KW_NON_PY; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_NON_PY); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_KW_NON_PY); opcode = CALL_KW_NON_PY; static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); _PyStackRef callable; @@ -16993,17 +16993,17 @@ int opcode = CALL_KW_PY; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW_PY); + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -17110,17 +17110,17 @@ int opcode = CALL_LEN; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_LEN); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_LEN); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef null; _PyStackRef callable; @@ -17189,17 +17189,17 @@ int opcode = CALL_LIST_APPEND; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_LIST_APPEND); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_LIST_APPEND); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef nos; @@ -17284,17 +17284,17 @@ int opcode = CALL_METHOD_DESCRIPTOR_FAST; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -17412,17 +17412,17 @@ int opcode = CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -17542,17 +17542,17 @@ int opcode = CALL_METHOD_DESCRIPTOR_NOARGS; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -17642,17 +17642,17 @@ int opcode = CALL_METHOD_DESCRIPTOR_O; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -17755,17 +17755,17 @@ int opcode = CALL_NON_PY_GENERAL; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_NON_PY_GENERAL); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_NON_PY_GENERAL); opcode = CALL_NON_PY_GENERAL; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; @@ -17877,17 +17877,17 @@ int opcode = CALL_PY_EXACT_ARGS; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -17997,17 +17997,17 @@ int opcode = CALL_PY_GENERAL; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_GENERAL); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_PY_GENERAL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -18104,17 +18104,17 @@ int opcode = CALL_STR_1; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_STR_1); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_STR_1); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef null; _PyStackRef callable; @@ -18183,17 +18183,17 @@ int opcode = CALL_TUPLE_1; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_TUPLE_1); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_TUPLE_1); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef null; _PyStackRef callable; @@ -18262,17 +18262,17 @@ int opcode = CALL_TYPE_1; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_TYPE_1); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_TYPE_1); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef null; _PyStackRef callable; @@ -18324,17 +18324,17 @@ int opcode = CHECK_EG_MATCH; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CHECK_EG_MATCH); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CHECK_EG_MATCH); _PyStackRef exc_value_st; _PyStackRef match_type_st; _PyStackRef rest; @@ -18404,17 +18404,17 @@ int opcode = CHECK_EXC_MATCH; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CHECK_EXC_MATCH); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CHECK_EXC_MATCH); _PyStackRef left; _PyStackRef right; _PyStackRef b; @@ -18450,17 +18450,17 @@ int opcode = CLEANUP_THROW; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CLEANUP_THROW); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CLEANUP_THROW); _PyStackRef sub_iter; _PyStackRef last_sent_val; _PyStackRef exc_value_st; @@ -18516,18 +18516,18 @@ int opcode = COMPARE_OP; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP); PREDICTED_TRACING_COMPARE_OP:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; _PyStackRef left; _PyStackRef right; _PyStackRef res; @@ -18596,17 +18596,17 @@ int opcode = COMPARE_OP_FLOAT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_FLOAT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_FLOAT); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); _PyStackRef value; _PyStackRef left; @@ -18658,17 +18658,17 @@ int opcode = COMPARE_OP_INT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_INT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_INT); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); _PyStackRef value; _PyStackRef left; @@ -18724,17 +18724,17 @@ int opcode = COMPARE_OP_STR; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_STR); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_STR); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; @@ -18790,18 +18790,18 @@ int opcode = CONTAINS_OP; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP); PREDICTED_TRACING_CONTAINS_OP:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; _PyStackRef right; _PyStackRef left; _PyStackRef b; @@ -18857,17 +18857,17 @@ int opcode = CONTAINS_OP_DICT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP_DICT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(CONTAINS_OP_DICT); static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); _PyStackRef tos; _PyStackRef left; @@ -18922,17 +18922,17 @@ int opcode = CONTAINS_OP_SET; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP_SET); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(CONTAINS_OP_SET); static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); _PyStackRef tos; _PyStackRef left; @@ -18987,17 +18987,17 @@ int opcode = CONVERT_VALUE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CONVERT_VALUE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CONVERT_VALUE); _PyStackRef value; _PyStackRef result; value = stack_pointer[-1]; @@ -19028,17 +19028,17 @@ int opcode = COPY; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(COPY); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(COPY); _PyStackRef bottom; _PyStackRef top; bottom = stack_pointer[-1 - (oparg-1)]; @@ -19055,17 +19055,17 @@ int opcode = COPY_FREE_VARS; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(COPY_FREE_VARS); + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; PyCodeObject *co = _PyFrame_GetCode(frame); assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -19085,17 +19085,17 @@ int opcode = DELETE_ATTR; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_ATTR); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_ATTR); _PyStackRef owner; owner = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -19119,17 +19119,17 @@ int opcode = DELETE_DEREF; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_DEREF); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_DEREF); PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); if (oldobj == NULL) { @@ -19150,17 +19150,17 @@ int opcode = DELETE_FAST; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_FAST); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_FAST); _PyStackRef v = GETLOCAL(oparg); if (PyStackRef_IsNull(v)) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -19185,17 +19185,17 @@ int opcode = DELETE_GLOBAL; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_GLOBAL); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_GLOBAL); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyDict_Pop(GLOBALS(), name, NULL); @@ -19219,17 +19219,17 @@ int opcode = DELETE_NAME; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_NAME); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_NAME); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *ns = LOCALS(); int err; @@ -19260,17 +19260,17 @@ int opcode = DELETE_SUBSCR; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_SUBSCR); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_SUBSCR); _PyStackRef container; _PyStackRef sub; sub = stack_pointer[-1]; @@ -19301,17 +19301,17 @@ int opcode = DICT_MERGE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DICT_MERGE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DICT_MERGE); _PyStackRef callable; _PyStackRef dict; _PyStackRef update; @@ -19349,17 +19349,17 @@ int opcode = DICT_UPDATE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DICT_UPDATE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DICT_UPDATE); _PyStackRef dict; _PyStackRef update; update = stack_pointer[-1]; @@ -19401,17 +19401,17 @@ int opcode = END_ASYNC_FOR; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(END_ASYNC_FOR); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(END_ASYNC_FOR); _PyStackRef awaitable_st; _PyStackRef exc_st; exc_st = stack_pointer[-1]; @@ -19453,16 +19453,16 @@ int opcode = END_FOR; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(END_FOR); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(END_FOR); _PyStackRef value; value = stack_pointer[-1]; stack_pointer += -1; @@ -19479,17 +19479,17 @@ int opcode = END_SEND; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(END_SEND); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(END_SEND); _PyStackRef receiver; _PyStackRef value; _PyStackRef val; @@ -19511,17 +19511,17 @@ int opcode = ENTER_EXECUTOR; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(ENTER_EXECUTOR); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(ENTER_EXECUTOR); opcode = ENTER_EXECUTOR; #ifdef _Py_TIER2 PyCodeObject *code = _PyFrame_GetCode(frame); @@ -19557,17 +19557,17 @@ int opcode = EXIT_INIT_CHECK; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(EXIT_INIT_CHECK); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(EXIT_INIT_CHECK); _PyStackRef should_be_none; should_be_none = stack_pointer[-1]; if (!PyStackRef_IsNone(should_be_none)) { @@ -19589,17 +19589,17 @@ int opcode = EXTENDED_ARG; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(EXTENDED_ARG); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(EXTENDED_ARG); opcode = EXTENDED_ARG; assert(oparg); opcode = next_instr->op.code; @@ -19614,17 +19614,17 @@ int opcode = FORMAT_SIMPLE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(FORMAT_SIMPLE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(FORMAT_SIMPLE); _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; @@ -19659,17 +19659,17 @@ int opcode = FORMAT_WITH_SPEC; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(FORMAT_WITH_SPEC); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(FORMAT_WITH_SPEC); _PyStackRef value; _PyStackRef fmt_spec; _PyStackRef res; @@ -19704,18 +19704,18 @@ int opcode = FOR_ITER; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER); PREDICTED_TRACING_FOR_ITER:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; _PyStackRef iter; _PyStackRef null_or_index; _PyStackRef next; @@ -19766,17 +19766,17 @@ int opcode = FOR_ITER_GEN; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_GEN); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_GEN); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; _PyStackRef gen_frame; @@ -19844,17 +19844,17 @@ int opcode = FOR_ITER_LIST; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_LIST); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_LIST); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; _PyStackRef null_or_index; @@ -19936,17 +19936,17 @@ int opcode = FOR_ITER_RANGE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_RANGE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_RANGE); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; _PyStackRef next; @@ -20011,17 +20011,17 @@ int opcode = FOR_ITER_TUPLE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_TUPLE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_TUPLE); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; _PyStackRef null_or_index; @@ -20075,17 +20075,17 @@ int opcode = GET_AITER; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_AITER); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_AITER); _PyStackRef obj; _PyStackRef iter; obj = stack_pointer[-1]; @@ -20145,17 +20145,17 @@ int opcode = GET_ANEXT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_ANEXT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_ANEXT); _PyStackRef aiter; _PyStackRef awaitable; aiter = stack_pointer[-1]; @@ -20178,17 +20178,17 @@ int opcode = GET_AWAITABLE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_AWAITABLE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_AWAITABLE); _PyStackRef iterable; _PyStackRef iter; iterable = stack_pointer[-1]; @@ -20216,17 +20216,17 @@ int opcode = GET_ITER; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_ITER); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_ITER); _PyStackRef iterable; _PyStackRef iter; _PyStackRef index_or_null; @@ -20271,17 +20271,17 @@ int opcode = GET_LEN; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_LEN); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_LEN); _PyStackRef obj; _PyStackRef len; obj = stack_pointer[-1]; @@ -20308,17 +20308,17 @@ int opcode = GET_YIELD_FROM_ITER; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_YIELD_FROM_ITER); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_YIELD_FROM_ITER); _PyStackRef iterable; _PyStackRef iter; iterable = stack_pointer[-1]; @@ -20362,17 +20362,17 @@ int opcode = IMPORT_FROM; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IMPORT_FROM); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IMPORT_FROM); _PyStackRef from; _PyStackRef res; from = stack_pointer[-1]; @@ -20396,17 +20396,17 @@ int opcode = IMPORT_NAME; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IMPORT_NAME); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IMPORT_NAME); _PyStackRef level; _PyStackRef fromlist; _PyStackRef res; @@ -20444,17 +20444,17 @@ int opcode = INSTRUMENTED_CALL; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(INSTRUMENTED_CALL); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(INSTRUMENTED_CALL); opcode = INSTRUMENTED_CALL; _PyStackRef callable; _PyStackRef self_or_null; @@ -20639,17 +20639,17 @@ int opcode = INSTRUMENTED_CALL_FUNCTION_EX; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); opcode = INSTRUMENTED_CALL_FUNCTION_EX; _PyStackRef func; _PyStackRef callargs; @@ -20813,17 +20813,17 @@ int opcode = INSTRUMENTED_CALL_KW; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); opcode = INSTRUMENTED_CALL_KW; _PyStackRef callable; _PyStackRef self_or_null; @@ -21006,17 +21006,17 @@ int opcode = INSTRUMENTED_END_ASYNC_FOR; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_ASYNC_FOR); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_END_ASYNC_FOR); _PyStackRef awaitable_st; _PyStackRef exc_st; // _MONITOR_END_ASYNC_FOR @@ -21066,16 +21066,16 @@ int opcode = INSTRUMENTED_END_FOR; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_FOR); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_END_FOR); _PyStackRef receiver; _PyStackRef value; value = stack_pointer[-1]; @@ -21102,17 +21102,17 @@ int opcode = INSTRUMENTED_END_SEND; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_SEND); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_END_SEND); _PyStackRef receiver; _PyStackRef value; _PyStackRef val; @@ -21143,17 +21143,17 @@ int opcode = INSTRUMENTED_FOR_ITER; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); _PyStackRef iter; _PyStackRef null_or_index; _PyStackRef next; @@ -21186,17 +21186,17 @@ int opcode = INSTRUMENTED_INSTRUCTION; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); opcode = INSTRUMENTED_INSTRUCTION; _PyFrame_SetStackPointer(frame, stack_pointer); int next_opcode = _Py_call_instrumentation_instruction( @@ -21220,17 +21220,17 @@ int opcode = INSTRUMENTED_JUMP_BACKWARD; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); /* Skip 1 cache entry */ // _CHECK_PERIODIC { @@ -21254,17 +21254,17 @@ int opcode = INSTRUMENTED_JUMP_FORWARD; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); TRACING_DISPATCH(); } @@ -21275,18 +21275,18 @@ int opcode = INSTRUMENTED_LINE; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_LINE); + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; opcode = INSTRUMENTED_LINE; int original_opcode = 0; if (tstate->tracing) { @@ -21322,17 +21322,17 @@ int opcode = INSTRUMENTED_LOAD_SUPER_ATTR; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); opcode = INSTRUMENTED_LOAD_SUPER_ATTR; _PyStackRef global_super_st; _PyStackRef class_st; @@ -21449,18 +21449,18 @@ int opcode = INSTRUMENTED_NOT_TAKEN; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_NOT_TAKEN); + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; (void)this_instr; INSTRUMENTED_JUMP(prev_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); TRACING_DISPATCH(); @@ -21472,18 +21472,18 @@ int opcode = INSTRUMENTED_POP_ITER; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _Py_CODEUNIT* const prev_instr = frame->instr_ptr; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_POP_ITER); + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; _PyStackRef iter; _PyStackRef index_or_null; index_or_null = stack_pointer[-1]; @@ -21504,17 +21504,17 @@ int opcode = INSTRUMENTED_POP_JUMP_IF_FALSE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); _PyStackRef cond; /* Skip 1 cache entry */ cond = stack_pointer[-1]; @@ -21535,17 +21535,17 @@ int opcode = INSTRUMENTED_POP_JUMP_IF_NONE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); _PyStackRef value; /* Skip 1 cache entry */ value = stack_pointer[-1]; @@ -21573,17 +21573,17 @@ int opcode = INSTRUMENTED_POP_JUMP_IF_NOT_NONE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); _PyStackRef value; /* Skip 1 cache entry */ value = stack_pointer[-1]; @@ -21609,17 +21609,17 @@ int opcode = INSTRUMENTED_POP_JUMP_IF_TRUE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); _PyStackRef cond; /* Skip 1 cache entry */ cond = stack_pointer[-1]; @@ -21640,17 +21640,17 @@ int opcode = INSTRUMENTED_RESUME; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RESUME); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_RESUME); // _LOAD_BYTECODE { #ifdef Py_GIL_DISABLED @@ -21727,17 +21727,17 @@ int opcode = INSTRUMENTED_RETURN_VALUE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); _PyStackRef val; _PyStackRef retval; _PyStackRef res; @@ -21786,17 +21786,17 @@ int opcode = INSTRUMENTED_YIELD_VALUE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); _PyStackRef val; _PyStackRef retval; _PyStackRef value; @@ -21864,17 +21864,17 @@ int opcode = INTERPRETER_EXIT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INTERPRETER_EXIT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INTERPRETER_EXIT); _PyStackRef retval; retval = stack_pointer[-1]; assert(frame->owner == FRAME_OWNED_BY_INTERPRETER); @@ -21908,17 +21908,17 @@ int opcode = IS_OP; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IS_OP); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IS_OP); _PyStackRef left; _PyStackRef right; _PyStackRef b; @@ -21950,18 +21950,18 @@ int opcode = JUMP_BACKWARD; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(JUMP_BACKWARD); PREDICTED_TRACING_JUMP_BACKWARD:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; /* Skip 1 cache entry */ // _SPECIALIZE_JUMP_BACKWARD { @@ -21998,17 +21998,17 @@ int opcode = JUMP_BACKWARD_JIT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(JUMP_BACKWARD_JIT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(JUMP_BACKWARD_JIT); static_assert(1 == 1, "incorrect cache size"); /* Skip 1 cache entry */ // _CHECK_PERIODIC @@ -22066,17 +22066,17 @@ int opcode = JUMP_BACKWARD_NO_INTERRUPT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); #if TIER_ONE assert(oparg <= INSTR_OFFSET()); #endif @@ -22090,17 +22090,17 @@ int opcode = JUMP_BACKWARD_NO_JIT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(JUMP_BACKWARD_NO_JIT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(JUMP_BACKWARD_NO_JIT); static_assert(1 == 1, "incorrect cache size"); /* Skip 1 cache entry */ // _CHECK_PERIODIC @@ -22128,17 +22128,17 @@ int opcode = JUMP_FORWARD; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(JUMP_FORWARD); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(JUMP_FORWARD); JUMPBY(oparg); TRACING_DISPATCH(); } @@ -22149,17 +22149,17 @@ int opcode = LIST_APPEND; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LIST_APPEND); + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; _PyStackRef list; _PyStackRef v; v = stack_pointer[-1]; @@ -22180,17 +22180,17 @@ int opcode = LIST_EXTEND; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LIST_EXTEND); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LIST_EXTEND); _PyStackRef list_st; _PyStackRef iterable_st; iterable_st = stack_pointer[-1]; @@ -22236,18 +22236,18 @@ int opcode = LOAD_ATTR; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR); PREDICTED_TRACING_LOAD_ATTR:; _Py_CODEUNIT* const this_instr = next_instr - 10; (void)this_instr; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; _PyStackRef owner; _PyStackRef *attr; _PyStackRef *self_or_null; @@ -22324,17 +22324,17 @@ int opcode = LOAD_ATTR_CLASS; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_CLASS); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_CLASS); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -22389,17 +22389,17 @@ int opcode = LOAD_ATTR_CLASS_WITH_METACLASS_CHECK; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -22464,17 +22464,17 @@ int opcode = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; /* Skip 1 cache entry */ @@ -22529,17 +22529,17 @@ int opcode = LOAD_ATTR_INSTANCE_VALUE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -22616,17 +22616,17 @@ int opcode = LOAD_ATTR_METHOD_LAZY_DICT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -22679,17 +22679,17 @@ int opcode = LOAD_ATTR_METHOD_NO_DICT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -22732,17 +22732,17 @@ int opcode = LOAD_ATTR_METHOD_WITH_VALUES; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -22806,17 +22806,17 @@ int opcode = LOAD_ATTR_MODULE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_MODULE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_MODULE); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -22887,17 +22887,17 @@ int opcode = LOAD_ATTR_NONDESCRIPTOR_NO_DICT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -22941,17 +22941,17 @@ int opcode = LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -23016,17 +23016,17 @@ int opcode = LOAD_ATTR_PROPERTY; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef new_frame; @@ -23117,17 +23117,17 @@ int opcode = LOAD_ATTR_SLOT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_SLOT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_SLOT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -23193,17 +23193,17 @@ int opcode = LOAD_ATTR_WITH_HINT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -23310,17 +23310,17 @@ int opcode = LOAD_BUILD_CLASS; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_BUILD_CLASS); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_BUILD_CLASS); _PyStackRef bc; PyObject *bc_o; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -23349,17 +23349,17 @@ int opcode = LOAD_COMMON_CONSTANT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); _PyStackRef value; assert(oparg < NUM_COMMON_CONSTANTS); value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); @@ -23375,17 +23375,17 @@ int opcode = LOAD_CONST; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_CONST); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_CONST); _PyStackRef value; PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); value = PyStackRef_FromPyObjectBorrow(obj); @@ -23401,17 +23401,17 @@ int opcode = LOAD_DEREF; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_DEREF); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_DEREF); _PyStackRef value; PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -23438,17 +23438,17 @@ int opcode = LOAD_FAST; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST); _PyStackRef value; assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_DUP(GETLOCAL(oparg)); @@ -23464,17 +23464,17 @@ int opcode = LOAD_FAST_AND_CLEAR; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); _PyStackRef value; value = GETLOCAL(oparg); GETLOCAL(oparg) = PyStackRef_NULL; @@ -23490,17 +23490,17 @@ int opcode = LOAD_FAST_BORROW; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_BORROW); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_BORROW); _PyStackRef value; assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_Borrow(GETLOCAL(oparg)); @@ -23516,17 +23516,17 @@ int opcode = LOAD_FAST_BORROW_LOAD_FAST_BORROW; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_BORROW_LOAD_FAST_BORROW); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_BORROW_LOAD_FAST_BORROW); _PyStackRef value1; _PyStackRef value2; uint32_t oparg1 = oparg >> 4; @@ -23546,17 +23546,17 @@ int opcode = LOAD_FAST_CHECK; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_CHECK); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_CHECK); _PyStackRef value; _PyStackRef value_s = GETLOCAL(oparg); if (PyStackRef_IsNull(value_s)) { @@ -23581,17 +23581,17 @@ int opcode = LOAD_FAST_LOAD_FAST; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _Py_CODEUNIT* const this_instr = next_instr; (void)this_instr; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; _PyStackRef value1; _PyStackRef value2; uint32_t oparg1 = oparg >> 4; @@ -23611,17 +23611,17 @@ int opcode = LOAD_FROM_DICT_OR_DEREF; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); _PyStackRef class_dict_st; _PyStackRef value; class_dict_st = stack_pointer[-1]; @@ -23665,17 +23665,17 @@ int opcode = LOAD_FROM_DICT_OR_GLOBALS; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); _PyStackRef mod_or_class_dict; _PyStackRef v; mod_or_class_dict = stack_pointer[-1]; @@ -23749,18 +23749,18 @@ int opcode = LOAD_GLOBAL; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL); PREDICTED_TRACING_LOAD_GLOBAL:; _Py_CODEUNIT* const this_instr = next_instr - 5; (void)this_instr; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; _PyStackRef *res; _PyStackRef *null; // _SPECIALIZE_LOAD_GLOBAL @@ -23812,17 +23812,17 @@ int opcode = LOAD_GLOBAL_BUILTIN; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); _PyStackRef res; _PyStackRef *null; @@ -23899,17 +23899,17 @@ int opcode = LOAD_GLOBAL_MODULE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); _PyStackRef res; _PyStackRef *null; @@ -23973,17 +23973,17 @@ int opcode = LOAD_LOCALS; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_LOCALS); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_LOCALS); _PyStackRef locals; PyObject *l = LOCALS(); if (l == NULL) { @@ -24006,17 +24006,17 @@ int opcode = LOAD_NAME; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_NAME); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_NAME); _PyStackRef v; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -24038,17 +24038,17 @@ int opcode = LOAD_SMALL_INT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_SMALL_INT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_SMALL_INT); _PyStackRef value; assert(oparg < _PY_NSMALLPOSINTS); PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; @@ -24065,17 +24065,17 @@ int opcode = LOAD_SPECIAL; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_SPECIAL); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_SPECIAL); _PyStackRef self; _PyStackRef *method_and_self; // _INSERT_NULL @@ -24120,18 +24120,18 @@ int opcode = LOAD_SUPER_ATTR; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR); PREDICTED_TRACING_LOAD_SUPER_ATTR:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; opcode = LOAD_SUPER_ATTR; _PyStackRef global_super_st; _PyStackRef class_st; @@ -24264,17 +24264,17 @@ int opcode = LOAD_SUPER_ATTR_ATTR; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); _PyStackRef global_super_st; _PyStackRef class_st; @@ -24333,17 +24333,17 @@ int opcode = LOAD_SUPER_ATTR_METHOD; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); _PyStackRef global_super_st; _PyStackRef class_st; @@ -24418,17 +24418,17 @@ int opcode = MAKE_CELL; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAKE_CELL); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MAKE_CELL); PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); PyObject *cell = PyCell_New(initial); if (cell == NULL) { @@ -24448,17 +24448,17 @@ int opcode = MAKE_FUNCTION; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAKE_FUNCTION); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MAKE_FUNCTION); _PyStackRef codeobj_st; _PyStackRef func; codeobj_st = stack_pointer[-1]; @@ -24490,17 +24490,17 @@ int opcode = MAP_ADD; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAP_ADD); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MAP_ADD); _PyStackRef dict_st; _PyStackRef key; _PyStackRef value; @@ -24530,17 +24530,17 @@ int opcode = MATCH_CLASS; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_CLASS); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_CLASS); _PyStackRef subject; _PyStackRef type; _PyStackRef names; @@ -24591,17 +24591,17 @@ int opcode = MATCH_KEYS; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_KEYS); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_KEYS); _PyStackRef subject; _PyStackRef keys; _PyStackRef values_or_none; @@ -24627,17 +24627,17 @@ int opcode = MATCH_MAPPING; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_MAPPING); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_MAPPING); _PyStackRef subject; _PyStackRef res; subject = stack_pointer[-1]; @@ -24655,17 +24655,17 @@ int opcode = MATCH_SEQUENCE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_SEQUENCE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_SEQUENCE); _PyStackRef subject; _PyStackRef res; subject = stack_pointer[-1]; @@ -24683,17 +24683,17 @@ int opcode = NOP; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(NOP); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(NOP); TRACING_DISPATCH(); } @@ -24703,17 +24703,17 @@ int opcode = NOT_TAKEN; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(NOT_TAKEN); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(NOT_TAKEN); TRACING_DISPATCH(); } @@ -24723,17 +24723,17 @@ int opcode = POP_EXCEPT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_EXCEPT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(POP_EXCEPT); _PyStackRef exc_value; exc_value = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; @@ -24753,17 +24753,17 @@ int opcode = POP_ITER; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_ITER); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(POP_ITER); _PyStackRef iter; _PyStackRef index_or_null; index_or_null = stack_pointer[-1]; @@ -24783,17 +24783,17 @@ int opcode = POP_JUMP_IF_FALSE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_FALSE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_FALSE); _PyStackRef cond; /* Skip 1 cache entry */ cond = stack_pointer[-1]; @@ -24811,17 +24811,17 @@ int opcode = POP_JUMP_IF_NONE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_NONE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_NONE); _PyStackRef value; _PyStackRef b; _PyStackRef cond; @@ -24860,17 +24860,17 @@ int opcode = POP_JUMP_IF_NOT_NONE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); _PyStackRef value; _PyStackRef b; _PyStackRef cond; @@ -24909,17 +24909,17 @@ int opcode = POP_JUMP_IF_TRUE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_TRUE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_TRUE); _PyStackRef cond; /* Skip 1 cache entry */ cond = stack_pointer[-1]; @@ -24937,17 +24937,17 @@ int opcode = POP_TOP; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_TOP); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(POP_TOP); _PyStackRef value; value = stack_pointer[-1]; stack_pointer += -1; @@ -24964,17 +24964,17 @@ int opcode = PUSH_EXC_INFO; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(PUSH_EXC_INFO); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(PUSH_EXC_INFO); _PyStackRef exc; _PyStackRef prev_exc; _PyStackRef new_exc; @@ -25002,17 +25002,17 @@ int opcode = PUSH_NULL; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(PUSH_NULL); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(PUSH_NULL); _PyStackRef res; res = PyStackRef_NULL; stack_pointer[0] = res; @@ -25027,17 +25027,17 @@ int opcode = RAISE_VARARGS; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RAISE_VARARGS); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RAISE_VARARGS); _PyStackRef *args; args = &stack_pointer[-oparg]; assert(oparg < 3); @@ -25063,17 +25063,17 @@ int opcode = RERAISE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RERAISE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RERAISE); _PyStackRef *values; _PyStackRef exc_st; exc_st = stack_pointer[-1]; @@ -25098,17 +25098,17 @@ int opcode = RESERVED; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESERVED); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RESERVED); assert(0 && "Executing RESERVED instruction."); Py_FatalError("Executing RESERVED instruction."); TRACING_DISPATCH(); @@ -25120,18 +25120,18 @@ int opcode = RESUME; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESUME); PREDICTED_TRACING_RESUME:; _Py_CODEUNIT* const this_instr = next_instr - 1; (void)this_instr; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; // _LOAD_BYTECODE { #ifdef Py_GIL_DISABLED @@ -25203,17 +25203,17 @@ int opcode = RESUME_CHECK; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESUME_CHECK); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RESUME_CHECK); static_assert(0 == 0, "incorrect cache size"); #if defined(__EMSCRIPTEN__) if (_Py_emscripten_signal_clock == 0) { @@ -25248,17 +25248,17 @@ int opcode = RETURN_GENERATOR; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_GENERATOR); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RETURN_GENERATOR); _PyStackRef res; assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -25299,17 +25299,17 @@ int opcode = RETURN_VALUE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_VALUE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RETURN_VALUE); _PyStackRef retval; _PyStackRef res; retval = stack_pointer[-1]; @@ -25342,18 +25342,18 @@ int opcode = SEND; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(SEND); PREDICTED_TRACING_SEND:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; _PyStackRef receiver; _PyStackRef v; _PyStackRef retval; @@ -25454,17 +25454,17 @@ int opcode = SEND_GEN; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND_GEN); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(SEND_GEN); static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); _PyStackRef receiver; _PyStackRef v; @@ -25530,17 +25530,17 @@ int opcode = SETUP_ANNOTATIONS; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SETUP_ANNOTATIONS); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SETUP_ANNOTATIONS); PyObject *ann_dict; if (LOCALS() == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -25585,17 +25585,17 @@ int opcode = SET_ADD; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_ADD); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SET_ADD); _PyStackRef set; _PyStackRef v; v = stack_pointer[-1]; @@ -25618,17 +25618,17 @@ int opcode = SET_FUNCTION_ATTRIBUTE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); _PyStackRef attr_st; _PyStackRef func_in; _PyStackRef func_out; @@ -25655,17 +25655,17 @@ int opcode = SET_UPDATE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_UPDATE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SET_UPDATE); _PyStackRef set; _PyStackRef iterable; iterable = stack_pointer[-1]; @@ -25691,18 +25691,18 @@ int opcode = STORE_ATTR; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR); PREDICTED_TRACING_STORE_ATTR:; _Py_CODEUNIT* const this_instr = next_instr - 5; (void)this_instr; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; _PyStackRef owner; _PyStackRef v; // _SPECIALIZE_STORE_ATTR @@ -25755,17 +25755,17 @@ int opcode = STORE_ATTR_INSTANCE_VALUE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); _PyStackRef owner; _PyStackRef value; @@ -25838,17 +25838,17 @@ int opcode = STORE_ATTR_SLOT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_SLOT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_SLOT); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); _PyStackRef owner; _PyStackRef value; @@ -25896,17 +25896,17 @@ int opcode = STORE_ATTR_WITH_HINT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); _PyStackRef owner; _PyStackRef value; @@ -25991,17 +25991,17 @@ int opcode = STORE_DEREF; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_DEREF); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_DEREF); _PyStackRef v; v = stack_pointer[-1]; PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); @@ -26019,17 +26019,17 @@ int opcode = STORE_FAST; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST); _PyStackRef value; value = stack_pointer[-1]; _PyStackRef tmp = GETLOCAL(oparg); @@ -26048,17 +26048,17 @@ int opcode = STORE_FAST_LOAD_FAST; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); _PyStackRef value1; _PyStackRef value2; value1 = stack_pointer[-1]; @@ -26080,17 +26080,17 @@ int opcode = STORE_FAST_STORE_FAST; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST_STORE_FAST); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST_STORE_FAST); _PyStackRef value2; _PyStackRef value1; value1 = stack_pointer[-1]; @@ -26120,17 +26120,17 @@ int opcode = STORE_GLOBAL; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_GLOBAL); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_GLOBAL); _PyStackRef v; v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -26154,17 +26154,17 @@ int opcode = STORE_NAME; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_NAME); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_NAME); _PyStackRef v; v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -26209,17 +26209,17 @@ int opcode = STORE_SLICE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_SLICE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_SLICE); _PyStackRef v; _PyStackRef container; _PyStackRef start; @@ -26278,18 +26278,18 @@ int opcode = STORE_SUBSCR; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR); PREDICTED_TRACING_STORE_SUBSCR:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; _PyStackRef container; _PyStackRef sub; _PyStackRef v; @@ -26344,17 +26344,17 @@ int opcode = STORE_SUBSCR_DICT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_DICT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR_DICT); static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); _PyStackRef nos; _PyStackRef value; @@ -26402,17 +26402,17 @@ int opcode = STORE_SUBSCR_LIST_INT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; @@ -26490,17 +26490,17 @@ int opcode = SWAP; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SWAP); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SWAP); _PyStackRef bottom; _PyStackRef top; top = stack_pointer[-1]; @@ -26519,18 +26519,18 @@ int opcode = TO_BOOL; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL); PREDICTED_TRACING_TO_BOOL:; _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; _PyStackRef value; _PyStackRef res; // _SPECIALIZE_TO_BOOL @@ -26578,17 +26578,17 @@ int opcode = TO_BOOL_ALWAYS_TRUE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef owner; _PyStackRef value; @@ -26628,17 +26628,17 @@ int opcode = TO_BOOL_BOOL; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_BOOL); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_BOOL); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef value; /* Skip 1 cache entry */ @@ -26659,17 +26659,17 @@ int opcode = TO_BOOL_INT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_INT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_INT); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef value; _PyStackRef res; @@ -26706,17 +26706,17 @@ int opcode = TO_BOOL_LIST; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_LIST); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_LIST); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef tos; _PyStackRef value; @@ -26756,17 +26756,17 @@ int opcode = TO_BOOL_NONE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_NONE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_NONE); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef value; _PyStackRef res; @@ -26790,17 +26790,17 @@ int opcode = TO_BOOL_STR; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_STR); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_STR); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef value; _PyStackRef res; @@ -26845,17 +26845,17 @@ int opcode = UNARY_INVERT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_INVERT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_INVERT); _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; @@ -26883,17 +26883,17 @@ int opcode = UNARY_NEGATIVE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_NEGATIVE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_NEGATIVE); _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; @@ -26921,17 +26921,17 @@ int opcode = UNARY_NOT; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_NOT); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_NOT); _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; @@ -26948,17 +26948,17 @@ int opcode = UNPACK_EX; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNPACK_EX); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNPACK_EX); _PyStackRef seq; _PyStackRef *top; seq = stack_pointer[-1]; @@ -26984,18 +26984,18 @@ int opcode = UNPACK_SEQUENCE; (void)(opcode); #endif - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE); PREDICTED_TRACING_UNPACK_SEQUENCE:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; _PyStackRef seq; _PyStackRef *top; // _SPECIALIZE_UNPACK_SEQUENCE @@ -27042,17 +27042,17 @@ int opcode = UNPACK_SEQUENCE_LIST; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); _PyStackRef tos; _PyStackRef seq; @@ -27108,17 +27108,17 @@ int opcode = UNPACK_SEQUENCE_TUPLE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); _PyStackRef tos; _PyStackRef seq; @@ -27165,17 +27165,17 @@ int opcode = UNPACK_SEQUENCE_TWO_TUPLE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); _PyStackRef tos; _PyStackRef seq; @@ -27223,17 +27223,17 @@ int opcode = WITH_EXCEPT_START; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(WITH_EXCEPT_START); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(WITH_EXCEPT_START); _PyStackRef exit_func; _PyStackRef exit_self; _PyStackRef lasti; @@ -27277,17 +27277,17 @@ int opcode = YIELD_VALUE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(YIELD_VALUE); PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); (void)old_func; int _jump_taken = false; (void)_jump_taken; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(YIELD_VALUE); _PyStackRef retval; _PyStackRef value; retval = stack_pointer[-1]; diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 4738cac33df8db..59055223518cf4 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -235,12 +235,6 @@ def generate_tier1_cases( out.emit(f"int opcode = {name};\n") out.emit(f"(void)(opcode);\n") out.emit(f"#endif\n") - out.emit(f"PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable);\n") - out.emit(f"(void)old_code;\n") - out.emit(f"PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj);\n") - out.emit(f"(void)old_func;\n") - out.emit(f"int _jump_taken = false;\n") - out.emit(f"(void)_jump_taken;\n") needs_this = is_tracing or uses_this(inst) unused_guard = "(void)this_instr;\n" if inst.properties.needs_prev: @@ -259,6 +253,12 @@ def generate_tier1_cases( if needs_this: out.emit(f"_Py_CODEUNIT* const this_instr = next_instr - {inst.size};\n") out.emit(unused_guard) + out.emit(f"PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable);\n") + out.emit(f"(void)old_code;\n") + out.emit(f"PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj);\n") + out.emit(f"(void)old_func;\n") + out.emit(f"int _jump_taken = false;\n") + out.emit(f"(void)_jump_taken;\n") if inst.properties.uses_opcode: out.emit(f"opcode = {name};\n") if inst.family is not None: From ff929373aa4a663ddc207d3026d06087ed5f383f Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 17 Oct 2025 17:21:57 +0100 Subject: [PATCH 040/190] Properly record the predicted ops --- Python/generated_cases.c.h | 225 +++++++++++ Tools/cases_generator/tier1_generator.py | 3 + hello.gvz | 478 +++++++++++++++++++++++ 3 files changed, 706 insertions(+) create mode 100644 hello.gvz diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 476d433e617ea7..95b966cc3aeb80 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -13471,6 +13471,7 @@ PREDICTED_TRACING_BINARY_OP:; _Py_CODEUNIT* const this_instr = next_instr - 6; (void)this_instr; + opcode = BINARY_OP; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -13540,6 +13541,7 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); + opcode = BINARY_OP_ADD_FLOAT; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -13605,6 +13607,7 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_INT); + opcode = BINARY_OP_ADD_INT; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -13672,6 +13675,7 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); + opcode = BINARY_OP_ADD_UNICODE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -13739,6 +13743,7 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_EXTEND); + opcode = BINARY_OP_EXTEND; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -13810,6 +13815,7 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); + opcode = BINARY_OP_INPLACE_ADD_UNICODE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -13900,6 +13906,7 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); + opcode = BINARY_OP_MULTIPLY_FLOAT; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -13965,6 +13972,7 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); + opcode = BINARY_OP_MULTIPLY_INT; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -14032,6 +14040,7 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBSCR_DICT); + opcode = BINARY_OP_SUBSCR_DICT; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -14105,6 +14114,7 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBSCR_GETITEM); + opcode = BINARY_OP_SUBSCR_GETITEM; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -14196,6 +14206,7 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_INT); + opcode = BINARY_OP_SUBSCR_LIST_INT; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -14293,6 +14304,7 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_SLICE); + opcode = BINARY_OP_SUBSCR_LIST_SLICE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -14372,6 +14384,7 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBSCR_STR_INT); + opcode = BINARY_OP_SUBSCR_STR_INT; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -14457,6 +14470,7 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBSCR_TUPLE_INT); + opcode = BINARY_OP_SUBSCR_TUPLE_INT; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -14537,6 +14551,7 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); + opcode = BINARY_OP_SUBTRACT_FLOAT; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -14602,6 +14617,7 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); + opcode = BINARY_OP_SUBTRACT_INT; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -14669,6 +14685,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BINARY_SLICE); + opcode = BINARY_SLICE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -14734,6 +14751,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_INTERPOLATION); + opcode = BUILD_INTERPOLATION; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -14801,6 +14819,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_LIST); + opcode = BUILD_LIST; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -14834,6 +14853,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_MAP); + opcode = BUILD_MAP; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -14895,6 +14915,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_SET); + opcode = BUILD_SET; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -14961,6 +14982,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_SLICE); + opcode = BUILD_SLICE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -15005,6 +15027,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_STRING); + opcode = BUILD_STRING; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -15061,6 +15084,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_TEMPLATE); + opcode = BUILD_TEMPLATE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -15108,6 +15132,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_TUPLE); + opcode = BUILD_TUPLE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -15139,6 +15164,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CACHE); + opcode = CACHE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -15162,6 +15188,7 @@ PREDICTED_TRACING_CALL:; _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; + opcode = CALL; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -15346,6 +15373,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); + opcode = CALL_ALLOC_AND_ENTER_INIT; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -15474,6 +15502,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); + opcode = CALL_BOUND_METHOD_EXACT_ARGS; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -15624,6 +15653,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); + opcode = CALL_BOUND_METHOD_GENERAL; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -15759,6 +15789,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_CLASS); + opcode = CALL_BUILTIN_CLASS; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -15871,6 +15902,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_FAST); + opcode = CALL_BUILTIN_FAST; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -15987,6 +16019,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); + opcode = CALL_BUILTIN_FAST_WITH_KEYWORDS; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -16103,6 +16136,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_O); + opcode = CALL_BUILTIN_O; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -16194,6 +16228,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CALL_FUNCTION_EX); + opcode = CALL_FUNCTION_EX; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -16368,6 +16403,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CALL_INTRINSIC_1); + opcode = CALL_INTRINSIC_1; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -16407,6 +16443,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CALL_INTRINSIC_2); + opcode = CALL_INTRINSIC_2; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -16455,6 +16492,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_ISINSTANCE); + opcode = CALL_ISINSTANCE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -16539,6 +16577,7 @@ PREDICTED_TRACING_CALL_KW:; _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; + opcode = CALL_KW; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -16727,6 +16766,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); + opcode = CALL_KW_BOUND_METHOD; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -16864,6 +16904,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW_NON_PY); + opcode = CALL_KW_NON_PY; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -16998,6 +17039,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW_PY); + opcode = CALL_KW_PY; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -17115,6 +17157,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_LEN); + opcode = CALL_LEN; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -17194,6 +17237,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_LIST_APPEND); + opcode = CALL_LIST_APPEND; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -17289,6 +17333,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); + opcode = CALL_METHOD_DESCRIPTOR_FAST; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -17417,6 +17462,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); + opcode = CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -17547,6 +17593,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); + opcode = CALL_METHOD_DESCRIPTOR_NOARGS; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -17647,6 +17694,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); + opcode = CALL_METHOD_DESCRIPTOR_O; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -17760,6 +17808,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_NON_PY_GENERAL); + opcode = CALL_NON_PY_GENERAL; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -17882,6 +17931,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); + opcode = CALL_PY_EXACT_ARGS; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -18002,6 +18052,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_PY_GENERAL); + opcode = CALL_PY_GENERAL; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -18109,6 +18160,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_STR_1); + opcode = CALL_STR_1; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -18188,6 +18240,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_TUPLE_1); + opcode = CALL_TUPLE_1; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -18267,6 +18320,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_TYPE_1); + opcode = CALL_TYPE_1; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -18329,6 +18383,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CHECK_EG_MATCH); + opcode = CHECK_EG_MATCH; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -18409,6 +18464,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CHECK_EXC_MATCH); + opcode = CHECK_EXC_MATCH; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -18455,6 +18511,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CLEANUP_THROW); + opcode = CLEANUP_THROW; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -18522,6 +18579,7 @@ PREDICTED_TRACING_COMPARE_OP:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; + opcode = COMPARE_OP; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -18601,6 +18659,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_FLOAT); + opcode = COMPARE_OP_FLOAT; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -18663,6 +18722,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_INT); + opcode = COMPARE_OP_INT; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -18729,6 +18789,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_STR); + opcode = COMPARE_OP_STR; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -18796,6 +18857,7 @@ PREDICTED_TRACING_CONTAINS_OP:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; + opcode = CONTAINS_OP; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -18862,6 +18924,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP_DICT); + opcode = CONTAINS_OP_DICT; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -18927,6 +18990,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP_SET); + opcode = CONTAINS_OP_SET; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -18992,6 +19056,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CONVERT_VALUE); + opcode = CONVERT_VALUE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -19033,6 +19098,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(COPY); + opcode = COPY; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -19060,6 +19126,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(COPY_FREE_VARS); + opcode = COPY_FREE_VARS; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -19090,6 +19157,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_ATTR); + opcode = DELETE_ATTR; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -19124,6 +19192,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_DEREF); + opcode = DELETE_DEREF; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -19155,6 +19224,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_FAST); + opcode = DELETE_FAST; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -19190,6 +19260,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_GLOBAL); + opcode = DELETE_GLOBAL; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -19224,6 +19295,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_NAME); + opcode = DELETE_NAME; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -19265,6 +19337,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_SUBSCR); + opcode = DELETE_SUBSCR; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -19306,6 +19379,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DICT_MERGE); + opcode = DICT_MERGE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -19354,6 +19428,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DICT_UPDATE); + opcode = DICT_UPDATE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -19406,6 +19481,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(END_ASYNC_FOR); + opcode = END_ASYNC_FOR; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -19457,6 +19533,7 @@ (void)this_instr; next_instr += 1; INSTRUCTION_STATS(END_FOR); + opcode = END_FOR; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -19484,6 +19561,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(END_SEND); + opcode = END_SEND; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -19516,6 +19594,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(ENTER_EXECUTOR); + opcode = ENTER_EXECUTOR; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -19562,6 +19641,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(EXIT_INIT_CHECK); + opcode = EXIT_INIT_CHECK; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -19594,6 +19674,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(EXTENDED_ARG); + opcode = EXTENDED_ARG; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -19619,6 +19700,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(FORMAT_SIMPLE); + opcode = FORMAT_SIMPLE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -19664,6 +19746,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(FORMAT_WITH_SPEC); + opcode = FORMAT_WITH_SPEC; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -19710,6 +19793,7 @@ PREDICTED_TRACING_FOR_ITER:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; + opcode = FOR_ITER; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -19771,6 +19855,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_GEN); + opcode = FOR_ITER_GEN; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -19849,6 +19934,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_LIST); + opcode = FOR_ITER_LIST; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -19941,6 +20027,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_RANGE); + opcode = FOR_ITER_RANGE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -20016,6 +20103,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_TUPLE); + opcode = FOR_ITER_TUPLE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -20080,6 +20168,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_AITER); + opcode = GET_AITER; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -20150,6 +20239,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_ANEXT); + opcode = GET_ANEXT; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -20183,6 +20273,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_AWAITABLE); + opcode = GET_AWAITABLE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -20221,6 +20312,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_ITER); + opcode = GET_ITER; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -20276,6 +20368,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_LEN); + opcode = GET_LEN; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -20313,6 +20406,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_YIELD_FROM_ITER); + opcode = GET_YIELD_FROM_ITER; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -20367,6 +20461,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IMPORT_FROM); + opcode = IMPORT_FROM; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -20401,6 +20496,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IMPORT_NAME); + opcode = IMPORT_NAME; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -20449,6 +20545,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(INSTRUMENTED_CALL); + opcode = INSTRUMENTED_CALL; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -20644,6 +20741,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); + opcode = INSTRUMENTED_CALL_FUNCTION_EX; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -20818,6 +20916,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); + opcode = INSTRUMENTED_CALL_KW; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -21011,6 +21110,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_END_ASYNC_FOR); + opcode = INSTRUMENTED_END_ASYNC_FOR; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -21070,6 +21170,7 @@ (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_END_FOR); + opcode = INSTRUMENTED_END_FOR; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -21107,6 +21208,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_END_SEND); + opcode = INSTRUMENTED_END_SEND; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -21148,6 +21250,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); + opcode = INSTRUMENTED_FOR_ITER; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -21191,6 +21294,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); + opcode = INSTRUMENTED_INSTRUCTION; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -21225,6 +21329,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); + opcode = INSTRUMENTED_JUMP_BACKWARD; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -21259,6 +21364,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); + opcode = INSTRUMENTED_JUMP_FORWARD; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -21281,6 +21387,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_LINE); + opcode = INSTRUMENTED_LINE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -21327,6 +21434,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); + opcode = INSTRUMENTED_LOAD_SUPER_ATTR; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -21455,6 +21563,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_NOT_TAKEN); + opcode = INSTRUMENTED_NOT_TAKEN; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -21478,6 +21587,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_POP_ITER); + opcode = INSTRUMENTED_POP_ITER; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -21509,6 +21619,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); + opcode = INSTRUMENTED_POP_JUMP_IF_FALSE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -21540,6 +21651,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); + opcode = INSTRUMENTED_POP_JUMP_IF_NONE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -21578,6 +21690,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); + opcode = INSTRUMENTED_POP_JUMP_IF_NOT_NONE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -21614,6 +21727,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); + opcode = INSTRUMENTED_POP_JUMP_IF_TRUE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -21645,6 +21759,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_RESUME); + opcode = INSTRUMENTED_RESUME; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -21732,6 +21847,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); + opcode = INSTRUMENTED_RETURN_VALUE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -21791,6 +21907,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); + opcode = INSTRUMENTED_YIELD_VALUE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -21869,6 +21986,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INTERPRETER_EXIT); + opcode = INTERPRETER_EXIT; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -21913,6 +22031,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IS_OP); + opcode = IS_OP; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -21956,6 +22075,7 @@ PREDICTED_TRACING_JUMP_BACKWARD:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; + opcode = JUMP_BACKWARD; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -22003,6 +22123,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(JUMP_BACKWARD_JIT); + opcode = JUMP_BACKWARD_JIT; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -22071,6 +22192,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); + opcode = JUMP_BACKWARD_NO_INTERRUPT; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -22095,6 +22217,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(JUMP_BACKWARD_NO_JIT); + opcode = JUMP_BACKWARD_NO_JIT; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -22133,6 +22256,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(JUMP_FORWARD); + opcode = JUMP_FORWARD; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -22154,6 +22278,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LIST_APPEND); + opcode = LIST_APPEND; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -22185,6 +22310,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LIST_EXTEND); + opcode = LIST_EXTEND; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -22242,6 +22368,7 @@ PREDICTED_TRACING_LOAD_ATTR:; _Py_CODEUNIT* const this_instr = next_instr - 10; (void)this_instr; + opcode = LOAD_ATTR; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -22329,6 +22456,7 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_CLASS); + opcode = LOAD_ATTR_CLASS; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -22394,6 +22522,7 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); + opcode = LOAD_ATTR_CLASS_WITH_METACLASS_CHECK; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -22469,6 +22598,7 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); + opcode = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -22534,6 +22664,7 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); + opcode = LOAD_ATTR_INSTANCE_VALUE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -22621,6 +22752,7 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); + opcode = LOAD_ATTR_METHOD_LAZY_DICT; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -22684,6 +22816,7 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); + opcode = LOAD_ATTR_METHOD_NO_DICT; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -22737,6 +22870,7 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); + opcode = LOAD_ATTR_METHOD_WITH_VALUES; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -22811,6 +22945,7 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_MODULE); + opcode = LOAD_ATTR_MODULE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -22892,6 +23027,7 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); + opcode = LOAD_ATTR_NONDESCRIPTOR_NO_DICT; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -22946,6 +23082,7 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); + opcode = LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -23021,6 +23158,7 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); + opcode = LOAD_ATTR_PROPERTY; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -23122,6 +23260,7 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_SLOT); + opcode = LOAD_ATTR_SLOT; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -23198,6 +23337,7 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); + opcode = LOAD_ATTR_WITH_HINT; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -23315,6 +23455,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_BUILD_CLASS); + opcode = LOAD_BUILD_CLASS; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -23354,6 +23495,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); + opcode = LOAD_COMMON_CONSTANT; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -23380,6 +23522,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_CONST); + opcode = LOAD_CONST; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -23406,6 +23549,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_DEREF); + opcode = LOAD_DEREF; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -23443,6 +23587,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST); + opcode = LOAD_FAST; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -23469,6 +23614,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); + opcode = LOAD_FAST_AND_CLEAR; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -23495,6 +23641,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_BORROW); + opcode = LOAD_FAST_BORROW; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -23521,6 +23668,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_BORROW_LOAD_FAST_BORROW); + opcode = LOAD_FAST_BORROW_LOAD_FAST_BORROW; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -23551,6 +23699,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_CHECK); + opcode = LOAD_FAST_CHECK; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -23586,6 +23735,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); + opcode = LOAD_FAST_LOAD_FAST; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -23616,6 +23766,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); + opcode = LOAD_FROM_DICT_OR_DEREF; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -23670,6 +23821,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); + opcode = LOAD_FROM_DICT_OR_GLOBALS; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -23755,6 +23907,7 @@ PREDICTED_TRACING_LOAD_GLOBAL:; _Py_CODEUNIT* const this_instr = next_instr - 5; (void)this_instr; + opcode = LOAD_GLOBAL; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -23817,6 +23970,7 @@ frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); + opcode = LOAD_GLOBAL_BUILTIN; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -23904,6 +24058,7 @@ frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); + opcode = LOAD_GLOBAL_MODULE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -23978,6 +24133,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_LOCALS); + opcode = LOAD_LOCALS; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -24011,6 +24167,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_NAME); + opcode = LOAD_NAME; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -24043,6 +24200,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_SMALL_INT); + opcode = LOAD_SMALL_INT; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -24070,6 +24228,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_SPECIAL); + opcode = LOAD_SPECIAL; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -24126,6 +24285,7 @@ PREDICTED_TRACING_LOAD_SUPER_ATTR:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; + opcode = LOAD_SUPER_ATTR; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -24269,6 +24429,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); + opcode = LOAD_SUPER_ATTR_ATTR; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -24338,6 +24499,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); + opcode = LOAD_SUPER_ATTR_METHOD; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -24423,6 +24585,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAKE_CELL); + opcode = MAKE_CELL; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -24453,6 +24616,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAKE_FUNCTION); + opcode = MAKE_FUNCTION; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -24495,6 +24659,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAP_ADD); + opcode = MAP_ADD; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -24535,6 +24700,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_CLASS); + opcode = MATCH_CLASS; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -24596,6 +24762,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_KEYS); + opcode = MATCH_KEYS; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -24632,6 +24799,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_MAPPING); + opcode = MATCH_MAPPING; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -24660,6 +24828,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_SEQUENCE); + opcode = MATCH_SEQUENCE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -24688,6 +24857,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(NOP); + opcode = NOP; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -24708,6 +24878,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(NOT_TAKEN); + opcode = NOT_TAKEN; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -24728,6 +24899,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_EXCEPT); + opcode = POP_EXCEPT; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -24758,6 +24930,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_ITER); + opcode = POP_ITER; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -24788,6 +24961,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_FALSE); + opcode = POP_JUMP_IF_FALSE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -24816,6 +24990,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_NONE); + opcode = POP_JUMP_IF_NONE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -24865,6 +25040,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); + opcode = POP_JUMP_IF_NOT_NONE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -24914,6 +25090,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_TRUE); + opcode = POP_JUMP_IF_TRUE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -24942,6 +25119,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_TOP); + opcode = POP_TOP; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -24969,6 +25147,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(PUSH_EXC_INFO); + opcode = PUSH_EXC_INFO; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -25007,6 +25186,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(PUSH_NULL); + opcode = PUSH_NULL; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -25032,6 +25212,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RAISE_VARARGS); + opcode = RAISE_VARARGS; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -25068,6 +25249,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RERAISE); + opcode = RERAISE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -25103,6 +25285,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESERVED); + opcode = RESERVED; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -25126,6 +25309,7 @@ PREDICTED_TRACING_RESUME:; _Py_CODEUNIT* const this_instr = next_instr - 1; (void)this_instr; + opcode = RESUME; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -25208,6 +25392,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESUME_CHECK); + opcode = RESUME_CHECK; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -25253,6 +25438,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RETURN_GENERATOR); + opcode = RETURN_GENERATOR; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -25304,6 +25490,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RETURN_VALUE); + opcode = RETURN_VALUE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -25348,6 +25535,7 @@ PREDICTED_TRACING_SEND:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; + opcode = SEND; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -25459,6 +25647,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(SEND_GEN); + opcode = SEND_GEN; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -25535,6 +25724,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SETUP_ANNOTATIONS); + opcode = SETUP_ANNOTATIONS; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -25590,6 +25780,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_ADD); + opcode = SET_ADD; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -25623,6 +25814,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); + opcode = SET_FUNCTION_ATTRIBUTE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -25660,6 +25852,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_UPDATE); + opcode = SET_UPDATE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -25697,6 +25890,7 @@ PREDICTED_TRACING_STORE_ATTR:; _Py_CODEUNIT* const this_instr = next_instr - 5; (void)this_instr; + opcode = STORE_ATTR; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -25760,6 +25954,7 @@ frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); + opcode = STORE_ATTR_INSTANCE_VALUE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -25843,6 +26038,7 @@ frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_SLOT); + opcode = STORE_ATTR_SLOT; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -25901,6 +26097,7 @@ frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); + opcode = STORE_ATTR_WITH_HINT; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -25996,6 +26193,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_DEREF); + opcode = STORE_DEREF; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -26024,6 +26222,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_FAST); + opcode = STORE_FAST; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -26053,6 +26252,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); + opcode = STORE_FAST_LOAD_FAST; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -26085,6 +26285,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_FAST_STORE_FAST); + opcode = STORE_FAST_STORE_FAST; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -26125,6 +26326,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_GLOBAL); + opcode = STORE_GLOBAL; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -26159,6 +26361,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_NAME); + opcode = STORE_NAME; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -26214,6 +26417,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_SLICE); + opcode = STORE_SLICE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -26284,6 +26488,7 @@ PREDICTED_TRACING_STORE_SUBSCR:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; + opcode = STORE_SUBSCR; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -26349,6 +26554,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR_DICT); + opcode = STORE_SUBSCR_DICT; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -26407,6 +26613,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); + opcode = STORE_SUBSCR_LIST_INT; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -26495,6 +26702,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SWAP); + opcode = SWAP; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -26525,6 +26733,7 @@ PREDICTED_TRACING_TO_BOOL:; _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; + opcode = TO_BOOL; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -26583,6 +26792,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); + opcode = TO_BOOL_ALWAYS_TRUE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -26633,6 +26843,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_BOOL); + opcode = TO_BOOL_BOOL; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -26664,6 +26875,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_INT); + opcode = TO_BOOL_INT; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -26711,6 +26923,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_LIST); + opcode = TO_BOOL_LIST; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -26761,6 +26974,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_NONE); + opcode = TO_BOOL_NONE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -26795,6 +27009,7 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_STR); + opcode = TO_BOOL_STR; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -26850,6 +27065,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNARY_INVERT); + opcode = UNARY_INVERT; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -26888,6 +27104,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNARY_NEGATIVE); + opcode = UNARY_NEGATIVE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -26926,6 +27143,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNARY_NOT); + opcode = UNARY_NOT; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -26953,6 +27171,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNPACK_EX); + opcode = UNPACK_EX; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -26990,6 +27209,7 @@ PREDICTED_TRACING_UNPACK_SEQUENCE:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; + opcode = UNPACK_SEQUENCE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -27047,6 +27267,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); + opcode = UNPACK_SEQUENCE_LIST; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -27113,6 +27334,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); + opcode = UNPACK_SEQUENCE_TUPLE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -27170,6 +27392,7 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); + opcode = UNPACK_SEQUENCE_TWO_TUPLE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -27228,6 +27451,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(WITH_EXCEPT_START); + opcode = WITH_EXCEPT_START; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -27282,6 +27506,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(YIELD_VALUE); + opcode = YIELD_VALUE; PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); (void)old_code; PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 59055223518cf4..05243195807ccf 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -253,6 +253,9 @@ def generate_tier1_cases( if needs_this: out.emit(f"_Py_CODEUNIT* const this_instr = next_instr - {inst.size};\n") out.emit(unused_guard) + if is_tracing: + # This is required so that the predicted ops reflect the correct opcode. + out.emit(f"opcode = {name};\n") out.emit(f"PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable);\n") out.emit(f"(void)old_code;\n") out.emit(f"PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj);\n") diff --git a/hello.gvz b/hello.gvz new file mode 100644 index 00000000000000..4260bf1631ac7c --- /dev/null +++ b/hello.gvz @@ -0,0 +1,478 @@ +digraph ideal { + + rankdir = "LR" + +executor_0x65462fd7c7c0 [ + shape = none + label = < + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Executor
No code object
_START_EXECUTOR
_MAKE_WARM
_SET_IP
_CHECK_PEP_523
_CHECK_FUNCTION_VERSION
_CHECK_FUNCTION_EXACT_ARGS
_CHECK_STACK_SPACE
_CHECK_RECURSION_REMAINING
_INIT_CALL_PY_EXACT_ARGS_1
_SAVE_RETURN_OFFSET
_PUSH_FRAME
_GUARD_IP
_TIER2_RESUME_CHECK
_LOAD_FAST_BORROW_0
_SET_IP
_GUARD_TOS_TUPLE
_UNPACK_SEQUENCE_TWO_TUPLE
_CHECK_VALIDITY
_SET_IP
_STORE_FAST_1
_STORE_FAST_2
_CHECK_VALIDITY
_LOAD_SMALL_INT_0
_SET_IP
_STORE_FAST_3
_CHECK_VALIDITY
_GUARD_GLOBALS_VERSION
_LOAD_GLOBAL_BUILTINS
_PUSH_NULL_CONDITIONAL
_LOAD_FAST_BORROW_2
_SET_IP
_CALL_BUILTIN_CLASS
_TIER2_RESUME_CHECK
_CHECK_VALIDITY
_SET_IP
_GET_ITER
_CHECK_VALIDITY
_SET_IP
_FOR_ITER_TIER_TWO
_GUARD_IP
_CHECK_VALIDITY
_SET_IP
_GUARD_TOS_TUPLE
_UNPACK_SEQUENCE_TWO_TUPLE
_CHECK_VALIDITY
_SET_IP
_STORE_FAST_4
_STORE_FAST_5
_CHECK_VALIDITY
_LOAD_FAST_BORROW_3
_LOAD_GLOBAL_MODULE
_PUSH_NULL_CONDITIONAL
_LOAD_FAST_BORROW_1
_LOAD_FAST_BORROW_4
_SET_IP
_CHECK_PEP_523
_CHECK_FUNCTION_VERSION
_CHECK_FUNCTION_EXACT_ARGS
_CHECK_STACK_SPACE
_CHECK_RECURSION_REMAINING
_INIT_CALL_PY_EXACT_ARGS_2
_SAVE_RETURN_OFFSET
_PUSH_FRAME
_GUARD_IP
_TIER2_RESUME_CHECK
_LOAD_CONST
_LOAD_FAST_BORROW_0
_LOAD_FAST_BORROW_1
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_LOAD_FAST_BORROW_0
_LOAD_FAST_BORROW_1
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_LOAD_SMALL_INT_1
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_MULTIPLY_INT
_LOAD_SMALL_INT_2
_SET_IP
_BINARY_OP
_CHECK_VALIDITY
_LOAD_FAST_BORROW_0
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_LOAD_SMALL_INT_1
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_SET_IP
_GUARD_BINARY_OP_EXTEND
_BINARY_OP_EXTEND
_CHECK_VALIDITY
_SET_IP
_RETURN_VALUE
_GUARD_IP
_CHECK_VALIDITY
_LOAD_FAST_BORROW_5
_GUARD_TOS_FLOAT
_GUARD_NOS_FLOAT
_BINARY_OP_MULTIPLY_FLOAT
_SET_IP
_BINARY_OP
_CHECK_VALIDITY
_SET_IP
_STORE_FAST_3
_EXIT_TRACE
> +] + +executor_0x65462fd7c7c0:i112 -> executor_0x65462fd78530:start +executor_0x65462fd738f0 [ + shape = none + label = < + + + + + + + + + + + + + + +
Executor
part_At_times_u: 44
_START_EXECUTOR
_MAKE_WARM
_SET_IP
_POP_ITER
_CHECK_VALIDITY
_LOAD_FAST_BORROW_3
_SET_IP
_RETURN_VALUE
_GUARD_IP
_CHECK_VALIDITY
_LIST_APPEND
_EXIT_TRACE
> +] + +executor_0x65462fd738f0:i11 -> executor_0x65462fd7b4e0:start +executor_0x65462fcca340 [ + shape = none + label = < + + + + + + + + + + + + + + +
Executor
part_A_times_u: 36
_START_EXECUTOR
_MAKE_WARM
_SET_IP
_POP_ITER
_CHECK_VALIDITY
_LOAD_FAST_BORROW_3
_SET_IP
_RETURN_VALUE
_GUARD_IP
_CHECK_VALIDITY
_LIST_APPEND
_EXIT_TRACE
> +] + +executor_0x65462fcca340:i11 -> executor_0x65462fd7b4e0:start +executor_0x65462fd7b4e0 [ + shape = none + label = < + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Executor
eval_times_u: 26
_START_EXECUTOR
_MAKE_WARM
_ITER_CHECK_RANGE
_GUARD_NOT_EXHAUSTED_RANGE
_ITER_NEXT_RANGE
_SET_IP
_STORE_FAST_2
_LOAD_FAST_0
_CHECK_VALIDITY
_PUSH_NULL
_LOAD_FAST_BORROW_2
_LOAD_FAST_BORROW_1
_BUILD_TUPLE
_SET_IP
_CHECK_PEP_523
_CHECK_FUNCTION_VERSION
_CHECK_FUNCTION_EXACT_ARGS
_CHECK_STACK_SPACE
_CHECK_RECURSION_REMAINING
_INIT_CALL_PY_EXACT_ARGS_1
_SAVE_RETURN_OFFSET
_PUSH_FRAME
_GUARD_IP
_TIER2_RESUME_CHECK
_LOAD_FAST_BORROW_0
_SET_IP
_GUARD_TOS_TUPLE
_UNPACK_SEQUENCE_TWO_TUPLE
_CHECK_VALIDITY
_SET_IP
_STORE_FAST_1
_STORE_FAST_2
_CHECK_VALIDITY
_LOAD_SMALL_INT_0
_SET_IP
_STORE_FAST_3
_CHECK_VALIDITY
_GUARD_GLOBALS_VERSION
_LOAD_GLOBAL_BUILTINS
_PUSH_NULL_CONDITIONAL
_LOAD_FAST_BORROW_2
_SET_IP
_CALL_BUILTIN_CLASS
_TIER2_RESUME_CHECK
_CHECK_VALIDITY
_SET_IP
_GET_ITER
_CHECK_VALIDITY
_SET_IP
_FOR_ITER_TIER_TWO
_GUARD_IP
_CHECK_VALIDITY
_SET_IP
_GUARD_TOS_TUPLE
_UNPACK_SEQUENCE_TWO_TUPLE
_CHECK_VALIDITY
_SET_IP
_STORE_FAST_4
_STORE_FAST_5
_CHECK_VALIDITY
_LOAD_FAST_BORROW_3
_LOAD_GLOBAL_MODULE
_PUSH_NULL_CONDITIONAL
_LOAD_FAST_BORROW_4
_LOAD_FAST_BORROW_1
_SET_IP
_CHECK_PEP_523
_CHECK_FUNCTION_VERSION
_CHECK_FUNCTION_EXACT_ARGS
_CHECK_STACK_SPACE
_CHECK_RECURSION_REMAINING
_INIT_CALL_PY_EXACT_ARGS_2
_SAVE_RETURN_OFFSET
_PUSH_FRAME
_GUARD_IP
_TIER2_RESUME_CHECK
_LOAD_CONST
_LOAD_FAST_BORROW_0
_LOAD_FAST_BORROW_1
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_LOAD_FAST_BORROW_0
_LOAD_FAST_BORROW_1
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_LOAD_SMALL_INT_1
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_MULTIPLY_INT
_LOAD_SMALL_INT_2
_SET_IP
_BINARY_OP
_CHECK_VALIDITY
_LOAD_FAST_BORROW_0
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_LOAD_SMALL_INT_1
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_SET_IP
_GUARD_BINARY_OP_EXTEND
_BINARY_OP_EXTEND
_CHECK_VALIDITY
_SET_IP
_RETURN_VALUE
_GUARD_IP
_CHECK_VALIDITY
_LOAD_FAST_BORROW_5
_GUARD_TOS_FLOAT
_GUARD_NOS_FLOAT
_BINARY_OP_MULTIPLY_FLOAT
_SET_IP
_BINARY_OP
_CHECK_VALIDITY
_SET_IP
_STORE_FAST_3
_EXIT_TRACE
> +] + +executor_0x65462fd7b4e0:i15 -> executor_0x65462fd7c7c0:start +executor_0x65462fd7b4e0:i16 -> executor_0x65462fd7c7c0:start +executor_0x65462fd7b4e0:i123 -> executor_0x65462fd79600:start +executor_0x65462fd79600 [ + shape = none + label = < + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Executor
part_At_times_u: 45
_START_EXECUTOR
_MAKE_WARM
_SET_IP
_FOR_ITER_TIER_TWO
_GUARD_IP
_CHECK_VALIDITY
_SET_IP
_GUARD_TOS_TUPLE
_UNPACK_SEQUENCE_TWO_TUPLE
_CHECK_VALIDITY
_SET_IP
_STORE_FAST_4
_STORE_FAST_5
_CHECK_VALIDITY
_LOAD_FAST_BORROW_3
_LOAD_GLOBAL_MODULE
_PUSH_NULL_CONDITIONAL
_LOAD_FAST_BORROW_4
_LOAD_FAST_BORROW_1
_SET_IP
_CHECK_PEP_523
_CHECK_FUNCTION_VERSION
_CHECK_FUNCTION_EXACT_ARGS
_CHECK_STACK_SPACE
_CHECK_RECURSION_REMAINING
_INIT_CALL_PY_EXACT_ARGS_2
_SAVE_RETURN_OFFSET
_PUSH_FRAME
_GUARD_IP
_TIER2_RESUME_CHECK
_LOAD_CONST
_LOAD_FAST_BORROW_0
_LOAD_FAST_BORROW_1
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_LOAD_FAST_BORROW_0
_LOAD_FAST_BORROW_1
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_LOAD_SMALL_INT_1
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_MULTIPLY_INT
_LOAD_SMALL_INT_2
_SET_IP
_BINARY_OP
_CHECK_VALIDITY
_LOAD_FAST_BORROW_0
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_LOAD_SMALL_INT_1
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_SET_IP
_GUARD_BINARY_OP_EXTEND
_BINARY_OP_EXTEND
_CHECK_VALIDITY
_SET_IP
_RETURN_VALUE
_GUARD_IP
_CHECK_VALIDITY
_LOAD_FAST_BORROW_5
_GUARD_TOS_FLOAT
_GUARD_NOS_FLOAT
_BINARY_OP_MULTIPLY_FLOAT
_GUARD_TOS_FLOAT
_GUARD_NOS_FLOAT
_BINARY_OP_ADD_FLOAT
_SET_IP
_STORE_FAST_3
_JUMP_TO_TOP
> +] + +executor_0x65462fd78530 [ + shape = none + label = < + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Executor
part_A_times_u: 37
_START_EXECUTOR
_MAKE_WARM
_SET_IP
_FOR_ITER_TIER_TWO
_GUARD_IP
_CHECK_VALIDITY
_SET_IP
_GUARD_TOS_TUPLE
_UNPACK_SEQUENCE_TWO_TUPLE
_CHECK_VALIDITY
_SET_IP
_STORE_FAST_4
_STORE_FAST_5
_CHECK_VALIDITY
_LOAD_FAST_BORROW_3
_LOAD_GLOBAL_MODULE
_PUSH_NULL_CONDITIONAL
_LOAD_FAST_BORROW_1
_LOAD_FAST_BORROW_4
_SET_IP
_CHECK_PEP_523
_CHECK_FUNCTION_VERSION
_CHECK_FUNCTION_EXACT_ARGS
_CHECK_STACK_SPACE
_CHECK_RECURSION_REMAINING
_INIT_CALL_PY_EXACT_ARGS_2
_SAVE_RETURN_OFFSET
_PUSH_FRAME
_GUARD_IP
_TIER2_RESUME_CHECK
_LOAD_CONST
_LOAD_FAST_BORROW_0
_LOAD_FAST_BORROW_1
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_LOAD_FAST_BORROW_0
_LOAD_FAST_BORROW_1
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_LOAD_SMALL_INT_1
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_MULTIPLY_INT
_LOAD_SMALL_INT_2
_SET_IP
_BINARY_OP
_CHECK_VALIDITY
_LOAD_FAST_BORROW_0
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_LOAD_SMALL_INT_1
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_SET_IP
_GUARD_BINARY_OP_EXTEND
_BINARY_OP_EXTEND
_CHECK_VALIDITY
_SET_IP
_RETURN_VALUE
_GUARD_IP
_CHECK_VALIDITY
_LOAD_FAST_BORROW_5
_SET_IP
_GUARD_BINARY_OP_EXTEND
_BINARY_OP_EXTEND
_CHECK_VALIDITY
_GUARD_TOS_FLOAT
_GUARD_NOS_FLOAT
_BINARY_OP_ADD_FLOAT
_SET_IP
_STORE_FAST_3
_JUMP_TO_TOP
> +] + +} + From 2589eb019052dd7caa471b411a8b996ead8fb324 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 17 Oct 2025 18:09:25 +0100 Subject: [PATCH 041/190] Re-enable the optimizer --- Include/internal/pycore_optimizer.h | 3 +- Lib/test/test_capi/test_opt.py | 72 +---- Python/optimizer.c | 11 +- Python/optimizer_analysis.c | 16 +- Python/optimizer_bytecodes.c | 8 +- Python/optimizer_cases.c.h | 8 +- Python/optimizer_symbols.c | 13 +- hello.gvz | 472 ---------------------------- 8 files changed, 50 insertions(+), 553 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index c7c304943bf87e..748bf854768c0b 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -88,7 +88,8 @@ PyAPI_FUNC(void) _Py_Executors_InvalidateCold(PyInterpreterState *interp); #define TRACE_STACK_SIZE 5 -int _Py_uop_analyze_and_optimize(_PyInterpreterFrame *frame, +int _Py_uop_analyze_and_optimize( + PyFunctionObject *initial_func, _PyUOpInstruction *trace, int trace_len, int curr_stackentries, _PyBloomFilter *dependencies); diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index c3fed50cee9736..3cc8fbe0de0a6c 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -17,6 +17,8 @@ from _testinternalcapi import TIER2_THRESHOLD +# We need one more iteration as one iteration is spent on tracing. +TIER2_THRESHOLD = TIER2_THRESHOLD + 1 #For test of issue 136154 GLOBAL_136154 = 42 @@ -139,6 +141,7 @@ def testfunc(x): self.assertIn("_JUMP_TO_TOP", uops) self.assertIn("_LOAD_FAST_BORROW_0", uops) + @unittest.skip("gh-139109 WIP") def test_extended_arg(self): "Check EXTENDED_ARG handling in superblock creation" ns = {} @@ -422,32 +425,6 @@ def testfunc(n, m): uops = get_opnames(ex) self.assertIn("_FOR_ITER_TIER_TWO", uops) - def test_confidence_score(self): - def testfunc(n): - bits = 0 - for i in range(n): - if i & 0x01: - bits += 1 - if i & 0x02: - bits += 1 - if i&0x04: - bits += 1 - if i&0x08: - bits += 1 - if i&0x10: - bits += 1 - return bits - - x = testfunc(TIER2_THRESHOLD * 2) - - self.assertEqual(x, TIER2_THRESHOLD * 5) - ex = get_first_executor(testfunc) - self.assertIsNotNone(ex) - ops = list(iter_opnames(ex)) - #Since branch is 50/50 the trace could go either way. - count = ops.count("_GUARD_IS_TRUE_POP") + ops.count("_GUARD_IS_FALSE_POP") - self.assertLessEqual(count, 2) - @requires_specialization @unittest.skipIf(Py_GIL_DISABLED, "optimizer not yet supported in free-threaded builds") @@ -651,7 +628,7 @@ def testfunc(n): x = range(i) return x - testfunc(_testinternalcapi.TIER2_THRESHOLD) + testfunc(_testinternalcapi.TIER2_THRESHOLD + 1) ex = get_first_executor(testfunc) assert ex is not None @@ -847,38 +824,7 @@ def testfunc(n): self.assertLessEqual(len(guard_nos_unicode_count), 1) self.assertIn("_COMPARE_OP_STR", uops) - def test_type_inconsistency(self): - ns = {} - src = textwrap.dedent(""" - def testfunc(n): - for i in range(n): - x = _test_global + _test_global - """) - exec(src, ns, ns) - testfunc = ns['testfunc'] - ns['_test_global'] = 0 - _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD - 1) - self.assertIsNone(ex) - ns['_test_global'] = 1 - _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD - 1) - self.assertIsNotNone(ex) - uops = get_opnames(ex) - self.assertNotIn("_GUARD_TOS_INT", uops) - self.assertNotIn("_GUARD_NOS_INT", uops) - self.assertNotIn("_BINARY_OP_ADD_INT", uops) - self.assertNotIn("_POP_TWO_LOAD_CONST_INLINE_BORROW", uops) - # Try again, but between the runs, set the global to a float. - # This should result in no executor the second time. - ns = {} - exec(src, ns, ns) - testfunc = ns['testfunc'] - ns['_test_global'] = 0 - _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD - 1) - self.assertIsNone(ex) - ns['_test_global'] = 3.14 - _, ex = self._run_with_optimizer(testfunc, TIER2_THRESHOLD - 1) - self.assertIsNone(ex) - + @unittest.skip("gh-139109 WIP") def test_combine_stack_space_checks_sequential(self): def dummy12(x): return x - 1 @@ -907,6 +853,7 @@ def testfunc(n): largest_stack = _testinternalcapi.get_co_framesize(dummy13.__code__) self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + @unittest.skip("gh-139109 WIP") def test_combine_stack_space_checks_nested(self): def dummy12(x): return x + 3 @@ -937,6 +884,7 @@ def testfunc(n): ) self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + @unittest.skip("gh-139109 WIP") def test_combine_stack_space_checks_several_calls(self): def dummy12(x): return x + 3 @@ -972,6 +920,7 @@ def testfunc(n): ) self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + @unittest.skip("gh-139109 WIP") def test_combine_stack_space_checks_several_calls_different_order(self): # same as `several_calls` but with top-level calls reversed def dummy12(x): @@ -1008,6 +957,7 @@ def testfunc(n): ) self.assertIn(("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands) + @unittest.skip("gh-139109 WIP") def test_combine_stack_space_complex(self): def dummy0(x): return x @@ -1057,6 +1007,7 @@ def testfunc(n): ("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands ) + @unittest.skip("gh-139109 WIP") def test_combine_stack_space_checks_large_framesize(self): # Create a function with a large framesize. This ensures _CHECK_STACK_SPACE is # actually doing its job. Note that the resulting trace hits @@ -1118,6 +1069,7 @@ def testfunc(n): ("_CHECK_STACK_SPACE_OPERAND", largest_stack), uops_and_operands ) + @unittest.skip("gh-139109 WIP") def test_combine_stack_space_checks_recursion(self): def dummy15(x): while x > 0: @@ -2511,7 +2463,7 @@ def testfunc(n): del email.jit_testing - testfunc(_testinternalcapi.TIER2_THRESHOLD) + testfunc(_testinternalcapi.TIER2_THRESHOLD + 1) ex = get_first_executor(testfunc) assert ex is not None """)) diff --git a/Python/optimizer.c b/Python/optimizer.c index 67368fefa3452e..98ac0457477694 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -854,7 +854,7 @@ _PyJIT_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_ tstate->interp->jit_tracer_initial_code = (PyCodeObject *)Py_NewRef(code); tstate->interp->jit_tracer_initial_func = (PyFunctionObject *)Py_NewRef(_PyFrame_GetFunction(frame)); tstate->interp->jit_tracer_previous_exit = exit; - memset(&tstate->interp->jit_tracer_dependencies.bits, 0, sizeof(tstate->interp->jit_tracer_dependencies.bits)); + _Py_BloomFilter_Init(&tstate->interp->jit_tracer_dependencies); tstate->interp->jit_tracer_initial_stack_depth = curr_stackdepth; tstate->interp->jit_tracer_initial_chain_depth = chain_depth; tstate->interp->jit_tracer_current_frame = frame; @@ -1177,8 +1177,7 @@ uop_optimize( _PyExecutorObject **exec_ptr, bool progress_needed) { - _PyBloomFilter dependencies; - _Py_BloomFilter_Init(&dependencies); + _PyBloomFilter *dependencies = &tstate->interp->jit_tracer_dependencies; PyInterpreterState *interp = _PyInterpreterState_GET(); if (interp->jit_uop_buffer == NULL) { interp->jit_uop_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); @@ -1203,9 +1202,9 @@ uop_optimize( assert(length < UOP_MAX_TRACE_LENGTH); OPT_STAT_INC(traces_created); if (!is_noopt) { - length = _Py_uop_analyze_and_optimize(frame, buffer, + length = _Py_uop_analyze_and_optimize(tstate->interp->jit_tracer_initial_func, buffer, length, - curr_stackentries, &dependencies); + curr_stackentries, dependencies); if (length <= 0) { return length; } @@ -1228,7 +1227,7 @@ uop_optimize( OPT_HIST(effective_trace_length(buffer, length), optimized_trace_length_hist); length = prepare_for_execution(buffer, length); assert(length <= UOP_MAX_TRACE_LENGTH); - _PyExecutorObject *executor = make_executor_from_uops(buffer, length, &dependencies, tstate->interp->jit_tracer_initial_chain_depth); + _PyExecutorObject *executor = make_executor_from_uops(buffer, length, dependencies, tstate->interp->jit_tracer_initial_chain_depth); if (executor == NULL) { return -1; } diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index b90575abf51cdd..814575e50978dd 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -519,7 +519,7 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) // > 0 - length of optimized trace int _Py_uop_analyze_and_optimize( - _PyInterpreterFrame *frame, + PyFunctionObject *initial_func, _PyUOpInstruction *buffer, int length, int curr_stacklen, @@ -528,13 +528,13 @@ _Py_uop_analyze_and_optimize( { OPT_STAT_INC(optimizer_attempts); - // int err = optimize_uops( - // _PyFrame_GetFunction(frame), buffer, - // length, curr_stacklen, dependencies); - // - // if (err == 0) { - // return err; - // } + int err = optimize_uops( + initial_func, buffer, + length, curr_stacklen, dependencies); + + if (err == 0) { + return err; + } assert(length > 0); diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index da3d3c96bc1d97..f9f8c15ab964d6 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -775,7 +775,9 @@ dummy_func(void) { SAVE_STACK(); PyCodeObject *co = get_current_code_object(ctx); ctx->frame->stack_pointer = stack_pointer; - frame_pop(ctx); + if (frame_pop(ctx)) { + break; + } stack_pointer = ctx->frame->stack_pointer; /* Stack space handling */ @@ -794,7 +796,9 @@ dummy_func(void) { SYNC_SP(); PyCodeObject *co = get_current_code_object(ctx); ctx->frame->stack_pointer = stack_pointer; - frame_pop(ctx); + if (frame_pop(ctx)) { + break; + } stack_pointer = ctx->frame->stack_pointer; res = sym_new_unknown(ctx); diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index ef76374d244f38..3d31c7f04dad89 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1118,7 +1118,9 @@ assert(WITHIN_STACK_BOUNDS()); PyCodeObject *co = get_current_code_object(ctx); ctx->frame->stack_pointer = stack_pointer; - frame_pop(ctx); + if (frame_pop(ctx)) { + break; + } stack_pointer = ctx->frame->stack_pointer; assert(corresponding_check_stack == NULL); assert(co != NULL); @@ -3011,7 +3013,9 @@ JitOptRef res; PyCodeObject *co = get_current_code_object(ctx); ctx->frame->stack_pointer = stack_pointer; - frame_pop(ctx); + if (frame_pop(ctx)) { + break; + } stack_pointer = ctx->frame->stack_pointer; res = sym_new_unknown(ctx); assert(corresponding_check_stack == NULL); diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 0e6884b99232e9..e0563122e0186e 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -818,7 +818,11 @@ _Py_uop_frame_new( JitOptRef *args, int arg_len) { - assert(ctx->curr_frame_depth < MAX_ABSTRACT_FRAME_DEPTH); + if (ctx->curr_frame_depth >= MAX_ABSTRACT_FRAME_DEPTH) { + ctx->done = true; + ctx->out_of_space = true; + return NULL; + } _Py_UOpsAbstractFrame *frame = &ctx->frames[ctx->curr_frame_depth]; frame->stack_len = co->co_stacksize; @@ -907,7 +911,12 @@ _Py_uop_frame_pop(JitOptContext *ctx) _Py_UOpsAbstractFrame *frame = ctx->frame; ctx->n_consumed = frame->locals; ctx->curr_frame_depth--; - assert(ctx->curr_frame_depth >= 1); + // TODO gh-139109: Handle trace recording underflow + if (ctx->curr_frame_depth == 0) { + ctx->done = true; + ctx->out_of_space = true; + return 1; + } ctx->frame = &ctx->frames[ctx->curr_frame_depth - 1]; return 0; diff --git a/hello.gvz b/hello.gvz index 4260bf1631ac7c..9448c085123d25 100644 --- a/hello.gvz +++ b/hello.gvz @@ -2,477 +2,5 @@ digraph ideal { rankdir = "LR" -executor_0x65462fd7c7c0 [ - shape = none - label = < - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Executor
No code object
_START_EXECUTOR
_MAKE_WARM
_SET_IP
_CHECK_PEP_523
_CHECK_FUNCTION_VERSION
_CHECK_FUNCTION_EXACT_ARGS
_CHECK_STACK_SPACE
_CHECK_RECURSION_REMAINING
_INIT_CALL_PY_EXACT_ARGS_1
_SAVE_RETURN_OFFSET
_PUSH_FRAME
_GUARD_IP
_TIER2_RESUME_CHECK
_LOAD_FAST_BORROW_0
_SET_IP
_GUARD_TOS_TUPLE
_UNPACK_SEQUENCE_TWO_TUPLE
_CHECK_VALIDITY
_SET_IP
_STORE_FAST_1
_STORE_FAST_2
_CHECK_VALIDITY
_LOAD_SMALL_INT_0
_SET_IP
_STORE_FAST_3
_CHECK_VALIDITY
_GUARD_GLOBALS_VERSION
_LOAD_GLOBAL_BUILTINS
_PUSH_NULL_CONDITIONAL
_LOAD_FAST_BORROW_2
_SET_IP
_CALL_BUILTIN_CLASS
_TIER2_RESUME_CHECK
_CHECK_VALIDITY
_SET_IP
_GET_ITER
_CHECK_VALIDITY
_SET_IP
_FOR_ITER_TIER_TWO
_GUARD_IP
_CHECK_VALIDITY
_SET_IP
_GUARD_TOS_TUPLE
_UNPACK_SEQUENCE_TWO_TUPLE
_CHECK_VALIDITY
_SET_IP
_STORE_FAST_4
_STORE_FAST_5
_CHECK_VALIDITY
_LOAD_FAST_BORROW_3
_LOAD_GLOBAL_MODULE
_PUSH_NULL_CONDITIONAL
_LOAD_FAST_BORROW_1
_LOAD_FAST_BORROW_4
_SET_IP
_CHECK_PEP_523
_CHECK_FUNCTION_VERSION
_CHECK_FUNCTION_EXACT_ARGS
_CHECK_STACK_SPACE
_CHECK_RECURSION_REMAINING
_INIT_CALL_PY_EXACT_ARGS_2
_SAVE_RETURN_OFFSET
_PUSH_FRAME
_GUARD_IP
_TIER2_RESUME_CHECK
_LOAD_CONST
_LOAD_FAST_BORROW_0
_LOAD_FAST_BORROW_1
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_LOAD_FAST_BORROW_0
_LOAD_FAST_BORROW_1
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_LOAD_SMALL_INT_1
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_MULTIPLY_INT
_LOAD_SMALL_INT_2
_SET_IP
_BINARY_OP
_CHECK_VALIDITY
_LOAD_FAST_BORROW_0
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_LOAD_SMALL_INT_1
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_SET_IP
_GUARD_BINARY_OP_EXTEND
_BINARY_OP_EXTEND
_CHECK_VALIDITY
_SET_IP
_RETURN_VALUE
_GUARD_IP
_CHECK_VALIDITY
_LOAD_FAST_BORROW_5
_GUARD_TOS_FLOAT
_GUARD_NOS_FLOAT
_BINARY_OP_MULTIPLY_FLOAT
_SET_IP
_BINARY_OP
_CHECK_VALIDITY
_SET_IP
_STORE_FAST_3
_EXIT_TRACE
> -] - -executor_0x65462fd7c7c0:i112 -> executor_0x65462fd78530:start -executor_0x65462fd738f0 [ - shape = none - label = < - - - - - - - - - - - - - - -
Executor
part_At_times_u: 44
_START_EXECUTOR
_MAKE_WARM
_SET_IP
_POP_ITER
_CHECK_VALIDITY
_LOAD_FAST_BORROW_3
_SET_IP
_RETURN_VALUE
_GUARD_IP
_CHECK_VALIDITY
_LIST_APPEND
_EXIT_TRACE
> -] - -executor_0x65462fd738f0:i11 -> executor_0x65462fd7b4e0:start -executor_0x65462fcca340 [ - shape = none - label = < - - - - - - - - - - - - - - -
Executor
part_A_times_u: 36
_START_EXECUTOR
_MAKE_WARM
_SET_IP
_POP_ITER
_CHECK_VALIDITY
_LOAD_FAST_BORROW_3
_SET_IP
_RETURN_VALUE
_GUARD_IP
_CHECK_VALIDITY
_LIST_APPEND
_EXIT_TRACE
> -] - -executor_0x65462fcca340:i11 -> executor_0x65462fd7b4e0:start -executor_0x65462fd7b4e0 [ - shape = none - label = < - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Executor
eval_times_u: 26
_START_EXECUTOR
_MAKE_WARM
_ITER_CHECK_RANGE
_GUARD_NOT_EXHAUSTED_RANGE
_ITER_NEXT_RANGE
_SET_IP
_STORE_FAST_2
_LOAD_FAST_0
_CHECK_VALIDITY
_PUSH_NULL
_LOAD_FAST_BORROW_2
_LOAD_FAST_BORROW_1
_BUILD_TUPLE
_SET_IP
_CHECK_PEP_523
_CHECK_FUNCTION_VERSION
_CHECK_FUNCTION_EXACT_ARGS
_CHECK_STACK_SPACE
_CHECK_RECURSION_REMAINING
_INIT_CALL_PY_EXACT_ARGS_1
_SAVE_RETURN_OFFSET
_PUSH_FRAME
_GUARD_IP
_TIER2_RESUME_CHECK
_LOAD_FAST_BORROW_0
_SET_IP
_GUARD_TOS_TUPLE
_UNPACK_SEQUENCE_TWO_TUPLE
_CHECK_VALIDITY
_SET_IP
_STORE_FAST_1
_STORE_FAST_2
_CHECK_VALIDITY
_LOAD_SMALL_INT_0
_SET_IP
_STORE_FAST_3
_CHECK_VALIDITY
_GUARD_GLOBALS_VERSION
_LOAD_GLOBAL_BUILTINS
_PUSH_NULL_CONDITIONAL
_LOAD_FAST_BORROW_2
_SET_IP
_CALL_BUILTIN_CLASS
_TIER2_RESUME_CHECK
_CHECK_VALIDITY
_SET_IP
_GET_ITER
_CHECK_VALIDITY
_SET_IP
_FOR_ITER_TIER_TWO
_GUARD_IP
_CHECK_VALIDITY
_SET_IP
_GUARD_TOS_TUPLE
_UNPACK_SEQUENCE_TWO_TUPLE
_CHECK_VALIDITY
_SET_IP
_STORE_FAST_4
_STORE_FAST_5
_CHECK_VALIDITY
_LOAD_FAST_BORROW_3
_LOAD_GLOBAL_MODULE
_PUSH_NULL_CONDITIONAL
_LOAD_FAST_BORROW_4
_LOAD_FAST_BORROW_1
_SET_IP
_CHECK_PEP_523
_CHECK_FUNCTION_VERSION
_CHECK_FUNCTION_EXACT_ARGS
_CHECK_STACK_SPACE
_CHECK_RECURSION_REMAINING
_INIT_CALL_PY_EXACT_ARGS_2
_SAVE_RETURN_OFFSET
_PUSH_FRAME
_GUARD_IP
_TIER2_RESUME_CHECK
_LOAD_CONST
_LOAD_FAST_BORROW_0
_LOAD_FAST_BORROW_1
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_LOAD_FAST_BORROW_0
_LOAD_FAST_BORROW_1
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_LOAD_SMALL_INT_1
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_MULTIPLY_INT
_LOAD_SMALL_INT_2
_SET_IP
_BINARY_OP
_CHECK_VALIDITY
_LOAD_FAST_BORROW_0
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_LOAD_SMALL_INT_1
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_SET_IP
_GUARD_BINARY_OP_EXTEND
_BINARY_OP_EXTEND
_CHECK_VALIDITY
_SET_IP
_RETURN_VALUE
_GUARD_IP
_CHECK_VALIDITY
_LOAD_FAST_BORROW_5
_GUARD_TOS_FLOAT
_GUARD_NOS_FLOAT
_BINARY_OP_MULTIPLY_FLOAT
_SET_IP
_BINARY_OP
_CHECK_VALIDITY
_SET_IP
_STORE_FAST_3
_EXIT_TRACE
> -] - -executor_0x65462fd7b4e0:i15 -> executor_0x65462fd7c7c0:start -executor_0x65462fd7b4e0:i16 -> executor_0x65462fd7c7c0:start -executor_0x65462fd7b4e0:i123 -> executor_0x65462fd79600:start -executor_0x65462fd79600 [ - shape = none - label = < - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Executor
part_At_times_u: 45
_START_EXECUTOR
_MAKE_WARM
_SET_IP
_FOR_ITER_TIER_TWO
_GUARD_IP
_CHECK_VALIDITY
_SET_IP
_GUARD_TOS_TUPLE
_UNPACK_SEQUENCE_TWO_TUPLE
_CHECK_VALIDITY
_SET_IP
_STORE_FAST_4
_STORE_FAST_5
_CHECK_VALIDITY
_LOAD_FAST_BORROW_3
_LOAD_GLOBAL_MODULE
_PUSH_NULL_CONDITIONAL
_LOAD_FAST_BORROW_4
_LOAD_FAST_BORROW_1
_SET_IP
_CHECK_PEP_523
_CHECK_FUNCTION_VERSION
_CHECK_FUNCTION_EXACT_ARGS
_CHECK_STACK_SPACE
_CHECK_RECURSION_REMAINING
_INIT_CALL_PY_EXACT_ARGS_2
_SAVE_RETURN_OFFSET
_PUSH_FRAME
_GUARD_IP
_TIER2_RESUME_CHECK
_LOAD_CONST
_LOAD_FAST_BORROW_0
_LOAD_FAST_BORROW_1
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_LOAD_FAST_BORROW_0
_LOAD_FAST_BORROW_1
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_LOAD_SMALL_INT_1
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_MULTIPLY_INT
_LOAD_SMALL_INT_2
_SET_IP
_BINARY_OP
_CHECK_VALIDITY
_LOAD_FAST_BORROW_0
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_LOAD_SMALL_INT_1
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_SET_IP
_GUARD_BINARY_OP_EXTEND
_BINARY_OP_EXTEND
_CHECK_VALIDITY
_SET_IP
_RETURN_VALUE
_GUARD_IP
_CHECK_VALIDITY
_LOAD_FAST_BORROW_5
_GUARD_TOS_FLOAT
_GUARD_NOS_FLOAT
_BINARY_OP_MULTIPLY_FLOAT
_GUARD_TOS_FLOAT
_GUARD_NOS_FLOAT
_BINARY_OP_ADD_FLOAT
_SET_IP
_STORE_FAST_3
_JUMP_TO_TOP
> -] - -executor_0x65462fd78530 [ - shape = none - label = < - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Executor
part_A_times_u: 37
_START_EXECUTOR
_MAKE_WARM
_SET_IP
_FOR_ITER_TIER_TWO
_GUARD_IP
_CHECK_VALIDITY
_SET_IP
_GUARD_TOS_TUPLE
_UNPACK_SEQUENCE_TWO_TUPLE
_CHECK_VALIDITY
_SET_IP
_STORE_FAST_4
_STORE_FAST_5
_CHECK_VALIDITY
_LOAD_FAST_BORROW_3
_LOAD_GLOBAL_MODULE
_PUSH_NULL_CONDITIONAL
_LOAD_FAST_BORROW_1
_LOAD_FAST_BORROW_4
_SET_IP
_CHECK_PEP_523
_CHECK_FUNCTION_VERSION
_CHECK_FUNCTION_EXACT_ARGS
_CHECK_STACK_SPACE
_CHECK_RECURSION_REMAINING
_INIT_CALL_PY_EXACT_ARGS_2
_SAVE_RETURN_OFFSET
_PUSH_FRAME
_GUARD_IP
_TIER2_RESUME_CHECK
_LOAD_CONST
_LOAD_FAST_BORROW_0
_LOAD_FAST_BORROW_1
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_LOAD_FAST_BORROW_0
_LOAD_FAST_BORROW_1
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_LOAD_SMALL_INT_1
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_MULTIPLY_INT
_LOAD_SMALL_INT_2
_SET_IP
_BINARY_OP
_CHECK_VALIDITY
_LOAD_FAST_BORROW_0
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_LOAD_SMALL_INT_1
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_SET_IP
_GUARD_BINARY_OP_EXTEND
_BINARY_OP_EXTEND
_CHECK_VALIDITY
_SET_IP
_RETURN_VALUE
_GUARD_IP
_CHECK_VALIDITY
_LOAD_FAST_BORROW_5
_SET_IP
_GUARD_BINARY_OP_EXTEND
_BINARY_OP_EXTEND
_CHECK_VALIDITY
_GUARD_TOS_FLOAT
_GUARD_NOS_FLOAT
_BINARY_OP_ADD_FLOAT
_SET_IP
_STORE_FAST_3
_JUMP_TO_TOP
> -] - } From a2e92a6570a65ff996b560d4fecaf5eb44101614 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 17 Oct 2025 18:09:37 +0100 Subject: [PATCH 042/190] Delete hello.gvz --- hello.gvz | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 hello.gvz diff --git a/hello.gvz b/hello.gvz deleted file mode 100644 index 9448c085123d25..00000000000000 --- a/hello.gvz +++ /dev/null @@ -1,6 +0,0 @@ -digraph ideal { - - rankdir = "LR" - -} - From cbb3ad24ede156d13a1df6ee97d72401cdc30a69 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 17 Oct 2025 18:17:42 +0100 Subject: [PATCH 043/190] turn off optimizer again (for now) --- hello.gvz | 426 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 426 insertions(+) create mode 100644 hello.gvz diff --git a/hello.gvz b/hello.gvz new file mode 100644 index 00000000000000..849b52472cd30b --- /dev/null +++ b/hello.gvz @@ -0,0 +1,426 @@ +digraph ideal { + + rankdir = "LR" + +executor_0x555555ebd390 [ + shape = none + label = < + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Executor
No code object
_START_EXECUTOR
_MAKE_WARM
_SET_IP
_CHECK_FUNCTION_VERSION
_CHECK_FUNCTION_EXACT_ARGS
_CHECK_STACK_SPACE_OPERAND
_CHECK_RECURSION_REMAINING
_INIT_CALL_PY_EXACT_ARGS_1
_SAVE_RETURN_OFFSET
_PUSH_FRAME
_GUARD_IP
_TIER2_RESUME_CHECK
_LOAD_FAST_BORROW_0
_SET_IP
_GUARD_TOS_TUPLE
_UNPACK_SEQUENCE_TWO_TUPLE
_CHECK_VALIDITY
_SET_IP
_STORE_FAST_1
_STORE_FAST_2
_CHECK_VALIDITY
_LOAD_CONST_INLINE_BORROW
_SET_IP
_STORE_FAST_3
_CHECK_VALIDITY
_GUARD_GLOBALS_VERSION
_LOAD_CONST_INLINE_BORROW
_PUSH_NULL
_LOAD_FAST_BORROW_2
_SET_IP
_CALL_BUILTIN_CLASS
_TIER2_RESUME_CHECK
_CHECK_VALIDITY
_SET_IP
_GET_ITER
_CHECK_VALIDITY
_SET_IP
_FOR_ITER_TIER_TWO
_GUARD_IP
_CHECK_VALIDITY
_SET_IP
_GUARD_TOS_TUPLE
_UNPACK_SEQUENCE_TWO_TUPLE
_CHECK_VALIDITY
_SET_IP
_STORE_FAST_4
_STORE_FAST_5
_CHECK_VALIDITY
_LOAD_FAST_BORROW_3
_LOAD_CONST_INLINE
_PUSH_NULL
_LOAD_FAST_BORROW_1
_LOAD_FAST_BORROW_4
_SET_IP
_CHECK_FUNCTION_VERSION_INLINE
_CHECK_RECURSION_REMAINING
_INIT_CALL_PY_EXACT_ARGS_2
_SAVE_RETURN_OFFSET
_PUSH_FRAME
_GUARD_IP
_TIER2_RESUME_CHECK
_LOAD_CONST_INLINE_BORROW
_LOAD_FAST_BORROW_0
_LOAD_FAST_BORROW_1
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_LOAD_FAST_BORROW_0
_LOAD_FAST_BORROW_1
_BINARY_OP_ADD_INT
_LOAD_CONST_INLINE_BORROW
_BINARY_OP_ADD_INT
_BINARY_OP_MULTIPLY_INT
_LOAD_CONST_INLINE_BORROW
_SET_IP
_BINARY_OP
_CHECK_VALIDITY
_LOAD_FAST_BORROW_0
_GUARD_NOS_OVERFLOWED
_BINARY_OP_ADD_INT
_LOAD_CONST_INLINE_BORROW
_BINARY_OP_ADD_INT
_SET_IP
_BINARY_OP
_CHECK_VALIDITY
_SET_IP
_RETURN_VALUE
_GUARD_IP
_CHECK_VALIDITY
_LOAD_FAST_BORROW_5
_GUARD_TOS_FLOAT
_BINARY_OP_MULTIPLY_FLOAT
_SET_IP
_BINARY_OP
_CHECK_VALIDITY
_SET_IP
_STORE_FAST_3
_EXIT_TRACE
> +] + +executor_0x555555ebd390:i97 -> executor_0x555555eb9530:start +executor_0x555555eb48f0 [ + shape = none + label = < + + + + + + + + + + + + + + +
Executor
part_At_times_u: 44
_START_EXECUTOR
_MAKE_WARM
_SET_IP
_POP_ITER
_CHECK_VALIDITY
_LOAD_FAST_BORROW_3
_SET_IP
_RETURN_VALUE
_GUARD_IP
_CHECK_VALIDITY
_LIST_APPEND
_EXIT_TRACE
> +] + +executor_0x555555eb48f0:i11 -> executor_0x555555ebc2b0:start +executor_0x555555e0b190 [ + shape = none + label = < + + + + + + + + + + + + + + +
Executor
part_A_times_u: 36
_START_EXECUTOR
_MAKE_WARM
_SET_IP
_POP_ITER
_CHECK_VALIDITY
_LOAD_FAST_BORROW_3
_SET_IP
_RETURN_VALUE
_GUARD_IP
_CHECK_VALIDITY
_LIST_APPEND
_EXIT_TRACE
> +] + +executor_0x555555e0b190:i11 -> executor_0x555555ebc2b0:start +executor_0x555555ebc2b0 [ + shape = none + label = < + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Executor
eval_times_u: 26
_START_EXECUTOR
_MAKE_WARM
_ITER_CHECK_RANGE
_GUARD_NOT_EXHAUSTED_RANGE
_ITER_NEXT_RANGE
_SET_IP
_STORE_FAST_2
_LOAD_FAST_0
_CHECK_VALIDITY
_PUSH_NULL
_LOAD_FAST_BORROW_2
_LOAD_FAST_BORROW_1
_BUILD_TUPLE
_SET_IP
_CHECK_FUNCTION_VERSION
_CHECK_FUNCTION_EXACT_ARGS
_CHECK_STACK_SPACE_OPERAND
_CHECK_RECURSION_REMAINING
_INIT_CALL_PY_EXACT_ARGS_1
_SAVE_RETURN_OFFSET
_PUSH_FRAME
_GUARD_IP
_TIER2_RESUME_CHECK
_LOAD_FAST_BORROW_0
_SET_IP
_UNPACK_SEQUENCE_TWO_TUPLE
_CHECK_VALIDITY
_SET_IP
_STORE_FAST_1
_STORE_FAST_2
_CHECK_VALIDITY
_LOAD_CONST_INLINE_BORROW
_SET_IP
_STORE_FAST_3
_CHECK_VALIDITY
_GUARD_GLOBALS_VERSION
_LOAD_CONST_INLINE_BORROW
_PUSH_NULL
_LOAD_FAST_BORROW_2
_SET_IP
_CALL_BUILTIN_CLASS
_TIER2_RESUME_CHECK
_CHECK_VALIDITY
_SET_IP
_GET_ITER
_CHECK_VALIDITY
_SET_IP
_FOR_ITER_TIER_TWO
_GUARD_IP
_CHECK_VALIDITY
_SET_IP
_GUARD_TOS_TUPLE
_UNPACK_SEQUENCE_TWO_TUPLE
_CHECK_VALIDITY
_SET_IP
_STORE_FAST_4
_STORE_FAST_5
_CHECK_VALIDITY
_LOAD_FAST_BORROW_3
_LOAD_CONST_INLINE
_PUSH_NULL
_LOAD_FAST_BORROW_4
_LOAD_FAST_BORROW_1
_SET_IP
_CHECK_FUNCTION_VERSION_INLINE
_CHECK_RECURSION_REMAINING
_INIT_CALL_PY_EXACT_ARGS_2
_SAVE_RETURN_OFFSET
_PUSH_FRAME
_GUARD_IP
_TIER2_RESUME_CHECK
_LOAD_CONST_INLINE_BORROW
_LOAD_FAST_BORROW_0
_LOAD_FAST_BORROW_1
_GUARD_TOS_OVERFLOWED
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_LOAD_FAST_BORROW_0
_LOAD_FAST_BORROW_1
_BINARY_OP_ADD_INT
_LOAD_CONST_INLINE_BORROW
_BINARY_OP_ADD_INT
_BINARY_OP_MULTIPLY_INT
_LOAD_CONST_INLINE_BORROW
_SET_IP
_BINARY_OP
_CHECK_VALIDITY
_LOAD_FAST_BORROW_0
_GUARD_NOS_OVERFLOWED
_BINARY_OP_ADD_INT
_LOAD_CONST_INLINE_BORROW
_BINARY_OP_ADD_INT
_SET_IP
_BINARY_OP
_CHECK_VALIDITY
_SET_IP
_RETURN_VALUE
_GUARD_IP
_CHECK_VALIDITY
_LOAD_FAST_BORROW_5
_GUARD_TOS_FLOAT
_BINARY_OP_MULTIPLY_FLOAT
_SET_IP
_BINARY_OP
_CHECK_VALIDITY
_SET_IP
_STORE_FAST_3
_EXIT_TRACE
> +] + +executor_0x555555ebc2b0:i14 -> executor_0x555555ebd390:start +executor_0x555555ebc2b0:i15 -> executor_0x555555ebd390:start +executor_0x555555ebc2b0:i107 -> executor_0x555555eba4f0:start +executor_0x555555eba4f0 [ + shape = none + label = < + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Executor
part_At_times_u: 45
_START_EXECUTOR
_MAKE_WARM
_SET_IP
_FOR_ITER_TIER_TWO
_GUARD_IP
_CHECK_VALIDITY
_SET_IP
_GUARD_TOS_TUPLE
_UNPACK_SEQUENCE_TWO_TUPLE
_CHECK_VALIDITY
_SET_IP
_STORE_FAST_4
_STORE_FAST_5
_CHECK_VALIDITY
_LOAD_FAST_BORROW_3
_GUARD_GLOBALS_VERSION
_LOAD_CONST_INLINE
_PUSH_NULL
_LOAD_FAST_BORROW_4
_LOAD_FAST_BORROW_1
_SET_IP
_CHECK_FUNCTION_VERSION_INLINE
_CHECK_STACK_SPACE_OPERAND
_CHECK_RECURSION_REMAINING
_INIT_CALL_PY_EXACT_ARGS_2
_SAVE_RETURN_OFFSET
_PUSH_FRAME
_GUARD_IP
_TIER2_RESUME_CHECK
_LOAD_CONST_INLINE_BORROW
_LOAD_FAST_BORROW_0
_LOAD_FAST_BORROW_1
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_LOAD_FAST_BORROW_0
_LOAD_FAST_BORROW_1
_BINARY_OP_ADD_INT
_LOAD_CONST_INLINE_BORROW
_BINARY_OP_ADD_INT
_BINARY_OP_MULTIPLY_INT
_LOAD_CONST_INLINE_BORROW
_SET_IP
_BINARY_OP
_CHECK_VALIDITY
_LOAD_FAST_BORROW_0
_GUARD_NOS_OVERFLOWED
_BINARY_OP_ADD_INT
_LOAD_CONST_INLINE_BORROW
_BINARY_OP_ADD_INT
_SET_IP
_GUARD_BINARY_OP_EXTEND
_BINARY_OP_EXTEND
_CHECK_VALIDITY
_SET_IP
_RETURN_VALUE
_GUARD_IP
_CHECK_VALIDITY
_LOAD_FAST_BORROW_5
_GUARD_TOS_FLOAT
_GUARD_NOS_FLOAT
_BINARY_OP_MULTIPLY_FLOAT
_GUARD_NOS_FLOAT
_BINARY_OP_ADD_FLOAT
_SET_IP
_STORE_FAST_3
_JUMP_TO_TOP
> +] + +executor_0x555555eb9530 [ + shape = none + label = < + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Executor
part_A_times_u: 37
_START_EXECUTOR
_MAKE_WARM
_SET_IP
_FOR_ITER_TIER_TWO
_GUARD_IP
_CHECK_VALIDITY
_SET_IP
_GUARD_TOS_TUPLE
_UNPACK_SEQUENCE_TWO_TUPLE
_CHECK_VALIDITY
_SET_IP
_STORE_FAST_4
_STORE_FAST_5
_CHECK_VALIDITY
_LOAD_FAST_BORROW_3
_GUARD_GLOBALS_VERSION
_LOAD_CONST_INLINE
_PUSH_NULL
_LOAD_FAST_BORROW_1
_LOAD_FAST_BORROW_4
_SET_IP
_CHECK_FUNCTION_VERSION_INLINE
_CHECK_STACK_SPACE_OPERAND
_CHECK_RECURSION_REMAINING
_INIT_CALL_PY_EXACT_ARGS_2
_SAVE_RETURN_OFFSET
_PUSH_FRAME
_GUARD_IP
_TIER2_RESUME_CHECK
_LOAD_CONST_INLINE_BORROW
_LOAD_FAST_BORROW_0
_LOAD_FAST_BORROW_1
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_LOAD_FAST_BORROW_0
_LOAD_FAST_BORROW_1
_BINARY_OP_ADD_INT
_LOAD_CONST_INLINE_BORROW
_BINARY_OP_ADD_INT
_BINARY_OP_MULTIPLY_INT
_LOAD_CONST_INLINE_BORROW
_SET_IP
_BINARY_OP
_CHECK_VALIDITY
_LOAD_FAST_BORROW_0
_GUARD_NOS_OVERFLOWED
_BINARY_OP_ADD_INT
_LOAD_CONST_INLINE_BORROW
_BINARY_OP_ADD_INT
_SET_IP
_GUARD_BINARY_OP_EXTEND
_BINARY_OP_EXTEND
_CHECK_VALIDITY
_SET_IP
_RETURN_VALUE
_GUARD_IP
_CHECK_VALIDITY
_LOAD_FAST_BORROW_5
_SET_IP
_GUARD_BINARY_OP_EXTEND
_BINARY_OP_EXTEND
_CHECK_VALIDITY
_GUARD_TOS_FLOAT
_GUARD_NOS_FLOAT
_BINARY_OP_ADD_FLOAT
_SET_IP
_STORE_FAST_3
_JUMP_TO_TOP
> +] + +} + From 9910b653a2168a7a8647c832ede62322aff3e395 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 17 Oct 2025 18:22:59 +0100 Subject: [PATCH 044/190] Turn off optimizer for real, trace through init --- Python/optimizer.c | 5 +- Python/optimizer_analysis.c | 14 +- hello.gvz | 426 ------------------------------------ 3 files changed, 9 insertions(+), 436 deletions(-) delete mode 100644 hello.gvz diff --git a/Python/optimizer.c b/Python/optimizer.c index 98ac0457477694..77f7dee553f364 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -597,9 +597,8 @@ _PyJIT_translate_single_bytecode_to_trace( // TODO handle extended args. oparg > 255 || opcode == EXTENDED_ARG || opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO || - frame->owner >= FRAME_OWNED_BY_INTERPRETER || - // This can be supported, but requires a tracing shim frame. - opcode == CALL_ALLOC_AND_ENTER_INIT) { + frame->owner >= FRAME_OWNED_BY_INTERPRETER + ) { unsupported: { // Rewind to previous instruction and replace with _EXIT_TRACE. diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 814575e50978dd..5e18b4e40f3d13 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -528,13 +528,13 @@ _Py_uop_analyze_and_optimize( { OPT_STAT_INC(optimizer_attempts); - int err = optimize_uops( - initial_func, buffer, - length, curr_stacklen, dependencies); - - if (err == 0) { - return err; - } + // int err = optimize_uops( + // initial_func, buffer, + // length, curr_stacklen, dependencies); + // + // if (err == 0) { + // return err; + // } assert(length > 0); diff --git a/hello.gvz b/hello.gvz deleted file mode 100644 index 849b52472cd30b..00000000000000 --- a/hello.gvz +++ /dev/null @@ -1,426 +0,0 @@ -digraph ideal { - - rankdir = "LR" - -executor_0x555555ebd390 [ - shape = none - label = < - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Executor
No code object
_START_EXECUTOR
_MAKE_WARM
_SET_IP
_CHECK_FUNCTION_VERSION
_CHECK_FUNCTION_EXACT_ARGS
_CHECK_STACK_SPACE_OPERAND
_CHECK_RECURSION_REMAINING
_INIT_CALL_PY_EXACT_ARGS_1
_SAVE_RETURN_OFFSET
_PUSH_FRAME
_GUARD_IP
_TIER2_RESUME_CHECK
_LOAD_FAST_BORROW_0
_SET_IP
_GUARD_TOS_TUPLE
_UNPACK_SEQUENCE_TWO_TUPLE
_CHECK_VALIDITY
_SET_IP
_STORE_FAST_1
_STORE_FAST_2
_CHECK_VALIDITY
_LOAD_CONST_INLINE_BORROW
_SET_IP
_STORE_FAST_3
_CHECK_VALIDITY
_GUARD_GLOBALS_VERSION
_LOAD_CONST_INLINE_BORROW
_PUSH_NULL
_LOAD_FAST_BORROW_2
_SET_IP
_CALL_BUILTIN_CLASS
_TIER2_RESUME_CHECK
_CHECK_VALIDITY
_SET_IP
_GET_ITER
_CHECK_VALIDITY
_SET_IP
_FOR_ITER_TIER_TWO
_GUARD_IP
_CHECK_VALIDITY
_SET_IP
_GUARD_TOS_TUPLE
_UNPACK_SEQUENCE_TWO_TUPLE
_CHECK_VALIDITY
_SET_IP
_STORE_FAST_4
_STORE_FAST_5
_CHECK_VALIDITY
_LOAD_FAST_BORROW_3
_LOAD_CONST_INLINE
_PUSH_NULL
_LOAD_FAST_BORROW_1
_LOAD_FAST_BORROW_4
_SET_IP
_CHECK_FUNCTION_VERSION_INLINE
_CHECK_RECURSION_REMAINING
_INIT_CALL_PY_EXACT_ARGS_2
_SAVE_RETURN_OFFSET
_PUSH_FRAME
_GUARD_IP
_TIER2_RESUME_CHECK
_LOAD_CONST_INLINE_BORROW
_LOAD_FAST_BORROW_0
_LOAD_FAST_BORROW_1
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_LOAD_FAST_BORROW_0
_LOAD_FAST_BORROW_1
_BINARY_OP_ADD_INT
_LOAD_CONST_INLINE_BORROW
_BINARY_OP_ADD_INT
_BINARY_OP_MULTIPLY_INT
_LOAD_CONST_INLINE_BORROW
_SET_IP
_BINARY_OP
_CHECK_VALIDITY
_LOAD_FAST_BORROW_0
_GUARD_NOS_OVERFLOWED
_BINARY_OP_ADD_INT
_LOAD_CONST_INLINE_BORROW
_BINARY_OP_ADD_INT
_SET_IP
_BINARY_OP
_CHECK_VALIDITY
_SET_IP
_RETURN_VALUE
_GUARD_IP
_CHECK_VALIDITY
_LOAD_FAST_BORROW_5
_GUARD_TOS_FLOAT
_BINARY_OP_MULTIPLY_FLOAT
_SET_IP
_BINARY_OP
_CHECK_VALIDITY
_SET_IP
_STORE_FAST_3
_EXIT_TRACE
> -] - -executor_0x555555ebd390:i97 -> executor_0x555555eb9530:start -executor_0x555555eb48f0 [ - shape = none - label = < - - - - - - - - - - - - - - -
Executor
part_At_times_u: 44
_START_EXECUTOR
_MAKE_WARM
_SET_IP
_POP_ITER
_CHECK_VALIDITY
_LOAD_FAST_BORROW_3
_SET_IP
_RETURN_VALUE
_GUARD_IP
_CHECK_VALIDITY
_LIST_APPEND
_EXIT_TRACE
> -] - -executor_0x555555eb48f0:i11 -> executor_0x555555ebc2b0:start -executor_0x555555e0b190 [ - shape = none - label = < - - - - - - - - - - - - - - -
Executor
part_A_times_u: 36
_START_EXECUTOR
_MAKE_WARM
_SET_IP
_POP_ITER
_CHECK_VALIDITY
_LOAD_FAST_BORROW_3
_SET_IP
_RETURN_VALUE
_GUARD_IP
_CHECK_VALIDITY
_LIST_APPEND
_EXIT_TRACE
> -] - -executor_0x555555e0b190:i11 -> executor_0x555555ebc2b0:start -executor_0x555555ebc2b0 [ - shape = none - label = < - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Executor
eval_times_u: 26
_START_EXECUTOR
_MAKE_WARM
_ITER_CHECK_RANGE
_GUARD_NOT_EXHAUSTED_RANGE
_ITER_NEXT_RANGE
_SET_IP
_STORE_FAST_2
_LOAD_FAST_0
_CHECK_VALIDITY
_PUSH_NULL
_LOAD_FAST_BORROW_2
_LOAD_FAST_BORROW_1
_BUILD_TUPLE
_SET_IP
_CHECK_FUNCTION_VERSION
_CHECK_FUNCTION_EXACT_ARGS
_CHECK_STACK_SPACE_OPERAND
_CHECK_RECURSION_REMAINING
_INIT_CALL_PY_EXACT_ARGS_1
_SAVE_RETURN_OFFSET
_PUSH_FRAME
_GUARD_IP
_TIER2_RESUME_CHECK
_LOAD_FAST_BORROW_0
_SET_IP
_UNPACK_SEQUENCE_TWO_TUPLE
_CHECK_VALIDITY
_SET_IP
_STORE_FAST_1
_STORE_FAST_2
_CHECK_VALIDITY
_LOAD_CONST_INLINE_BORROW
_SET_IP
_STORE_FAST_3
_CHECK_VALIDITY
_GUARD_GLOBALS_VERSION
_LOAD_CONST_INLINE_BORROW
_PUSH_NULL
_LOAD_FAST_BORROW_2
_SET_IP
_CALL_BUILTIN_CLASS
_TIER2_RESUME_CHECK
_CHECK_VALIDITY
_SET_IP
_GET_ITER
_CHECK_VALIDITY
_SET_IP
_FOR_ITER_TIER_TWO
_GUARD_IP
_CHECK_VALIDITY
_SET_IP
_GUARD_TOS_TUPLE
_UNPACK_SEQUENCE_TWO_TUPLE
_CHECK_VALIDITY
_SET_IP
_STORE_FAST_4
_STORE_FAST_5
_CHECK_VALIDITY
_LOAD_FAST_BORROW_3
_LOAD_CONST_INLINE
_PUSH_NULL
_LOAD_FAST_BORROW_4
_LOAD_FAST_BORROW_1
_SET_IP
_CHECK_FUNCTION_VERSION_INLINE
_CHECK_RECURSION_REMAINING
_INIT_CALL_PY_EXACT_ARGS_2
_SAVE_RETURN_OFFSET
_PUSH_FRAME
_GUARD_IP
_TIER2_RESUME_CHECK
_LOAD_CONST_INLINE_BORROW
_LOAD_FAST_BORROW_0
_LOAD_FAST_BORROW_1
_GUARD_TOS_OVERFLOWED
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_LOAD_FAST_BORROW_0
_LOAD_FAST_BORROW_1
_BINARY_OP_ADD_INT
_LOAD_CONST_INLINE_BORROW
_BINARY_OP_ADD_INT
_BINARY_OP_MULTIPLY_INT
_LOAD_CONST_INLINE_BORROW
_SET_IP
_BINARY_OP
_CHECK_VALIDITY
_LOAD_FAST_BORROW_0
_GUARD_NOS_OVERFLOWED
_BINARY_OP_ADD_INT
_LOAD_CONST_INLINE_BORROW
_BINARY_OP_ADD_INT
_SET_IP
_BINARY_OP
_CHECK_VALIDITY
_SET_IP
_RETURN_VALUE
_GUARD_IP
_CHECK_VALIDITY
_LOAD_FAST_BORROW_5
_GUARD_TOS_FLOAT
_BINARY_OP_MULTIPLY_FLOAT
_SET_IP
_BINARY_OP
_CHECK_VALIDITY
_SET_IP
_STORE_FAST_3
_EXIT_TRACE
> -] - -executor_0x555555ebc2b0:i14 -> executor_0x555555ebd390:start -executor_0x555555ebc2b0:i15 -> executor_0x555555ebd390:start -executor_0x555555ebc2b0:i107 -> executor_0x555555eba4f0:start -executor_0x555555eba4f0 [ - shape = none - label = < - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Executor
part_At_times_u: 45
_START_EXECUTOR
_MAKE_WARM
_SET_IP
_FOR_ITER_TIER_TWO
_GUARD_IP
_CHECK_VALIDITY
_SET_IP
_GUARD_TOS_TUPLE
_UNPACK_SEQUENCE_TWO_TUPLE
_CHECK_VALIDITY
_SET_IP
_STORE_FAST_4
_STORE_FAST_5
_CHECK_VALIDITY
_LOAD_FAST_BORROW_3
_GUARD_GLOBALS_VERSION
_LOAD_CONST_INLINE
_PUSH_NULL
_LOAD_FAST_BORROW_4
_LOAD_FAST_BORROW_1
_SET_IP
_CHECK_FUNCTION_VERSION_INLINE
_CHECK_STACK_SPACE_OPERAND
_CHECK_RECURSION_REMAINING
_INIT_CALL_PY_EXACT_ARGS_2
_SAVE_RETURN_OFFSET
_PUSH_FRAME
_GUARD_IP
_TIER2_RESUME_CHECK
_LOAD_CONST_INLINE_BORROW
_LOAD_FAST_BORROW_0
_LOAD_FAST_BORROW_1
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_LOAD_FAST_BORROW_0
_LOAD_FAST_BORROW_1
_BINARY_OP_ADD_INT
_LOAD_CONST_INLINE_BORROW
_BINARY_OP_ADD_INT
_BINARY_OP_MULTIPLY_INT
_LOAD_CONST_INLINE_BORROW
_SET_IP
_BINARY_OP
_CHECK_VALIDITY
_LOAD_FAST_BORROW_0
_GUARD_NOS_OVERFLOWED
_BINARY_OP_ADD_INT
_LOAD_CONST_INLINE_BORROW
_BINARY_OP_ADD_INT
_SET_IP
_GUARD_BINARY_OP_EXTEND
_BINARY_OP_EXTEND
_CHECK_VALIDITY
_SET_IP
_RETURN_VALUE
_GUARD_IP
_CHECK_VALIDITY
_LOAD_FAST_BORROW_5
_GUARD_TOS_FLOAT
_GUARD_NOS_FLOAT
_BINARY_OP_MULTIPLY_FLOAT
_GUARD_NOS_FLOAT
_BINARY_OP_ADD_FLOAT
_SET_IP
_STORE_FAST_3
_JUMP_TO_TOP
> -] - -executor_0x555555eb9530 [ - shape = none - label = < - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Executor
part_A_times_u: 37
_START_EXECUTOR
_MAKE_WARM
_SET_IP
_FOR_ITER_TIER_TWO
_GUARD_IP
_CHECK_VALIDITY
_SET_IP
_GUARD_TOS_TUPLE
_UNPACK_SEQUENCE_TWO_TUPLE
_CHECK_VALIDITY
_SET_IP
_STORE_FAST_4
_STORE_FAST_5
_CHECK_VALIDITY
_LOAD_FAST_BORROW_3
_GUARD_GLOBALS_VERSION
_LOAD_CONST_INLINE
_PUSH_NULL
_LOAD_FAST_BORROW_1
_LOAD_FAST_BORROW_4
_SET_IP
_CHECK_FUNCTION_VERSION_INLINE
_CHECK_STACK_SPACE_OPERAND
_CHECK_RECURSION_REMAINING
_INIT_CALL_PY_EXACT_ARGS_2
_SAVE_RETURN_OFFSET
_PUSH_FRAME
_GUARD_IP
_TIER2_RESUME_CHECK
_LOAD_CONST_INLINE_BORROW
_LOAD_FAST_BORROW_0
_LOAD_FAST_BORROW_1
_GUARD_TOS_INT
_GUARD_NOS_INT
_BINARY_OP_ADD_INT
_LOAD_FAST_BORROW_0
_LOAD_FAST_BORROW_1
_BINARY_OP_ADD_INT
_LOAD_CONST_INLINE_BORROW
_BINARY_OP_ADD_INT
_BINARY_OP_MULTIPLY_INT
_LOAD_CONST_INLINE_BORROW
_SET_IP
_BINARY_OP
_CHECK_VALIDITY
_LOAD_FAST_BORROW_0
_GUARD_NOS_OVERFLOWED
_BINARY_OP_ADD_INT
_LOAD_CONST_INLINE_BORROW
_BINARY_OP_ADD_INT
_SET_IP
_GUARD_BINARY_OP_EXTEND
_BINARY_OP_EXTEND
_CHECK_VALIDITY
_SET_IP
_RETURN_VALUE
_GUARD_IP
_CHECK_VALIDITY
_LOAD_FAST_BORROW_5
_SET_IP
_GUARD_BINARY_OP_EXTEND
_BINARY_OP_EXTEND
_CHECK_VALIDITY
_GUARD_TOS_FLOAT
_GUARD_NOS_FLOAT
_BINARY_OP_ADD_FLOAT
_SET_IP
_STORE_FAST_3
_JUMP_TO_TOP
> -] - -} - From 5102ab6028fba189ffb7f409046ab125381a5503 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 18 Oct 2025 00:47:16 +0100 Subject: [PATCH 045/190] fix a few tests and their exposed bugs --- Include/internal/pycore_opcode_metadata.h | 2 +- Include/internal/pycore_uop_ids.h | 218 +++++++++++----------- Lib/test/test_capi/test_opt.py | 6 +- Lib/test/test_sys.py | 5 +- Modules/_testinternalcapi.c | 3 +- Python/bytecodes.c | 37 ++-- Python/ceval.c | 6 +- Python/executor_cases.c.h | 26 ++- Python/optimizer.c | 32 ++-- 9 files changed, 165 insertions(+), 170 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index b96033eadf90b6..477553e2563f8c 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1778,6 +1778,7 @@ const uint8_t _PyOpcode_NeedsGuardIp[256] = { [RETURN_VALUE] = 1, [YIELD_VALUE] = 1, [JUMP_FORWARD] = 1, + [JUMP_BACKWARD_NO_INTERRUPT] = 1, [INSTRUMENTED_FOR_ITER] = 1, [RETURN_GENERATOR] = 1, [BINARY_OP_SUBSCR_GETITEM] = 1, @@ -1795,7 +1796,6 @@ const uint8_t _PyOpcode_NeedsGuardIp[256] = { [POP_JUMP_IF_FALSE] = 1, [POP_JUMP_IF_NONE] = 1, [POP_JUMP_IF_NOT_NONE] = 1, - [JUMP_BACKWARD_NO_INTERRUPT] = 1, [FOR_ITER] = 1, [FOR_ITER_LIST] = 1, [FOR_ITER_TUPLE] = 1, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 2ef1fccc8d8ee2..9f4ae71a313aa0 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -199,166 +199,166 @@ extern "C" { #define _ITER_NEXT_LIST_TIER_TWO 442 #define _ITER_NEXT_RANGE 443 #define _ITER_NEXT_TUPLE 444 -#define _JUMP_BACKWARD_NO_INTERRUPT 445 -#define _JUMP_TO_TOP 446 +#define _JUMP_BACKWARD_NO_INTERRUPT JUMP_BACKWARD_NO_INTERRUPT +#define _JUMP_TO_TOP 445 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 447 -#define _LOAD_ATTR_CLASS 448 +#define _LOAD_ATTR 446 +#define _LOAD_ATTR_CLASS 447 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 449 -#define _LOAD_ATTR_METHOD_LAZY_DICT 450 -#define _LOAD_ATTR_METHOD_NO_DICT 451 -#define _LOAD_ATTR_METHOD_WITH_VALUES 452 -#define _LOAD_ATTR_MODULE 453 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 454 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 455 -#define _LOAD_ATTR_PROPERTY_FRAME 456 -#define _LOAD_ATTR_SLOT 457 -#define _LOAD_ATTR_WITH_HINT 458 +#define _LOAD_ATTR_INSTANCE_VALUE 448 +#define _LOAD_ATTR_METHOD_LAZY_DICT 449 +#define _LOAD_ATTR_METHOD_NO_DICT 450 +#define _LOAD_ATTR_METHOD_WITH_VALUES 451 +#define _LOAD_ATTR_MODULE 452 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 453 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 454 +#define _LOAD_ATTR_PROPERTY_FRAME 455 +#define _LOAD_ATTR_SLOT 456 +#define _LOAD_ATTR_WITH_HINT 457 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _LOAD_BYTECODE 459 +#define _LOAD_BYTECODE 458 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 460 -#define _LOAD_CONST_INLINE_BORROW 461 -#define _LOAD_CONST_UNDER_INLINE 462 -#define _LOAD_CONST_UNDER_INLINE_BORROW 463 +#define _LOAD_CONST_INLINE 459 +#define _LOAD_CONST_INLINE_BORROW 460 +#define _LOAD_CONST_UNDER_INLINE 461 +#define _LOAD_CONST_UNDER_INLINE_BORROW 462 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 464 -#define _LOAD_FAST_0 465 -#define _LOAD_FAST_1 466 -#define _LOAD_FAST_2 467 -#define _LOAD_FAST_3 468 -#define _LOAD_FAST_4 469 -#define _LOAD_FAST_5 470 -#define _LOAD_FAST_6 471 -#define _LOAD_FAST_7 472 +#define _LOAD_FAST 463 +#define _LOAD_FAST_0 464 +#define _LOAD_FAST_1 465 +#define _LOAD_FAST_2 466 +#define _LOAD_FAST_3 467 +#define _LOAD_FAST_4 468 +#define _LOAD_FAST_5 469 +#define _LOAD_FAST_6 470 +#define _LOAD_FAST_7 471 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR -#define _LOAD_FAST_BORROW 473 -#define _LOAD_FAST_BORROW_0 474 -#define _LOAD_FAST_BORROW_1 475 -#define _LOAD_FAST_BORROW_2 476 -#define _LOAD_FAST_BORROW_3 477 -#define _LOAD_FAST_BORROW_4 478 -#define _LOAD_FAST_BORROW_5 479 -#define _LOAD_FAST_BORROW_6 480 -#define _LOAD_FAST_BORROW_7 481 +#define _LOAD_FAST_BORROW 472 +#define _LOAD_FAST_BORROW_0 473 +#define _LOAD_FAST_BORROW_1 474 +#define _LOAD_FAST_BORROW_2 475 +#define _LOAD_FAST_BORROW_3 476 +#define _LOAD_FAST_BORROW_4 477 +#define _LOAD_FAST_BORROW_5 478 +#define _LOAD_FAST_BORROW_6 479 +#define _LOAD_FAST_BORROW_7 480 #define _LOAD_FAST_BORROW_LOAD_FAST_BORROW LOAD_FAST_BORROW_LOAD_FAST_BORROW #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 482 -#define _LOAD_GLOBAL_BUILTINS 483 -#define _LOAD_GLOBAL_MODULE 484 +#define _LOAD_GLOBAL 481 +#define _LOAD_GLOBAL_BUILTINS 482 +#define _LOAD_GLOBAL_MODULE 483 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 485 -#define _LOAD_SMALL_INT_0 486 -#define _LOAD_SMALL_INT_1 487 -#define _LOAD_SMALL_INT_2 488 -#define _LOAD_SMALL_INT_3 489 -#define _LOAD_SPECIAL 490 +#define _LOAD_SMALL_INT 484 +#define _LOAD_SMALL_INT_0 485 +#define _LOAD_SMALL_INT_1 486 +#define _LOAD_SMALL_INT_2 487 +#define _LOAD_SMALL_INT_3 488 +#define _LOAD_SPECIAL 489 #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _MAKE_CALLARGS_A_TUPLE 491 +#define _MAKE_CALLARGS_A_TUPLE 490 #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_WARM 492 +#define _MAKE_WARM 491 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 493 -#define _MAYBE_EXPAND_METHOD_KW 494 -#define _MONITOR_CALL 495 -#define _MONITOR_CALL_KW 496 -#define _MONITOR_JUMP_BACKWARD 497 -#define _MONITOR_RESUME 498 +#define _MAYBE_EXPAND_METHOD 492 +#define _MAYBE_EXPAND_METHOD_KW 493 +#define _MONITOR_CALL 494 +#define _MONITOR_CALL_KW 495 +#define _MONITOR_JUMP_BACKWARD 496 +#define _MONITOR_RESUME 497 #define _NOP NOP -#define _POP_CALL 499 -#define _POP_CALL_LOAD_CONST_INLINE_BORROW 500 -#define _POP_CALL_ONE 501 -#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 502 -#define _POP_CALL_TWO 503 -#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 504 +#define _POP_CALL 498 +#define _POP_CALL_LOAD_CONST_INLINE_BORROW 499 +#define _POP_CALL_ONE 500 +#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 501 +#define _POP_CALL_TWO 502 +#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 503 #define _POP_EXCEPT POP_EXCEPT #define _POP_ITER POP_ITER -#define _POP_JUMP_IF_FALSE 505 -#define _POP_JUMP_IF_TRUE 506 +#define _POP_JUMP_IF_FALSE 504 +#define _POP_JUMP_IF_TRUE 505 #define _POP_TOP POP_TOP -#define _POP_TOP_FLOAT 507 -#define _POP_TOP_INT 508 -#define _POP_TOP_LOAD_CONST_INLINE 509 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 510 -#define _POP_TOP_NOP 511 -#define _POP_TOP_UNICODE 512 -#define _POP_TWO 513 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW 514 +#define _POP_TOP_FLOAT 506 +#define _POP_TOP_INT 507 +#define _POP_TOP_LOAD_CONST_INLINE 508 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 509 +#define _POP_TOP_NOP 510 +#define _POP_TOP_UNICODE 511 +#define _POP_TWO 512 +#define _POP_TWO_LOAD_CONST_INLINE_BORROW 513 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 515 +#define _PUSH_FRAME 514 #define _PUSH_NULL PUSH_NULL -#define _PUSH_NULL_CONDITIONAL 516 -#define _PY_FRAME_GENERAL 517 -#define _PY_FRAME_KW 518 -#define _QUICKEN_RESUME 519 -#define _REPLACE_WITH_TRUE 520 +#define _PUSH_NULL_CONDITIONAL 515 +#define _PY_FRAME_GENERAL 516 +#define _PY_FRAME_KW 517 +#define _QUICKEN_RESUME 518 +#define _REPLACE_WITH_TRUE 519 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 521 -#define _SEND 522 -#define _SEND_GEN_FRAME 523 +#define _SAVE_RETURN_OFFSET 520 +#define _SEND 521 +#define _SEND_GEN_FRAME 522 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 524 -#define _STORE_ATTR 525 -#define _STORE_ATTR_INSTANCE_VALUE 526 -#define _STORE_ATTR_SLOT 527 -#define _STORE_ATTR_WITH_HINT 528 +#define _START_EXECUTOR 523 +#define _STORE_ATTR 524 +#define _STORE_ATTR_INSTANCE_VALUE 525 +#define _STORE_ATTR_SLOT 526 +#define _STORE_ATTR_WITH_HINT 527 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 529 -#define _STORE_FAST_0 530 -#define _STORE_FAST_1 531 -#define _STORE_FAST_2 532 -#define _STORE_FAST_3 533 -#define _STORE_FAST_4 534 -#define _STORE_FAST_5 535 -#define _STORE_FAST_6 536 -#define _STORE_FAST_7 537 +#define _STORE_FAST 528 +#define _STORE_FAST_0 529 +#define _STORE_FAST_1 530 +#define _STORE_FAST_2 531 +#define _STORE_FAST_3 532 +#define _STORE_FAST_4 533 +#define _STORE_FAST_5 534 +#define _STORE_FAST_6 535 +#define _STORE_FAST_7 536 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 538 -#define _STORE_SUBSCR 539 -#define _STORE_SUBSCR_DICT 540 -#define _STORE_SUBSCR_LIST_INT 541 -#define _SWAP 542 -#define _SWAP_2 543 -#define _SWAP_3 544 -#define _TIER2_RESUME_CHECK 545 -#define _TO_BOOL 546 +#define _STORE_SLICE 537 +#define _STORE_SUBSCR 538 +#define _STORE_SUBSCR_DICT 539 +#define _STORE_SUBSCR_LIST_INT 540 +#define _SWAP 541 +#define _SWAP_2 542 +#define _SWAP_3 543 +#define _TIER2_RESUME_CHECK 544 +#define _TO_BOOL 545 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT -#define _TO_BOOL_LIST 547 +#define _TO_BOOL_LIST 546 #define _TO_BOOL_NONE TO_BOOL_NONE -#define _TO_BOOL_STR 548 +#define _TO_BOOL_STR 547 #define _UNARY_INVERT UNARY_INVERT #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 549 -#define _UNPACK_SEQUENCE_LIST 550 -#define _UNPACK_SEQUENCE_TUPLE 551 -#define _UNPACK_SEQUENCE_TWO_TUPLE 552 +#define _UNPACK_SEQUENCE 548 +#define _UNPACK_SEQUENCE_LIST 549 +#define _UNPACK_SEQUENCE_TUPLE 550 +#define _UNPACK_SEQUENCE_TWO_TUPLE 551 #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 552 +#define MAX_UOP_ID 551 #ifdef __cplusplus } diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 3cc8fbe0de0a6c..842a328f5d83b4 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -17,8 +17,6 @@ from _testinternalcapi import TIER2_THRESHOLD -# We need one more iteration as one iteration is spent on tracing. -TIER2_THRESHOLD = TIER2_THRESHOLD + 1 #For test of issue 136154 GLOBAL_136154 = 42 @@ -628,7 +626,7 @@ def testfunc(n): x = range(i) return x - testfunc(_testinternalcapi.TIER2_THRESHOLD + 1) + testfunc(_testinternalcapi.TIER2_THRESHOLD) ex = get_first_executor(testfunc) assert ex is not None @@ -2463,7 +2461,7 @@ def testfunc(n): del email.jit_testing - testfunc(_testinternalcapi.TIER2_THRESHOLD + 1) + testfunc(_testinternalcapi.TIER2_THRESHOLD) ex = get_first_executor(testfunc) assert ex is not None """)) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 1198c6d35113c8..98dd466849e555 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -2252,9 +2252,10 @@ def frame_2_jit(expected: bool) -> None: def frame_3_jit() -> None: # JITs just before the last loop: - for i in range(_testinternalcapi.TIER2_THRESHOLD + 1): + # 1 extra iteration for tracing. + for i in range(_testinternalcapi.TIER2_THRESHOLD + 2): # Careful, doing this in the reverse order breaks tracing: - expected = {enabled} and i == _testinternalcapi.TIER2_THRESHOLD + expected = {enabled} and i >= _testinternalcapi.TIER2_THRESHOLD + 1 assert sys._jit.is_active() is expected frame_2_jit(expected) assert sys._jit.is_active() is expected diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 7aa63f913af335..158ffb5b9d803b 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -2543,7 +2543,8 @@ module_exec(PyObject *module) } if (PyModule_Add(module, "TIER2_THRESHOLD", - PyLong_FromLong(JUMP_BACKWARD_INITIAL_VALUE + 1)) < 0) { + // + 1 more due to one loop spent on tracing. + PyLong_FromLong(JUMP_BACKWARD_INITIAL_VALUE + 2)) < 0) { return 1; } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 5ab80565d6d486..a4a75008c66cdf 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2993,17 +2993,17 @@ dummy_func( unused/1 + _SPECIALIZE_JUMP_BACKWARD + _CHECK_PERIODIC + - _JUMP_BACKWARD_NO_INTERRUPT; + JUMP_BACKWARD_NO_INTERRUPT; macro(JUMP_BACKWARD_NO_JIT) = unused/1 + _CHECK_PERIODIC + - _JUMP_BACKWARD_NO_INTERRUPT; + JUMP_BACKWARD_NO_INTERRUPT; macro(JUMP_BACKWARD_JIT) = unused/1 + _CHECK_PERIODIC + - _JUMP_BACKWARD_NO_INTERRUPT + + JUMP_BACKWARD_NO_INTERRUPT + _JIT; pseudo(JUMP, (--)) = { @@ -3088,10 +3088,7 @@ dummy_func( macro(POP_JUMP_IF_NOT_NONE) = unused/1 + _IS_NONE + _POP_JUMP_IF_FALSE; - // This actually has 1 cache entry, but for some reason it's not factored in the oparg. - macro(JUMP_BACKWARD_NO_INTERRUPT) = _JUMP_BACKWARD_NO_INTERRUPT; - - op(_JUMP_BACKWARD_NO_INTERRUPT, (--)) { + inst(JUMP_BACKWARD_NO_INTERRUPT, (--)) { /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. @@ -5431,31 +5428,29 @@ dummy_func( tier2 op(_COLD_EXIT, ( -- )) { _PyExitData *exit = tstate->jit_exit; assert(exit != NULL); - _Py_CODEUNIT *target = (_PyFrame_GetBytecode(frame) + exit->target); + _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; _Py_BackoffCounter temperature = exit->temperature; + if (!backoff_counter_triggers(temperature)) { + exit->temperature = advance_backoff_counter(temperature); + GOTO_TIER_ONE(target, 0); + } + _PyExecutorObject *executor; if (target->op.code == ENTER_EXECUTOR) { PyCodeObject *code = _PyFrame_GetCode(frame); - _PyExecutorObject *executor = code->co_executors->executors[target->op.arg]; + executor = code->co_executors->executors[target->op.arg]; Py_INCREF(executor); - assert(tstate->jit_exit == exit); - exit->executor = executor; - TIER2_TO_TIER2(exit->executor); } else { - if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) { - GOTO_TIER_ONE(target, 0); - } - if (!backoff_counter_triggers(temperature)) { - exit->temperature = advance_backoff_counter(temperature); - GOTO_TIER_ONE(target, 0); - } - exit->temperature = initial_temperature_backoff_counter(); _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); assert(tstate->current_executor == (PyObject *)previous_executor); int chain_depth = previous_executor->vm_data.chain_depth + 1; _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), chain_depth, exit); + exit->temperature = initial_temperature_backoff_counter(); GOTO_TIER_ONE(target, 1); } + assert(tstate->jit_exit == exit); + exit->executor = executor; + TIER2_TO_TIER2(exit->executor); } tier2 op(_GUARD_IP, (ip/4 --)) { @@ -5494,8 +5489,6 @@ dummy_func( GOTO_TIER_ONE(target, 0); } exit->temperature = initial_temperature_backoff_counter(); - _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); - assert(tstate->current_executor == (PyObject *)previous_executor); _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), 0, NULL); GOTO_TIER_ONE(target, 1); } diff --git a/Python/ceval.c b/Python/ceval.c index 035a22615ca961..844ca4297f4157 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1054,10 +1054,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int _Py_CODEUNIT *next_instr; _PyStackRef *stack_pointer; entry.stack[0] = PyStackRef_NULL; - entry.frame.f_funcobj = PyStackRef_None; -#ifdef Py_STACKREF_DEBUG - entry.frame.f_funcobj = PyStackRef_None; -#elif defined(Py_DEBUG) + entry.frame.f_funcobj = PyStackRef_NULL; +#if defined(Py_DEBUG) /* Set these to invalid but identifiable values for debugging. */ entry.frame.f_locals = (PyObject*)0xaaa1; entry.frame.frame_obj = (PyFrameObject*)0xaaa2; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index f13baadd3bc76f..0a664209830f1f 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7487,31 +7487,29 @@ case _COLD_EXIT: { _PyExitData *exit = tstate->jit_exit; assert(exit != NULL); - _Py_CODEUNIT *target = (_PyFrame_GetBytecode(frame) + exit->target); + _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; _Py_BackoffCounter temperature = exit->temperature; + if (!backoff_counter_triggers(temperature)) { + exit->temperature = advance_backoff_counter(temperature); + GOTO_TIER_ONE(target, 0); + } + _PyExecutorObject *executor; if (target->op.code == ENTER_EXECUTOR) { PyCodeObject *code = _PyFrame_GetCode(frame); - _PyExecutorObject *executor = code->co_executors->executors[target->op.arg]; + executor = code->co_executors->executors[target->op.arg]; Py_INCREF(executor); - assert(tstate->jit_exit == exit); - exit->executor = executor; - TIER2_TO_TIER2(exit->executor); } else { - if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) { - GOTO_TIER_ONE(target, 0); - } - if (!backoff_counter_triggers(temperature)) { - exit->temperature = advance_backoff_counter(temperature); - GOTO_TIER_ONE(target, 0); - } - exit->temperature = initial_temperature_backoff_counter(); _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); assert(tstate->current_executor == (PyObject *)previous_executor); int chain_depth = previous_executor->vm_data.chain_depth + 1; _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), chain_depth, exit); + exit->temperature = initial_temperature_backoff_counter(); GOTO_TIER_ONE(target, 1); } + assert(tstate->jit_exit == exit); + exit->executor = executor; + TIER2_TO_TIER2(exit->executor); break; } @@ -7557,8 +7555,6 @@ GOTO_TIER_ONE(target, 0); } exit->temperature = initial_temperature_backoff_counter(); - _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); - assert(tstate->current_executor == (PyObject *)previous_executor); _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), 0, NULL); GOTO_TIER_ONE(target, 1); } diff --git a/Python/optimizer.c b/Python/optimizer.c index 77f7dee553f364..5d75a87ee0de5e 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -128,7 +128,8 @@ _PyOptimizer_Optimize( // make progress in order to avoid infinite loops or excessively-long // side-exit chains. We can only insert the executor into the bytecode if // this is true, since a deopt won't infinitely re-enter the executor: - bool progress_needed = (chain_depth % MAX_CHAIN_DEPTH) == 0; + chain_depth %= MAX_CHAIN_DEPTH; + bool progress_needed = chain_depth == 0; PyCodeObject *code = (PyCodeObject *)tstate->interp->jit_tracer_initial_code; _Py_CODEUNIT *start = tstate->interp->jit_tracer_initial_instr; // A recursive trace might've cleared the values. In that case, bail. @@ -559,8 +560,8 @@ _PyJIT_translate_single_bytecode_to_trace( int oparg, int jump_taken) { - if (Py_IsNone((PyObject *)func)) { - func = NULL; + if (PyStackRef_IsNull(frame->f_funcobj)) { + goto unsupported; } int is_first_instr = tstate->interp->jit_tracer_initial_instr == this_instr; @@ -596,6 +597,11 @@ _PyJIT_translate_single_bytecode_to_trace( (frame != tstate->interp->jit_tracer_current_frame && !needs_guard_ip) || // TODO handle extended args. oparg > 255 || opcode == EXTENDED_ARG || + // TODO handle BINARY_OP_INPLACE_ADD_UNICODE + opcode == BINARY_OP_INPLACE_ADD_UNICODE || + // TODO (gh-140277): The constituent uops are invalid. + opcode == BINARY_OP_SUBSCR_GETITEM || + // Exception stuff, could be handled in the future maybe? opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO || frame->owner >= FRAME_OWNED_BY_INTERPRETER ) { @@ -603,12 +609,19 @@ _PyJIT_translate_single_bytecode_to_trace( { // Rewind to previous instruction and replace with _EXIT_TRACE. _PyUOpInstruction *curr = &trace[trace_length-1]; - while (curr->opcode != _SET_IP && trace_length > 1) { + while (curr->opcode != _SET_IP && trace_length > 2) { trace_length--; curr = &trace[trace_length-1]; } - assert(curr->opcode == _SET_IP || trace_length == 1); - curr->opcode = _EXIT_TRACE; + assert(curr->opcode == _SET_IP || trace_length == 2); + if (curr->opcode == _SET_IP) { + int32_t old_target = (int32_t)uop_get_target(curr); + curr++; + trace_length++; + curr->opcode = _EXIT_TRACE; + curr->format = UOP_FORMAT_TARGET; + curr->target = old_target; + } goto done; } } @@ -621,10 +634,6 @@ _PyJIT_translate_single_bytecode_to_trace( return 1; } - if (opcode == JUMP_BACKWARD_NO_INTERRUPT) { - return 1; - } - if (opcode == JUMP_FORWARD) { return 1; } @@ -661,8 +670,7 @@ _PyJIT_translate_single_bytecode_to_trace( // Loop back to the start if (is_first_instr && tstate->interp->jit_tracer_code_curr_size > 2) { - // Undo the last few instructions. - trace_length = tstate->interp->jit_tracer_code_curr_size; + ADD_TO_TRACE(_CHECK_PERIODIC, 0, 0, target); ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0, 0); goto done; } From c0c14b45b976ecc18f536db3292a001af4a2d149 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 18 Oct 2025 01:13:28 +0100 Subject: [PATCH 046/190] Restore the optimizer fully --- Python/ceval_macros.h | 25 +++++++++++++++++-------- Python/optimizer.c | 5 +++-- Python/optimizer_analysis.c | 14 +++++++------- Python/optimizer_bytecodes.c | 22 +++++++++------------- Python/optimizer_cases.c.h | 15 ++++----------- 5 files changed, 40 insertions(+), 41 deletions(-) diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index ff038026f9770a..557757f8a259a1 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -143,14 +143,23 @@ # define LEAVE_TRACING() \ DISPATCH_TABLE_VAR = DISPATCH_TABLE; # define BAIL_TRACING_NO_DISPATCH() \ - LEAVE_TRACING(); \ - _PyFrame_SetStackPointer(frame, stack_pointer); \ - int _err = _PyOptimizer_Optimize(frame, tstate); \ - _PyJIT_FinalizeTracing(tstate); \ - stack_pointer = _PyFrame_GetStackPointer(frame); \ - if (_err < 0) { \ - JUMP_TO_LABEL(error); \ - } + do { \ + LEAVE_TRACING(); \ + if (!_PyErr_Occurred(tstate)) { \ + _PyFrame_SetStackPointer(frame, stack_pointer); \ + int _err = _PyOptimizer_Optimize(frame, tstate); \ + _PyJIT_FinalizeTracing(tstate); \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ + if (_err < 0) { \ + JUMP_TO_LABEL(error); \ + } \ + } \ + else { \ + _PyFrame_SetStackPointer(frame, stack_pointer); \ + _PyJIT_FinalizeTracing(tstate); \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ + } \ + } while (0); # define RECORD_TRACE_NO_DISPATCH() do { \ if (IS_JIT_TRACING() && add_to_code_trace(tstate, frame, old_code, old_func, this_instr, next_instr, opcode, oparg, _jump_taken)) { \ BAIL_TRACING_NO_DISPATCH(); \ diff --git a/Python/optimizer.c b/Python/optimizer.c index 5d75a87ee0de5e..c9009528e20921 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -794,8 +794,9 @@ _PyJIT_translate_single_bytecode_to_trace( } if (uop == _PUSH_FRAME || uop == _RETURN_VALUE || uop == _RETURN_GENERATOR || uop == _YIELD_VALUE) { PyCodeObject *new_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - if (func != NULL) { - operand = (uintptr_t)func; + PyFunctionObject *new_func = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + if (new_func != NULL) { + operand = (uintptr_t)new_func; } else if (new_code != NULL) { operand = (uintptr_t)new_code | 1; diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 5e18b4e40f3d13..7cee20bd8ee2a0 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -528,13 +528,13 @@ _Py_uop_analyze_and_optimize( { OPT_STAT_INC(optimizer_attempts); - // int err = optimize_uops( - // initial_func, buffer, - // length, curr_stacklen, dependencies); - // - // if (err == 0) { - // return err; - // } + int err = optimize_uops( + initial_func, buffer, + length, curr_stacklen, dependencies); + + if (err == 0) { + return err; + } assert(length > 0); diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index f9f8c15ab964d6..734a9193af7f89 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -794,25 +794,21 @@ dummy_func(void) { op(_RETURN_GENERATOR, ( -- res)) { SYNC_SP(); - PyCodeObject *co = get_current_code_object(ctx); - ctx->frame->stack_pointer = stack_pointer; - if (frame_pop(ctx)) { - break; - } stack_pointer = ctx->frame->stack_pointer; res = sym_new_unknown(ctx); - - /* Stack space handling */ - assert(corresponding_check_stack == NULL); - assert(co != NULL); - int framesize = co->co_framesize; - assert(framesize > 0); - assert(framesize <= curr_space); - curr_space -= framesize; + ctx->done = true; + ctx->out_of_space = true; } op(_YIELD_VALUE, (unused -- value)) { + // TODO (gh-139109): handle this properly in a future optimization. + // A possibility to handle underflows is to just restore the current frame information + // from whatever is stored in the trace we record at that point of time. + // E.g. we record at this YIELD_VALUE, func_obj=x , stack_level=4 + // We can restore it to there. value = sym_new_unknown(ctx); + ctx->done = true; + ctx->out_of_space = true; } op(_GET_ITER, (iterable -- iter, index_or_null)) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 3d31c7f04dad89..f5bb8a5585af93 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1171,6 +1171,8 @@ case _YIELD_VALUE: { JitOptRef value; value = sym_new_unknown(ctx); + ctx->done = true; + ctx->out_of_space = true; stack_pointer[-1] = value; break; } @@ -3011,19 +3013,10 @@ case _RETURN_GENERATOR: { JitOptRef res; - PyCodeObject *co = get_current_code_object(ctx); - ctx->frame->stack_pointer = stack_pointer; - if (frame_pop(ctx)) { - break; - } stack_pointer = ctx->frame->stack_pointer; res = sym_new_unknown(ctx); - assert(corresponding_check_stack == NULL); - assert(co != NULL); - int framesize = co->co_framesize; - assert(framesize > 0); - assert(framesize <= curr_space); - curr_space -= framesize; + ctx->done = true; + ctx->out_of_space = true; stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); From 7d4f8664b98d6617178a82522f61e5a6c238c7bf Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 18 Oct 2025 20:13:19 +0100 Subject: [PATCH 047/190] invalidate freed code/function objects used for global promotion --- Include/internal/pycore_interp_structs.h | 1 + Include/internal/pycore_optimizer.h | 3 +++ Objects/codeobject.c | 1 + Objects/funcobject.c | 5 +++- Python/optimizer.c | 34 +++++++++++++++++++++--- 5 files changed, 39 insertions(+), 5 deletions(-) diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index e3264035ed9b4e..42295de90478f9 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -947,6 +947,7 @@ struct _is { int jit_tracer_code_max_size; int jit_tracer_code_curr_size; _PyBloomFilter jit_tracer_dependencies; + bool jit_tracer_dependencies_still_valid; _PyUOpInstruction *jit_tracer_code_buffer; _Py_CODEUNIT *jit_tracer_initial_instr; int jit_tracer_initial_stack_depth; diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 748bf854768c0b..ca9f9702df4cdf 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -375,6 +375,9 @@ void _PyJIT_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit); void _PyJIT_FinalizeTracing(PyThreadState *tstate); + +void _Py_JITTracer_InvalidateDependency(PyThreadState *old_tstate, void *obj); + #ifdef __cplusplus } #endif diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 0d264a6e346f95..d0dd65300a1d8e 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -2415,6 +2415,7 @@ code_dealloc(PyObject *self) PyMem_Free(co_extra); } #ifdef _Py_TIER2 + _Py_JITTracer_InvalidateDependency(tstate, self); if (co->co_executors != NULL) { clear_executors(co); } diff --git a/Objects/funcobject.c b/Objects/funcobject.c index d8a10075578087..24f7a2920fdb78 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -11,7 +11,7 @@ #include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_stats.h" #include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() - +#include "pycore_optimizer.h" // _Py_JITTracer_InvalidateDependency static const char * func_event_name(PyFunction_WatchEvent event) { @@ -1149,6 +1149,9 @@ func_dealloc(PyObject *self) if (_PyObject_ResurrectEnd(self)) { return; } +#if _Py_TIER2 + _Py_JITTracer_InvalidateDependency(PyThreadState_GET(), self); +#endif _PyObject_GC_UNTRACK(op); FT_CLEAR_WEAKREFS(self, op->func_weakreflist); (void)func_clear((PyObject*)op); diff --git a/Python/optimizer.c b/Python/optimizer.c index c9009528e20921..f3166afd7d59a6 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -147,6 +147,11 @@ _PyOptimizer_Optimize( interp->compiling = false; return 0; } + // One of our depencies while tracing was invalidated. Not worth compiling. + if (!tstate->interp->jit_tracer_dependencies_still_valid) { + interp->compiling = false; + return 0; + } _PyExecutorObject *executor; int err = uop_optimize(frame, tstate, &executor, progress_needed); if (err <= 0) { @@ -560,9 +565,6 @@ _PyJIT_translate_single_bytecode_to_trace( int oparg, int jump_taken) { - if (PyStackRef_IsNull(frame->f_funcobj)) { - goto unsupported; - } int is_first_instr = tstate->interp->jit_tracer_initial_instr == this_instr; bool progress_needed = (tstate->interp->jit_tracer_initial_chain_depth % MAX_CHAIN_DEPTH) == 0 && is_first_instr;; @@ -794,12 +796,16 @@ _PyJIT_translate_single_bytecode_to_trace( } if (uop == _PUSH_FRAME || uop == _RETURN_VALUE || uop == _RETURN_GENERATOR || uop == _YIELD_VALUE) { PyCodeObject *new_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - PyFunctionObject *new_func = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + PyFunctionObject *new_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); if (new_func != NULL) { operand = (uintptr_t)new_func; + DPRINTF(2, "Adding %p func to op\n", (void *)operand); + _Py_BloomFilter_Add(dependencies, new_func); } else if (new_code != NULL) { operand = (uintptr_t)new_code | 1; + DPRINTF(2, "Adding %p code to op\n", (void *)operand); + _Py_BloomFilter_Add(dependencies, new_code); } else { operand = 0; @@ -866,6 +872,7 @@ _PyJIT_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_ tstate->interp->jit_tracer_initial_stack_depth = curr_stackdepth; tstate->interp->jit_tracer_initial_chain_depth = chain_depth; tstate->interp->jit_tracer_current_frame = frame; + tstate->interp->jit_tracer_dependencies_still_valid = true; } void @@ -1530,6 +1537,25 @@ _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj, int is _Py_Executors_InvalidateAll(interp, is_invalidation); } +void +_Py_JITTracer_InvalidateDependency(PyThreadState *old_tstate, void *obj) +{ + _PyBloomFilter obj_filter; + _Py_BloomFilter_Init(&obj_filter); + _Py_BloomFilter_Add(&obj_filter, obj); + HEAD_LOCK(&_PyRuntime); + + PyInterpreterState *interp = old_tstate->interp; + + _Py_FOR_EACH_TSTATE_UNLOCKED(interp, tstate) { + if (bloom_filter_may_contain(&tstate->interp->jit_tracer_dependencies, &obj_filter)) + { + tstate->interp->jit_tracer_dependencies_still_valid = false; + } + + } + HEAD_UNLOCK(&_PyRuntime); +} /* Invalidate all executors */ void _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation) From 1981f500290b6410a2ee06754eed6f4a8a28c70a Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 18 Oct 2025 20:54:56 +0100 Subject: [PATCH 048/190] Fix tracing --- Python/ceval_macros.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 557757f8a259a1..52dd8f7c3acae0 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -227,6 +227,9 @@ do { \ #define TRACING_DISPATCH() \ { \ + if (tstate->c_tracefunc || tstate->c_profilefunc) { \ + DISPATCH(); \ + } \ assert(frame->stackpointer == NULL); \ RECORD_TRACE_NO_DISPATCH(); \ NEXTOPARG(); \ From b879dab207d0c75ac8c3b26f09365d5670adcf3a Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 18 Oct 2025 20:59:07 +0100 Subject: [PATCH 049/190] fix tracing completely --- Python/ceval_macros.h | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 52dd8f7c3acae0..f68e84e973896e 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -145,7 +145,7 @@ # define BAIL_TRACING_NO_DISPATCH() \ do { \ LEAVE_TRACING(); \ - if (!_PyErr_Occurred(tstate)) { \ + if (!_PyErr_Occurred(tstate) && !_is_sys_tracing) { \ _PyFrame_SetStackPointer(frame, stack_pointer); \ int _err = _PyOptimizer_Optimize(frame, tstate); \ _PyJIT_FinalizeTracing(tstate); \ @@ -161,7 +161,8 @@ } \ } while (0); # define RECORD_TRACE_NO_DISPATCH() do { \ - if (IS_JIT_TRACING() && add_to_code_trace(tstate, frame, old_code, old_func, this_instr, next_instr, opcode, oparg, _jump_taken)) { \ + int _is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc == NULL); \ + if (_is_sys_tracing || IS_JIT_TRACING() && add_to_code_trace(tstate, frame, old_code, old_func, this_instr, next_instr, opcode, oparg, _jump_taken)) { \ BAIL_TRACING_NO_DISPATCH(); \ } \ } while (0); @@ -227,9 +228,6 @@ do { \ #define TRACING_DISPATCH() \ { \ - if (tstate->c_tracefunc || tstate->c_profilefunc) { \ - DISPATCH(); \ - } \ assert(frame->stackpointer == NULL); \ RECORD_TRACE_NO_DISPATCH(); \ NEXTOPARG(); \ From 093578cce1858ba5156c0c9ab10449bf32fe569a Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 18 Oct 2025 21:20:12 +0100 Subject: [PATCH 050/190] Separate the tracer out into its own file --- Makefile.pre.in | 9 +- Python/bytecodes.c | 13 +- Python/ceval.c | 2 + Python/generated_cases.c.h | 15460 +------------------- Python/generated_tracer_cases.c.h | 14100 ++++++++++++++++++ Tools/cases_generator/tier1_generator.py | 14 +- Tools/cases_generator/tracer_generator.py | 52 +- 7 files changed, 14171 insertions(+), 15479 deletions(-) create mode 100644 Python/generated_tracer_cases.c.h diff --git a/Makefile.pre.in b/Makefile.pre.in index a2a5f10585d27a..4182fdc46e85e0 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2139,7 +2139,7 @@ Objects/mimalloc/page.o: $(srcdir)/Objects/mimalloc/page-queue.c regen-cases: \ regen-opcode-ids regen-opcode-targets regen-uop-ids regen-opcode-metadata-py \ regen-generated-cases regen-executor-cases regen-optimizer-cases \ - regen-opcode-metadata regen-uop-metadata + regen-opcode-metadata regen-uop-metadata regen-tracer-cases .PHONY: regen-opcode-ids regen-opcode-ids: @@ -2185,6 +2185,13 @@ regen-optimizer-cases: $(srcdir)/Python/bytecodes.c $(UPDATE_FILE) $(srcdir)/Python/optimizer_cases.c.h $(srcdir)/Python/optimizer_cases.c.h.new +.PHONY: regen-tracer-cases +regen-tracer-cases: + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/tracer_generator.py \ + -o $(srcdir)/Python/generated_tracer_cases.c.h.new $(srcdir)/Python/bytecodes.c + $(UPDATE_FILE) $(srcdir)/Python/generated_tracer_cases.c.h $(srcdir)/Python/generated_tracer_cases.c.h.new + + .PHONY: regen-opcode-metadata regen-opcode-metadata: $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/opcode_metadata_generator.py \ diff --git a/Python/bytecodes.c b/Python/bytecodes.c index a4a75008c66cdf..5b47782ea60087 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2964,13 +2964,6 @@ dummy_func( if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD_JIT && next_instr->op.code != ENTER_EXECUTOR) { - _Py_CODEUNIT *start = this_instr; - /* Back up over EXTENDED_ARGs so optimizer sees the whole instruction */ - int curr_oparg = oparg; - while (curr_oparg > 255) { - curr_oparg >>= 8; - start--; - } if (tstate->interp->jit_tracer_code_buffer == NULL) { tstate->interp->jit_tracer_code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); if (tstate->interp->jit_tracer_code_buffer == NULL) { @@ -3046,9 +3039,9 @@ dummy_func( } assert(executor != tstate->interp->cold_executor); tstate->jit_exit = NULL; - if (IS_JIT_TRACING()) { - RECORD_TRACE_NO_DISPATCH(); - } + #if TRACING_JIT + RECORD_TRACE_NO_DISPATCH(); + #endif TIER1_TO_TIER2(executor); #else Py_FatalError("ENTER_EXECUTOR is not supported in this build"); diff --git a/Python/ceval.c b/Python/ceval.c index 844ca4297f4157..560c0eb364c939 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -998,6 +998,7 @@ add_to_code_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, PyCodeObjec #if _Py_TAIL_CALL_INTERP #include "opcode_targets.h" #include "generated_cases.c.h" +#include "generated_tracer_cases.c.h" #endif #if (defined(__GNUC__) && __GNUC__ >= 10 && !defined(__clang__)) && defined(__x86_64__) @@ -1126,6 +1127,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #else goto start_frame; # include "generated_cases.c.h" + #include "generated_tracer_cases.c.h" #endif diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 95b966cc3aeb80..c47b6bec7b27ce 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -29,12 +29,6 @@ PREDICTED_BINARY_OP:; _Py_CODEUNIT* const this_instr = next_instr - 6; (void)this_instr; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef lhs; _PyStackRef rhs; _PyStackRef res; @@ -97,12 +91,6 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef left; @@ -161,12 +149,6 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_INT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef left; @@ -227,12 +209,6 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; @@ -293,12 +269,6 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_EXTEND); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; _PyStackRef right; @@ -363,12 +333,6 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; @@ -452,12 +416,6 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef left; @@ -516,12 +474,6 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef left; @@ -582,12 +534,6 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBSCR_DICT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef nos; _PyStackRef dict_st; @@ -654,12 +600,6 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBSCR_GETITEM); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef container; _PyStackRef getitem; @@ -744,12 +684,6 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_INT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; @@ -840,12 +774,6 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_SLICE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef tos; _PyStackRef nos; @@ -918,12 +846,6 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBSCR_STR_INT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; @@ -1002,12 +924,6 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBSCR_TUPLE_INT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; @@ -1081,12 +997,6 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef left; @@ -1145,12 +1055,6 @@ frame->instr_ptr = next_instr; next_instr += 6; INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef left; @@ -1209,12 +1113,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BINARY_SLICE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef container; _PyStackRef start; _PyStackRef stop; @@ -1271,12 +1169,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_INTERPOLATION); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; _PyStackRef str; _PyStackRef *format; @@ -1335,12 +1227,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_LIST); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef *values; _PyStackRef list; values = &stack_pointer[-oparg]; @@ -1365,12 +1251,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_MAP); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef *values; _PyStackRef map; values = &stack_pointer[-oparg*2]; @@ -1423,12 +1303,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_SET); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef *values; _PyStackRef set; values = &stack_pointer[-oparg]; @@ -1486,12 +1360,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_SLICE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef *args; _PyStackRef slice; args = &stack_pointer[-oparg]; @@ -1527,12 +1395,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_STRING); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef *pieces; _PyStackRef str; pieces = &stack_pointer[-oparg]; @@ -1580,12 +1442,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_TEMPLATE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef strings; _PyStackRef interpolations; _PyStackRef template; @@ -1624,12 +1480,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(BUILD_TUPLE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef *values; _PyStackRef tup; values = &stack_pointer[-oparg]; @@ -1652,12 +1502,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CACHE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; assert(0 && "Executing a cache."); Py_FatalError("Executing a cache."); DISPATCH(); @@ -1674,12 +1518,6 @@ PREDICTED_CALL:; _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; opcode = CALL; _PyStackRef callable; _PyStackRef self_or_null; @@ -1857,12 +1695,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -1984,12 +1816,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef null; @@ -2133,12 +1959,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef null; @@ -2267,12 +2087,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_CLASS); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -2378,12 +2192,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_FAST); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -2493,12 +2301,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -2608,12 +2410,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_BUILTIN_O); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -2698,12 +2494,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CALL_FUNCTION_EX); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; opcode = CALL_FUNCTION_EX; _PyStackRef func; _PyStackRef callargs; @@ -2869,12 +2659,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CALL_INTRINSIC_1); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; @@ -2905,12 +2689,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CALL_INTRINSIC_2); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value2_st; _PyStackRef value1_st; _PyStackRef res; @@ -2952,12 +2730,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_ISINSTANCE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef null; _PyStackRef callable; @@ -3035,12 +2807,6 @@ PREDICTED_CALL_KW:; _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; opcode = CALL_KW; _PyStackRef callable; _PyStackRef self_or_null; @@ -3222,12 +2988,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef null; @@ -3358,12 +3118,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW_NON_PY); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; opcode = CALL_KW_NON_PY; static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); _PyStackRef callable; @@ -3491,12 +3245,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_KW_PY); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -3607,12 +3355,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_LEN); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef null; _PyStackRef callable; @@ -3685,12 +3427,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_LIST_APPEND); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef nos; @@ -3779,12 +3515,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -3906,12 +3636,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -4035,12 +3759,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -4134,12 +3852,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -4246,12 +3958,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_NON_PY_GENERAL); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; opcode = CALL_NON_PY_GENERAL; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; @@ -4367,12 +4073,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -4486,12 +4186,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_PY_GENERAL); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -4592,12 +4286,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_STR_1); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef null; _PyStackRef callable; @@ -4670,12 +4358,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_TUPLE_1); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef null; _PyStackRef callable; @@ -4748,12 +4430,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(CALL_TYPE_1); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef null; _PyStackRef callable; @@ -4807,12 +4483,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CHECK_EG_MATCH); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef exc_value_st; _PyStackRef match_type_st; _PyStackRef rest; @@ -4884,12 +4554,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CHECK_EXC_MATCH); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef left; _PyStackRef right; _PyStackRef b; @@ -4929,12 +4593,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CLEANUP_THROW); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef sub_iter; _PyStackRef last_sent_val; _PyStackRef exc_value_st; @@ -4995,12 +4653,6 @@ PREDICTED_COMPARE_OP:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef left; _PyStackRef right; _PyStackRef res; @@ -5073,12 +4725,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_FLOAT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); _PyStackRef value; _PyStackRef left; @@ -5134,12 +4780,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_INT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); _PyStackRef value; _PyStackRef left; @@ -5199,12 +4839,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(COMPARE_OP_STR); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; @@ -5265,12 +4899,6 @@ PREDICTED_CONTAINS_OP:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef right; _PyStackRef left; _PyStackRef b; @@ -5330,12 +4958,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP_DICT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); _PyStackRef tos; _PyStackRef left; @@ -5394,12 +5016,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(CONTAINS_OP_SET); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); _PyStackRef tos; _PyStackRef left; @@ -5456,12 +5072,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(CONVERT_VALUE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; _PyStackRef result; value = stack_pointer[-1]; @@ -5494,12 +5104,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(COPY); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef bottom; _PyStackRef top; bottom = stack_pointer[-1 - (oparg-1)]; @@ -5518,12 +5122,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(COPY_FREE_VARS); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; PyCodeObject *co = _PyFrame_GetCode(frame); assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -5545,12 +5143,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_ATTR); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef owner; owner = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -5576,12 +5168,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_DEREF); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); if (oldobj == NULL) { @@ -5604,12 +5190,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_FAST); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef v = GETLOCAL(oparg); if (PyStackRef_IsNull(v)) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -5636,12 +5216,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_GLOBAL); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyDict_Pop(GLOBALS(), name, NULL); @@ -5667,12 +5241,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_NAME); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *ns = LOCALS(); int err; @@ -5705,12 +5273,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DELETE_SUBSCR); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef container; _PyStackRef sub; sub = stack_pointer[-1]; @@ -5743,12 +5305,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DICT_MERGE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef callable; _PyStackRef dict; _PyStackRef update; @@ -5788,12 +5344,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(DICT_UPDATE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef dict; _PyStackRef update; update = stack_pointer[-1]; @@ -5839,12 +5389,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(END_ASYNC_FOR); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef awaitable_st; _PyStackRef exc_st; exc_st = stack_pointer[-1]; @@ -5887,12 +5431,6 @@ #endif next_instr += 1; INSTRUCTION_STATS(END_FOR); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; value = stack_pointer[-1]; stack_pointer += -1; @@ -5911,12 +5449,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(END_SEND); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef receiver; _PyStackRef value; _PyStackRef val; @@ -5942,12 +5474,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(ENTER_EXECUTOR); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; opcode = ENTER_EXECUTOR; #ifdef _Py_TIER2 PyCodeObject *code = _PyFrame_GetCode(frame); @@ -5967,9 +5493,9 @@ } assert(executor != tstate->interp->cold_executor); tstate->jit_exit = NULL; - if (IS_JIT_TRACING()) { - RECORD_TRACE_NO_DISPATCH(); - } + #if TRACING_JIT + RECORD_TRACE_NO_DISPATCH(); + #endif TIER1_TO_TIER2(executor); #else Py_FatalError("ENTER_EXECUTOR is not supported in this build"); @@ -5985,12 +5511,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(EXIT_INIT_CHECK); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef should_be_none; should_be_none = stack_pointer[-1]; if (!PyStackRef_IsNone(should_be_none)) { @@ -6014,12 +5534,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(EXTENDED_ARG); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; opcode = EXTENDED_ARG; assert(oparg); opcode = next_instr->op.code; @@ -6036,12 +5550,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(FORMAT_SIMPLE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; @@ -6078,12 +5586,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(FORMAT_WITH_SPEC); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; _PyStackRef fmt_spec; _PyStackRef res; @@ -6123,12 +5625,6 @@ PREDICTED_FOR_ITER:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef iter; _PyStackRef null_or_index; _PyStackRef next; @@ -6182,12 +5678,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_GEN); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; _PyStackRef gen_frame; @@ -6259,12 +5749,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_LIST); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; _PyStackRef null_or_index; @@ -6349,12 +5833,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_RANGE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; _PyStackRef next; @@ -6422,12 +5900,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(FOR_ITER_TUPLE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; _PyStackRef null_or_index; @@ -6482,12 +5954,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_AITER); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef obj; _PyStackRef iter; obj = stack_pointer[-1]; @@ -6549,12 +6015,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_ANEXT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef aiter; _PyStackRef awaitable; aiter = stack_pointer[-1]; @@ -6579,12 +6039,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_AWAITABLE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef iterable; _PyStackRef iter; iterable = stack_pointer[-1]; @@ -6614,12 +6068,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_ITER); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef iterable; _PyStackRef iter; _PyStackRef index_or_null; @@ -6666,12 +6114,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_LEN); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef obj; _PyStackRef len; obj = stack_pointer[-1]; @@ -6700,12 +6142,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(GET_YIELD_FROM_ITER); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef iterable; _PyStackRef iter; iterable = stack_pointer[-1]; @@ -6751,12 +6187,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IMPORT_FROM); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef from; _PyStackRef res; from = stack_pointer[-1]; @@ -6782,12 +6212,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IMPORT_NAME); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef level; _PyStackRef fromlist; _PyStackRef res; @@ -6829,12 +6253,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(INSTRUMENTED_CALL); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; opcode = INSTRUMENTED_CALL; _PyStackRef callable; _PyStackRef self_or_null; @@ -7023,12 +6441,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; opcode = INSTRUMENTED_CALL_FUNCTION_EX; _PyStackRef func; _PyStackRef callargs; @@ -7196,12 +6608,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; opcode = INSTRUMENTED_CALL_KW; _PyStackRef callable; _PyStackRef self_or_null; @@ -7388,12 +6794,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_END_ASYNC_FOR); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef awaitable_st; _PyStackRef exc_st; // _MONITOR_END_ASYNC_FOR @@ -7446,12 +6846,6 @@ (void)this_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_END_FOR); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef receiver; _PyStackRef value; value = stack_pointer[-1]; @@ -7482,12 +6876,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_END_SEND); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef receiver; _PyStackRef value; _PyStackRef val; @@ -7522,12 +6910,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef iter; _PyStackRef null_or_index; _PyStackRef next; @@ -7564,12 +6946,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; opcode = INSTRUMENTED_INSTRUCTION; _PyFrame_SetStackPointer(frame, stack_pointer); int next_opcode = _Py_call_instrumentation_instruction( @@ -7597,12 +6973,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; /* Skip 1 cache entry */ // _CHECK_PERIODIC { @@ -7630,12 +7000,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); DISPATCH(); } @@ -7651,12 +7015,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_LINE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; opcode = INSTRUMENTED_LINE; int original_opcode = 0; if (tstate->tracing) { @@ -7696,12 +7054,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; opcode = INSTRUMENTED_LOAD_SUPER_ATTR; _PyStackRef global_super_st; _PyStackRef class_st; @@ -7823,12 +7175,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_NOT_TAKEN); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; (void)this_instr; INSTRUMENTED_JUMP(prev_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); DISPATCH(); @@ -7845,12 +7191,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_POP_ITER); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef iter; _PyStackRef index_or_null; index_or_null = stack_pointer[-1]; @@ -7875,12 +7215,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef cond; /* Skip 1 cache entry */ cond = stack_pointer[-1]; @@ -7905,12 +7239,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; /* Skip 1 cache entry */ value = stack_pointer[-1]; @@ -7942,12 +7270,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; /* Skip 1 cache entry */ value = stack_pointer[-1]; @@ -7977,12 +7299,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef cond; /* Skip 1 cache entry */ cond = stack_pointer[-1]; @@ -8007,12 +7323,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_RESUME); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; // _LOAD_BYTECODE { #ifdef Py_GIL_DISABLED @@ -8093,12 +7403,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef val; _PyStackRef retval; _PyStackRef res; @@ -8151,12 +7455,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef val; _PyStackRef retval; _PyStackRef value; @@ -8226,12 +7524,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INTERPRETER_EXIT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef retval; retval = stack_pointer[-1]; assert(frame->owner == FRAME_OWNED_BY_INTERPRETER); @@ -8267,12 +7559,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(IS_OP); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef left; _PyStackRef right; _PyStackRef b; @@ -8309,12 +7595,6 @@ PREDICTED_JUMP_BACKWARD:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; /* Skip 1 cache entry */ // _SPECIALIZE_JUMP_BACKWARD { @@ -8355,12 +7635,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(JUMP_BACKWARD_JIT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(1 == 1, "incorrect cache size"); /* Skip 1 cache entry */ // _CHECK_PERIODIC @@ -8386,12 +7660,6 @@ if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD_JIT && next_instr->op.code != ENTER_EXECUTOR) { - _Py_CODEUNIT *start = this_instr; - int curr_oparg = oparg; - while (curr_oparg > 255) { - curr_oparg >>= 8; - start--; - } if (tstate->interp->jit_tracer_code_buffer == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); tstate->interp->jit_tracer_code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); @@ -8420,12 +7688,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; #if TIER_ONE assert(oparg <= INSTR_OFFSET()); #endif @@ -8441,12 +7703,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(JUMP_BACKWARD_NO_JIT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(1 == 1, "incorrect cache size"); /* Skip 1 cache entry */ // _CHECK_PERIODIC @@ -8476,12 +7732,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(JUMP_FORWARD); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; JUMPBY(oparg); DISPATCH(); } @@ -8494,12 +7744,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LIST_APPEND); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef list; _PyStackRef v; v = stack_pointer[-1]; @@ -8522,12 +7766,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LIST_EXTEND); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef list_st; _PyStackRef iterable_st; iterable_st = stack_pointer[-1]; @@ -8578,12 +7816,6 @@ PREDICTED_LOAD_ATTR:; _Py_CODEUNIT* const this_instr = next_instr - 10; (void)this_instr; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef owner; _PyStackRef *attr; _PyStackRef *self_or_null; @@ -8664,12 +7896,6 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_CLASS); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -8728,12 +7954,6 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -8802,12 +8022,6 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; /* Skip 1 cache entry */ @@ -8866,12 +8080,6 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -8952,12 +8160,6 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -9014,12 +8216,6 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -9066,12 +8262,6 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -9139,12 +8329,6 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_MODULE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -9219,12 +8403,6 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -9272,12 +8450,6 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -9346,12 +8518,6 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef new_frame; @@ -9446,12 +8612,6 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_SLOT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -9521,12 +8681,6 @@ frame->instr_ptr = next_instr; next_instr += 10; INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -9635,12 +8789,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_BUILD_CLASS); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef bc; PyObject *bc_o; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -9671,12 +8819,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; assert(oparg < NUM_COMMON_CONSTANTS); value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); @@ -9694,12 +8836,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_CONST); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); value = PyStackRef_FromPyObjectBorrow(obj); @@ -9717,12 +8853,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_DEREF); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -9751,12 +8881,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_DUP(GETLOCAL(oparg)); @@ -9774,12 +8898,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; value = GETLOCAL(oparg); GETLOCAL(oparg) = PyStackRef_NULL; @@ -9797,12 +8915,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_BORROW); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_Borrow(GETLOCAL(oparg)); @@ -9820,12 +8932,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_BORROW_LOAD_FAST_BORROW); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value1; _PyStackRef value2; uint32_t oparg1 = oparg >> 4; @@ -9847,12 +8953,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_CHECK); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; _PyStackRef value_s = GETLOCAL(oparg); if (PyStackRef_IsNull(value_s)) { @@ -9879,12 +8979,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value1; _PyStackRef value2; uint32_t oparg1 = oparg >> 4; @@ -9906,12 +9000,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef class_dict_st; _PyStackRef value; class_dict_st = stack_pointer[-1]; @@ -9957,12 +9045,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef mod_or_class_dict; _PyStackRef v; mod_or_class_dict = stack_pointer[-1]; @@ -10041,12 +9123,6 @@ PREDICTED_LOAD_GLOBAL:; _Py_CODEUNIT* const this_instr = next_instr - 5; (void)this_instr; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef *res; _PyStackRef *null; // _SPECIALIZE_LOAD_GLOBAL @@ -10102,12 +9178,6 @@ frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); _PyStackRef res; _PyStackRef *null; @@ -10188,12 +9258,6 @@ frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); _PyStackRef res; _PyStackRef *null; @@ -10259,12 +9323,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_LOCALS); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef locals; PyObject *l = LOCALS(); if (l == NULL) { @@ -10289,12 +9347,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_NAME); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef v; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -10318,12 +9370,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_SMALL_INT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; assert(oparg < _PY_NSMALLPOSINTS); PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; @@ -10342,12 +9388,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(LOAD_SPECIAL); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef self; _PyStackRef *method_and_self; // _INSERT_NULL @@ -10397,12 +9437,6 @@ PREDICTED_LOAD_SUPER_ATTR:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; opcode = LOAD_SUPER_ATTR; _PyStackRef global_super_st; _PyStackRef class_st; @@ -10539,12 +9573,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); _PyStackRef global_super_st; _PyStackRef class_st; @@ -10607,12 +9635,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); _PyStackRef global_super_st; _PyStackRef class_st; @@ -10689,12 +9711,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAKE_CELL); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); PyObject *cell = PyCell_New(initial); if (cell == NULL) { @@ -10716,12 +9732,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAKE_FUNCTION); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef codeobj_st; _PyStackRef func; codeobj_st = stack_pointer[-1]; @@ -10755,12 +9765,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MAP_ADD); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef dict_st; _PyStackRef key; _PyStackRef value; @@ -10792,12 +9796,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_CLASS); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef subject; _PyStackRef type; _PyStackRef names; @@ -10850,12 +9848,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_KEYS); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef subject; _PyStackRef keys; _PyStackRef values_or_none; @@ -10883,12 +9875,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_MAPPING); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef subject; _PyStackRef res; subject = stack_pointer[-1]; @@ -10908,12 +9894,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(MATCH_SEQUENCE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef subject; _PyStackRef res; subject = stack_pointer[-1]; @@ -10933,12 +9913,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(NOP); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; DISPATCH(); } @@ -10950,12 +9924,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(NOT_TAKEN); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; DISPATCH(); } @@ -10967,12 +9935,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_EXCEPT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef exc_value; exc_value = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; @@ -10994,12 +9956,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_ITER); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef iter; _PyStackRef index_or_null; index_or_null = stack_pointer[-1]; @@ -11021,12 +9977,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_FALSE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef cond; /* Skip 1 cache entry */ cond = stack_pointer[-1]; @@ -11046,12 +9996,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_NONE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; _PyStackRef b; _PyStackRef cond; @@ -11092,12 +10036,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; _PyStackRef b; _PyStackRef cond; @@ -11138,12 +10076,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_TRUE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef cond; /* Skip 1 cache entry */ cond = stack_pointer[-1]; @@ -11163,12 +10095,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(POP_TOP); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; value = stack_pointer[-1]; stack_pointer += -1; @@ -11187,12 +10113,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(PUSH_EXC_INFO); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef exc; _PyStackRef prev_exc; _PyStackRef new_exc; @@ -11222,12 +10142,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(PUSH_NULL); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef res; res = PyStackRef_NULL; stack_pointer[0] = res; @@ -11246,12 +10160,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RAISE_VARARGS); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef *args; args = &stack_pointer[-oparg]; assert(oparg < 3); @@ -11281,12 +10189,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RERAISE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef *values; _PyStackRef exc_st; exc_st = stack_pointer[-1]; @@ -11313,12 +10215,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESERVED); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; assert(0 && "Executing RESERVED instruction."); Py_FatalError("Executing RESERVED instruction."); DISPATCH(); @@ -11335,12 +10231,6 @@ PREDICTED_RESUME:; _Py_CODEUNIT* const this_instr = next_instr - 1; (void)this_instr; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; // _LOAD_BYTECODE { #ifdef Py_GIL_DISABLED @@ -11416,12 +10306,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RESUME_CHECK); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(0 == 0, "incorrect cache size"); #if defined(__EMSCRIPTEN__) if (_Py_emscripten_signal_clock == 0) { @@ -11458,12 +10342,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RETURN_GENERATOR); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef res; assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -11506,12 +10384,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(RETURN_VALUE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef retval; _PyStackRef res; retval = stack_pointer[-1]; @@ -11549,12 +10421,6 @@ PREDICTED_SEND:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef receiver; _PyStackRef v; _PyStackRef retval; @@ -11659,12 +10525,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(SEND_GEN); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); _PyStackRef receiver; _PyStackRef v; @@ -11732,12 +10592,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SETUP_ANNOTATIONS); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; PyObject *ann_dict; if (LOCALS() == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -11784,12 +10638,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_ADD); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef set; _PyStackRef v; v = stack_pointer[-1]; @@ -11814,12 +10662,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef attr_st; _PyStackRef func_in; _PyStackRef func_out; @@ -11848,12 +10690,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SET_UPDATE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef set; _PyStackRef iterable; iterable = stack_pointer[-1]; @@ -11884,12 +10720,6 @@ PREDICTED_STORE_ATTR:; _Py_CODEUNIT* const this_instr = next_instr - 5; (void)this_instr; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef owner; _PyStackRef v; // _SPECIALIZE_STORE_ATTR @@ -11946,12 +10776,6 @@ frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); _PyStackRef owner; _PyStackRef value; @@ -12028,12 +10852,6 @@ frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_SLOT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); _PyStackRef owner; _PyStackRef value; @@ -12085,12 +10903,6 @@ frame->instr_ptr = next_instr; next_instr += 5; INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); _PyStackRef owner; _PyStackRef value; @@ -12177,12 +10989,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_DEREF); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef v; v = stack_pointer[-1]; PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); @@ -12202,12 +11008,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_FAST); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; value = stack_pointer[-1]; _PyStackRef tmp = GETLOCAL(oparg); @@ -12228,12 +11028,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value1; _PyStackRef value2; value1 = stack_pointer[-1]; @@ -12257,12 +11051,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_FAST_STORE_FAST); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value2; _PyStackRef value1; value1 = stack_pointer[-1]; @@ -12294,12 +11082,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_GLOBAL); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef v; v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -12325,12 +11107,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_NAME); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef v; v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -12377,12 +11153,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(STORE_SLICE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef v; _PyStackRef container; _PyStackRef start; @@ -12446,12 +11216,6 @@ PREDICTED_STORE_SUBSCR:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef container; _PyStackRef sub; _PyStackRef v; @@ -12510,12 +11274,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR_DICT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); _PyStackRef nos; _PyStackRef value; @@ -12567,12 +11325,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; @@ -12652,12 +11404,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(SWAP); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef bottom; _PyStackRef top; top = stack_pointer[-1]; @@ -12681,12 +11427,6 @@ PREDICTED_TO_BOOL:; _Py_CODEUNIT* const this_instr = next_instr - 4; (void)this_instr; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; _PyStackRef res; // _SPECIALIZE_TO_BOOL @@ -12738,12 +11478,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef owner; _PyStackRef value; @@ -12787,12 +11521,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_BOOL); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef value; /* Skip 1 cache entry */ @@ -12817,12 +11545,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_INT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef value; _PyStackRef res; @@ -12863,12 +11585,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_LIST); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef tos; _PyStackRef value; @@ -12912,12 +11628,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_NONE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef value; _PyStackRef res; @@ -12945,12 +11655,6 @@ frame->instr_ptr = next_instr; next_instr += 4; INSTRUCTION_STATS(TO_BOOL_STR); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef value; _PyStackRef res; @@ -12997,12 +11701,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNARY_INVERT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; @@ -13032,12 +11730,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNARY_NEGATIVE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; @@ -13067,12 +11759,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNARY_NOT); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; @@ -13091,12 +11777,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(UNPACK_EX); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef seq; _PyStackRef *top; seq = stack_pointer[-1]; @@ -13127,12 +11807,6 @@ PREDICTED_UNPACK_SEQUENCE:; _Py_CODEUNIT* const this_instr = next_instr - 2; (void)this_instr; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef seq; _PyStackRef *top; // _SPECIALIZE_UNPACK_SEQUENCE @@ -13183,12 +11857,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); _PyStackRef tos; _PyStackRef seq; @@ -13248,12 +11916,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); _PyStackRef tos; _PyStackRef seq; @@ -13304,12 +11966,6 @@ frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); _PyStackRef tos; _PyStackRef seq; @@ -13359,12 +12015,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(WITH_EXCEPT_START); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef exit_func; _PyStackRef exit_self; _PyStackRef lasti; @@ -13410,12 +12060,6 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(YIELD_VALUE); - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; _PyStackRef retval; _PyStackRef value; retval = stack_pointer[-1]; @@ -13456,14104 +12100,6 @@ assert(WITHIN_STACK_BOUNDS()); DISPATCH(); } - #ifdef _Py_TIER2 /* BEGIN TRACING INSTRUCTIONS */ - - - TRACING_TARGET(BINARY_OP) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP); - PREDICTED_TRACING_BINARY_OP:; - _Py_CODEUNIT* const this_instr = next_instr - 6; - (void)this_instr; - opcode = BINARY_OP; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef lhs; - _PyStackRef rhs; - _PyStackRef res; - // _SPECIALIZE_BINARY_OP - { - rhs = stack_pointer[-1]; - lhs = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(BINARY_OP); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - assert(NB_ADD <= oparg); - assert(oparg <= NB_OPARG_LAST); - } - /* Skip 4 cache entries */ - // _BINARY_OP - { - PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); - PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); - assert(_PyEval_BinaryOps[oparg]); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = lhs; - lhs = res; - stack_pointer[-2] = lhs; - PyStackRef_CLOSE(tmp); - tmp = rhs; - rhs = PyStackRef_NULL; - stack_pointer[-1] = rhs; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_ADD_FLOAT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_ADD_FLOAT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); - opcode = BINARY_OP_ADD_FLOAT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_TOS_FLOAT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyFloat_CheckExact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_FLOAT - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!PyFloat_CheckExact(left_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_ADD_FLOAT - { - right = value; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval + - ((PyFloatObject *)right_o)->ob_fval; - res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (PyStackRef_IsNull(res)) { - TRACING_JUMP_TO_LABEL(pop_2_error); - } - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_ADD_INT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_ADD_INT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_ADD_INT); - opcode = BINARY_OP_ADD_INT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_TOS_INT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_INT - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!_PyLong_CheckExactAndCompact(left_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_ADD_INT - { - right = value; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); - STAT_INC(BINARY_OP, hit); - res = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); - if (PyStackRef_IsNull(res)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_ADD_UNICODE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_ADD_UNICODE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); - opcode = BINARY_OP_ADD_UNICODE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef nos; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_TOS_UNICODE - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyUnicode_CheckExact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_UNICODE - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyUnicode_CheckExact(o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_ADD_UNICODE - { - right = value; - left = nos; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyUnicode_CheckExact(left_o)); - assert(PyUnicode_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - PyObject *res_o = PyUnicode_Concat(left_o, right_o); - PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(pop_2_error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_EXTEND) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_EXTEND; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_EXTEND); - opcode = BINARY_OP_EXTEND; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - /* Skip 1 cache entry */ - // _GUARD_BINARY_OP_EXTEND - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *descr = read_obj(&this_instr[2].cache); - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; - assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); - assert(d && d->guard); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = d->guard(left_o, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!res) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip -4 cache entry */ - // _BINARY_OP_EXTEND - { - PyObject *descr = read_obj(&this_instr[2].cache); - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); - _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; - STAT_INC(BINARY_OP, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = d->action(left_o, right_o); - _PyStackRef tmp = right; - right = PyStackRef_NULL; - stack_pointer[-1] = right; - PyStackRef_CLOSE(tmp); - tmp = left; - left = PyStackRef_NULL; - stack_pointer[-2] = left; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_INPLACE_ADD_UNICODE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); - opcode = BINARY_OP_INPLACE_ADD_UNICODE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef nos; - _PyStackRef left; - _PyStackRef right; - // _GUARD_TOS_UNICODE - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyUnicode_CheckExact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_UNICODE - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyUnicode_CheckExact(o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_INPLACE_ADD_UNICODE - { - right = value; - left = nos; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - assert(PyUnicode_CheckExact(left_o)); - assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(right))); - int next_oparg; - #if TIER_ONE - assert(next_instr->op.code == STORE_FAST); - next_oparg = next_instr->op.arg; - #else - next_oparg = (int)CURRENT_OPERAND0(); - #endif - _PyStackRef *target_local = &GETLOCAL(next_oparg); - assert(PyUnicode_CheckExact(left_o)); - if (PyStackRef_AsPyObjectBorrow(*target_local) != left_o) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - STAT_INC(BINARY_OP, hit); - assert(Py_REFCNT(left_o) >= 2 || !PyStackRef_IsHeapSafe(left)); - PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); - PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); - PyObject *right_o = PyStackRef_AsPyObjectSteal(right); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyUnicode_Append(&temp, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - *target_local = PyStackRef_FromPyObjectSteal(temp); - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(*target_local)) { - TRACING_JUMP_TO_LABEL(error); - } - #if TIER_ONE - - assert(next_instr->op.code == STORE_FAST); - SKIP_OVER(1); - #endif - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_MULTIPLY_FLOAT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_MULTIPLY_FLOAT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); - opcode = BINARY_OP_MULTIPLY_FLOAT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_TOS_FLOAT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyFloat_CheckExact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_FLOAT - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!PyFloat_CheckExact(left_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_MULTIPLY_FLOAT - { - right = value; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval * - ((PyFloatObject *)right_o)->ob_fval; - res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (PyStackRef_IsNull(res)) { - TRACING_JUMP_TO_LABEL(pop_2_error); - } - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_MULTIPLY_INT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_MULTIPLY_INT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); - opcode = BINARY_OP_MULTIPLY_INT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_TOS_INT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_INT - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!_PyLong_CheckExactAndCompact(left_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_MULTIPLY_INT - { - right = value; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); - STAT_INC(BINARY_OP, hit); - res = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); - if (PyStackRef_IsNull(res)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_SUBSCR_DICT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_SUBSCR_DICT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBSCR_DICT); - opcode = BINARY_OP_SUBSCR_DICT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef nos; - _PyStackRef dict_st; - _PyStackRef sub_st; - _PyStackRef res; - // _GUARD_NOS_DICT - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyDict_CheckExact(o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_SUBSCR_DICT - { - sub_st = stack_pointer[-1]; - dict_st = nos; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - assert(PyDict_CheckExact(dict)); - STAT_INC(BINARY_OP, hit); - PyObject *res_o; - _PyFrame_SetStackPointer(frame, stack_pointer); - int rc = PyDict_GetItemRef(dict, sub, &res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (rc == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetKeyError(sub); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = sub_st; - sub_st = PyStackRef_NULL; - stack_pointer[-1] = sub_st; - PyStackRef_CLOSE(tmp); - tmp = dict_st; - dict_st = PyStackRef_NULL; - stack_pointer[-2] = dict_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (rc <= 0) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_SUBSCR_GETITEM) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_SUBSCR_GETITEM; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBSCR_GETITEM); - opcode = BINARY_OP_SUBSCR_GETITEM; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef container; - _PyStackRef getitem; - _PyStackRef sub; - _PyStackRef new_frame; - /* Skip 5 cache entries */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _BINARY_OP_SUBSCR_CHECK_FUNC - { - container = stack_pointer[-2]; - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); - if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; - PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); - if (getitem_o == NULL) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - assert(PyFunction_Check(getitem_o)); - uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); - if (((PyFunctionObject *)getitem_o)->func_version != cached_version) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); - assert(code->co_argcount == 2); - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - getitem = PyStackRef_FromPyObjectNew(getitem_o); - STAT_INC(BINARY_OP, hit); - } - // _BINARY_OP_SUBSCR_INIT_CALL - { - sub = stack_pointer[-1]; - _PyInterpreterFrame* pushed_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); - pushed_frame->localsplus[0] = container; - pushed_frame->localsplus[1] = sub; - frame->return_offset = 6u ; - new_frame = PyStackRef_Wrap(pushed_frame); - } - // _PUSH_FRAME - { - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_SUBSCR_LIST_INT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_SUBSCR_LIST_INT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_INT); - opcode = BINARY_OP_SUBSCR_LIST_INT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef nos; - _PyStackRef list_st; - _PyStackRef sub_st; - _PyStackRef res; - // _GUARD_TOS_INT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_LIST - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyList_CheckExact(o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_SUBSCR_LIST_INT - { - sub_st = value; - list_st = nos; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - assert(PyLong_CheckExact(sub)); - assert(PyList_CheckExact(list)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - #ifdef Py_GIL_DISABLED - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - STAT_INC(BINARY_OP, hit); - res = PyStackRef_FromPyObjectSteal(res_o); - #else - if (index >= PyList_GET_SIZE(list)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - STAT_INC(BINARY_OP, hit); - PyObject *res_o = PyList_GET_ITEM(list, index); - assert(res_o != NULL); - res = PyStackRef_FromPyObjectNew(res_o); - #endif - STAT_INC(BINARY_OP, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = list_st; - list_st = res; - stack_pointer[-2] = list_st; - PyStackRef_CLOSE(tmp); - tmp = sub_st; - sub_st = PyStackRef_NULL; - stack_pointer[-1] = sub_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_SUBSCR_LIST_SLICE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_SUBSCR_LIST_SLICE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_SLICE); - opcode = BINARY_OP_SUBSCR_LIST_SLICE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef tos; - _PyStackRef nos; - _PyStackRef list_st; - _PyStackRef sub_st; - _PyStackRef res; - // _GUARD_TOS_SLICE - { - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PySlice_Check(o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_LIST - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyList_CheckExact(o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_SUBSCR_LIST_SLICE - { - sub_st = tos; - list_st = nos; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - assert(PySlice_Check(sub)); - assert(PyList_CheckExact(list)); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyList_SliceSubscript(list, sub); - stack_pointer = _PyFrame_GetStackPointer(frame); - STAT_INC(BINARY_OP, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = sub_st; - sub_st = PyStackRef_NULL; - stack_pointer[-1] = sub_st; - PyStackRef_CLOSE(tmp); - tmp = list_st; - list_st = PyStackRef_NULL; - stack_pointer[-2] = list_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_SUBSCR_STR_INT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_SUBSCR_STR_INT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBSCR_STR_INT); - opcode = BINARY_OP_SUBSCR_STR_INT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef nos; - _PyStackRef str_st; - _PyStackRef sub_st; - _PyStackRef res; - // _GUARD_TOS_INT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_UNICODE - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyUnicode_CheckExact(o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_SUBSCR_STR_INT - { - sub_st = value; - str_st = nos; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); - assert(PyLong_CheckExact(sub)); - assert(PyUnicode_CheckExact(str)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (PyUnicode_GET_LENGTH(str) <= index) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - STAT_INC(BINARY_OP, hit); - PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(str_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_FromPyObjectBorrow(res_o); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_SUBSCR_TUPLE_INT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_SUBSCR_TUPLE_INT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBSCR_TUPLE_INT); - opcode = BINARY_OP_SUBSCR_TUPLE_INT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef nos; - _PyStackRef tuple_st; - _PyStackRef sub_st; - _PyStackRef res; - // _GUARD_TOS_INT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_TUPLE - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyTuple_CheckExact(o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_SUBSCR_TUPLE_INT - { - sub_st = value; - tuple_st = nos; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); - assert(PyLong_CheckExact(sub)); - assert(PyTuple_CheckExact(tuple)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (index >= PyTuple_GET_SIZE(tuple)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - STAT_INC(BINARY_OP, hit); - PyObject *res_o = PyTuple_GET_ITEM(tuple, index); - assert(res_o != NULL); - PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - res = PyStackRef_FromPyObjectNew(res_o); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = tuple_st; - tuple_st = res; - stack_pointer[-1] = tuple_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_SUBTRACT_FLOAT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_SUBTRACT_FLOAT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); - opcode = BINARY_OP_SUBTRACT_FLOAT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_TOS_FLOAT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyFloat_CheckExact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_FLOAT - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!PyFloat_CheckExact(left_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_SUBTRACT_FLOAT - { - right = value; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval - - ((PyFloatObject *)right_o)->ob_fval; - res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (PyStackRef_IsNull(res)) { - TRACING_JUMP_TO_LABEL(pop_2_error); - } - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_SUBTRACT_INT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_SUBTRACT_INT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); - opcode = BINARY_OP_SUBTRACT_INT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_TOS_INT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_INT - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!_PyLong_CheckExactAndCompact(left_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_SUBTRACT_INT - { - right = value; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); - STAT_INC(BINARY_OP, hit); - res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - if (PyStackRef_IsNull(res)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_SLICE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_SLICE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BINARY_SLICE); - opcode = BINARY_SLICE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef container; - _PyStackRef start; - _PyStackRef stop; - _PyStackRef res; - // _SPECIALIZE_BINARY_SLICE - { - #if ENABLE_SPECIALIZATION - OPCODE_DEFERRED_INC(BINARY_SLICE); - #endif /* ENABLE_SPECIALIZATION */ - } - // _BINARY_SLICE - { - stop = stack_pointer[-1]; - start = stack_pointer[-2]; - container = stack_pointer[-3]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), - PyStackRef_AsPyObjectSteal(stop)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyObject *res_o; - if (slice == NULL) { - res_o = NULL; - } - else { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); - Py_DECREF(slice); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += 2; - } - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(container); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BUILD_INTERPOLATION) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BUILD_INTERPOLATION; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_INTERPOLATION); - opcode = BUILD_INTERPOLATION; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - _PyStackRef str; - _PyStackRef *format; - _PyStackRef interpolation; - format = &stack_pointer[-(oparg & 1)]; - str = stack_pointer[-1 - (oparg & 1)]; - value = stack_pointer[-2 - (oparg & 1)]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - PyObject *str_o = PyStackRef_AsPyObjectBorrow(str); - int conversion = oparg >> 2; - PyObject *format_o; - if (oparg & 1) { - format_o = PyStackRef_AsPyObjectBorrow(format[0]); - } - else { - format_o = &_Py_STR(empty); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *interpolation_o = _PyInterpolation_Build(value_o, str_o, conversion, format_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (oparg & 1) { - stack_pointer += -(oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(format[0]); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - stack_pointer += -(oparg & 1); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(str); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (interpolation_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - interpolation = PyStackRef_FromPyObjectSteal(interpolation_o); - stack_pointer[0] = interpolation; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BUILD_LIST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BUILD_LIST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_LIST); - opcode = BUILD_LIST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef *values; - _PyStackRef list; - values = &stack_pointer[-oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *list_o = _PyList_FromStackRefStealOnSuccess(values, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (list_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - list = PyStackRef_FromPyObjectStealMortal(list_o); - stack_pointer[-oparg] = list; - stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BUILD_MAP) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BUILD_MAP; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_MAP); - opcode = BUILD_MAP; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef *values; - _PyStackRef map; - values = &stack_pointer[-oparg*2]; - STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); - if (CONVERSION_FAILED(values_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg*2; --_i >= 0;) { - tmp = values[_i]; - values[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg*2; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *map_o = _PyDict_FromItems( - values_o, 2, - values_o+1, 2, - oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg*2; --_i >= 0;) { - tmp = values[_i]; - values[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg*2; - assert(WITHIN_STACK_BOUNDS()); - if (map_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - map = PyStackRef_FromPyObjectStealMortal(map_o); - stack_pointer[0] = map; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BUILD_SET) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BUILD_SET; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_SET); - opcode = BUILD_SET; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef *values; - _PyStackRef set; - values = &stack_pointer[-oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *set_o = PySet_New(NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (set_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = values[_i]; - values[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - int err = 0; - for (Py_ssize_t i = 0; i < oparg; i++) { - _PyStackRef value = values[i]; - values[i] = PyStackRef_NULL; - if (err == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - err = _PySet_AddTakeRef((PySetObject *)set_o, PyStackRef_AsPyObjectSteal(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - if (err) { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(set_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - set = PyStackRef_FromPyObjectStealMortal(set_o); - stack_pointer[-oparg] = set; - stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BUILD_SLICE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BUILD_SLICE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_SLICE); - opcode = BUILD_SLICE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef *args; - _PyStackRef slice; - args = &stack_pointer[-oparg]; - PyObject *start_o = PyStackRef_AsPyObjectBorrow(args[0]); - PyObject *stop_o = PyStackRef_AsPyObjectBorrow(args[1]); - PyObject *step_o = oparg == 3 ? PyStackRef_AsPyObjectBorrow(args[2]) : NULL; - PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - if (slice_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - slice = PyStackRef_FromPyObjectStealMortal(slice_o); - stack_pointer[0] = slice; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BUILD_STRING) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BUILD_STRING; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_STRING); - opcode = BUILD_STRING; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef *pieces; - _PyStackRef str; - pieces = &stack_pointer[-oparg]; - STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); - if (CONVERSION_FAILED(pieces_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = pieces[_i]; - pieces[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); - STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = pieces[_i]; - pieces[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - if (str_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - str = PyStackRef_FromPyObjectSteal(str_o); - stack_pointer[0] = str; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BUILD_TEMPLATE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BUILD_TEMPLATE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_TEMPLATE); - opcode = BUILD_TEMPLATE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef strings; - _PyStackRef interpolations; - _PyStackRef template; - interpolations = stack_pointer[-1]; - strings = stack_pointer[-2]; - PyObject *strings_o = PyStackRef_AsPyObjectBorrow(strings); - PyObject *interpolations_o = PyStackRef_AsPyObjectBorrow(interpolations); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *template_o = _PyTemplate_Build(strings_o, interpolations_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(interpolations); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(strings); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (template_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - template = PyStackRef_FromPyObjectSteal(template_o); - stack_pointer[0] = template; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BUILD_TUPLE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BUILD_TUPLE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_TUPLE); - opcode = BUILD_TUPLE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef *values; - _PyStackRef tup; - values = &stack_pointer[-oparg]; - PyObject *tup_o = _PyTuple_FromStackRefStealOnSuccess(values, oparg); - if (tup_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - tup = PyStackRef_FromPyObjectStealMortal(tup_o); - stack_pointer[-oparg] = tup; - stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CACHE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CACHE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CACHE); - opcode = CACHE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - assert(0 && "Executing a cache."); - Py_FatalError("Executing a cache."); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL); - PREDICTED_TRACING_CALL:; - _Py_CODEUNIT* const this_instr = next_instr - 4; - (void)this_instr; - opcode = CALL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - opcode = CALL; - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef res; - // _SPECIALIZE_CALL - { - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_Call(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(CALL); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 2 cache entries */ - // _MAYBE_EXPAND_METHOD - { - if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - self_or_null = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - _PyStackRef temp = callable; - callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - // _DO_CALL - { - args = &stack_pointer[-oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - if (Py_TYPE(callable_o) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) - { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, callable, locals, - arguments, total_args, NULL, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (new_frame == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - frame->return_offset = 4u ; - TRACING_DISPATCH_INLINED(new_frame); - } - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - if (opcode == INSTRUMENTED_CALL) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); - if (res_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - } - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_ALLOC_AND_ENTER_INIT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_ALLOC_AND_ENTER_INIT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); - opcode = CALL_ALLOC_AND_ENTER_INIT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef init; - _PyStackRef self; - _PyStackRef *args; - _PyStackRef init_frame; - _PyStackRef new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_AND_ALLOCATE_OBJECT - { - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (!PyStackRef_IsNull(self_or_null)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (!PyType_Check(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyTypeObject *tp = (PyTypeObject *)callable_o; - if (FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - assert(tp->tp_new == PyBaseObject_Type.tp_new); - assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); - assert(tp->tp_alloc == PyType_GenericAlloc); - PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; - PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); - PyCodeObject *code = (PyCodeObject *)init_func->func_code; - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - STAT_INC(CALL, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *self_o = PyType_GenericAlloc(tp, 0); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (self_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - self_or_null = PyStackRef_FromPyObjectSteal(self_o); - _PyStackRef temp = callable; - callable = PyStackRef_FromPyObjectNew(init_func); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - // _CREATE_INIT_FRAME - { - args = &stack_pointer[-oparg]; - self = self_or_null; - init = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( - tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); - assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); - assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); - shim->localsplus[0] = PyStackRef_DUP(self); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, init, NULL, args-1, oparg+1, NULL, shim); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FrameClearAndPop(tstate, shim); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; - tstate->py_recursion_remaining--; - init_frame = PyStackRef_Wrap(temp); - } - // _PUSH_FRAME - { - new_frame = init_frame; - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_BOUND_METHOD_EXACT_ARGS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); - opcode = CALL_BOUND_METHOD_EXACT_ARGS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef null; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS - { - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - if (!PyStackRef_IsNull(null)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _INIT_CALL_BOUND_METHOD_EXACT_ARGS - { - self_or_null = null; - assert(PyStackRef_IsNull(self_or_null)); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - STAT_INC(CALL, hit); - self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - _PyStackRef temp = callable; - callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - // flush - // _CHECK_FUNCTION_VERSION - { - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (!PyFunction_Check(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyFunctionObject *func = (PyFunctionObject *)callable_o; - if (func->func_version != func_version) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_FUNCTION_EXACT_ARGS - { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - assert(PyFunction_Check(callable_o)); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - PyCodeObject *code = (PyCodeObject *)func->func_code; - if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_STACK_SPACE - { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - PyCodeObject *code = (PyCodeObject *)func->func_code; - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_RECURSION_REMAINING - { - if (tstate->py_recursion_remaining <= 1) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _INIT_CALL_PY_EXACT_ARGS - { - args = &stack_pointer[-oparg]; - int has_self = !PyStackRef_IsNull(self_or_null); - STAT_INC(CALL, hit); - _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); - _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; - pushed_frame->localsplus[0] = self_or_null; - for (int i = 0; i < oparg; i++) { - first_non_self_local[i] = args[i]; - } - new_frame = PyStackRef_Wrap(pushed_frame); - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_BOUND_METHOD_GENERAL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_BOUND_METHOD_GENERAL; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); - opcode = CALL_BOUND_METHOD_GENERAL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef null; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_METHOD_VERSION - { - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (Py_TYPE(callable_o) != &PyMethod_Type) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyObject *func = ((PyMethodObject *)callable_o)->im_func; - if (!PyFunction_Check(func)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (((PyFunctionObject *)func)->func_version != func_version) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (!PyStackRef_IsNull(null)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _EXPAND_METHOD - { - self_or_null = null; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - assert(PyStackRef_IsNull(self_or_null)); - assert(Py_TYPE(callable_o) == &PyMethod_Type); - self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - _PyStackRef temp = callable; - callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - assert(PyStackRef_FunctionCheck(callable)); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - // flush - // _CHECK_RECURSION_REMAINING - { - if (tstate->py_recursion_remaining <= 1) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _PY_FRAME_GENERAL - { - args = &stack_pointer[-oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { - args--; - total_args++; - } - assert(Py_TYPE(callable_o) == &PyFunction_Type); - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, callable, locals, - args, total_args, NULL, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - new_frame = PyStackRef_Wrap(temp); - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_BUILTIN_CLASS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_BUILTIN_CLASS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_CLASS); - opcode = CALL_BUILTIN_CLASS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_BUILTIN_CLASS - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (!PyType_Check(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyTypeObject *tp = (PyTypeObject *)callable_o; - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - if (tp->tp_vectorcall == NULL) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - STAT_INC(CALL, hit); - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_BUILTIN_FAST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_BUILTIN_FAST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_FAST); - opcode = CALL_BUILTIN_FAST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_BUILTIN_FAST - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - if (!PyCFunction_CheckExact(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunctionFast_CAST(cfunc)( - PyCFunction_GET_SELF(callable_o), - args_o, - total_args); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_BUILTIN_FAST_WITH_KEYWORDS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); - opcode = CALL_BUILTIN_FAST_WITH_KEYWORDS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_BUILTIN_FAST_WITH_KEYWORDS - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - if (!PyCFunction_CheckExact(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - STAT_INC(CALL, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFastWithKeywords cfunc = - _PyCFunctionFastWithKeywords_CAST(PyCFunction_GET_FUNCTION(callable_o)); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_BUILTIN_O) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_BUILTIN_O; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_O); - opcode = CALL_BUILTIN_O; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_BUILTIN_O - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { - args--; - total_args++; - } - if (total_args != 1) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (!PyCFunction_CheckExact(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (_Py_ReachedRecursionLimit(tstate)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); - _PyStackRef arg = args[0]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg)); - stack_pointer = _PyFrame_GetStackPointer(frame); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_FUNCTION_EX) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_FUNCTION_EX; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_FUNCTION_EX); - opcode = CALL_FUNCTION_EX; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - opcode = CALL_FUNCTION_EX; - _PyStackRef func; - _PyStackRef callargs; - _PyStackRef func_st; - _PyStackRef null; - _PyStackRef callargs_st; - _PyStackRef kwargs_st; - _PyStackRef result; - // _MAKE_CALLARGS_A_TUPLE - { - callargs = stack_pointer[-2]; - func = stack_pointer[-4]; - PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); - if (!PyTuple_CheckExact(callargs_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *tuple_o = PySequence_Tuple(callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (tuple_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - _PyStackRef temp = callargs; - callargs = PyStackRef_FromPyObjectSteal(tuple_o); - stack_pointer[-2] = callargs; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - // _DO_CALL_FUNCTION_EX - { - kwargs_st = stack_pointer[-1]; - callargs_st = callargs; - null = stack_pointer[-3]; - func_st = func; - (void)null; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); - EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); - PyObject *result_o; - assert(!_PyErr_Occurred(tstate)); - if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - assert(PyTuple_CheckExact(callargs)); - PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? - PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; - stack_pointer[-2] = callargs_st; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!PyFunction_Check(func) && !PyMethod_Check(func)) { - if (result_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(result_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - } - } - else { - if (Py_TYPE(func) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { - PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); - assert(PyTuple_CheckExact(callargs)); - PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); - int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( - tstate, func_st, locals, - nargs, callargs, kwargs, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (new_frame == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - assert( 1u == 1); - frame->return_offset = 1; - TRACING_DISPATCH_INLINED(new_frame); - } - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - assert(PyTuple_CheckExact(callargs)); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - stack_pointer[-2] = callargs_st; - _PyFrame_SetStackPointer(frame, stack_pointer); - result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(kwargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(func_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (result_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - result = PyStackRef_FromPyObjectSteal(result_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = result; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_INTRINSIC_1) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_INTRINSIC_1; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_INTRINSIC_1); - opcode = CALL_INTRINSIC_1; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - assert(oparg <= MAX_INTRINSIC_1); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_INTRINSIC_2) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_INTRINSIC_2; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_INTRINSIC_2); - opcode = CALL_INTRINSIC_2; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value2_st; - _PyStackRef value1_st; - _PyStackRef res; - value1_st = stack_pointer[-1]; - value2_st = stack_pointer[-2]; - assert(oparg <= MAX_INTRINSIC_2); - PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); - PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); - _PyStackRef tmp = value1_st; - value1_st = PyStackRef_NULL; - stack_pointer[-1] = value1_st; - PyStackRef_CLOSE(tmp); - tmp = value2_st; - value2_st = PyStackRef_NULL; - stack_pointer[-2] = value2_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_ISINSTANCE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_ISINSTANCE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_ISINSTANCE); - opcode = CALL_ISINSTANCE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef null; - _PyStackRef callable; - _PyStackRef instance; - _PyStackRef cls; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _GUARD_THIRD_NULL - { - null = stack_pointer[-3]; - if (!PyStackRef_IsNull(null)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _GUARD_CALLABLE_ISINSTANCE - { - callable = stack_pointer[-4]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyInterpreterState *interp = tstate->interp; - if (callable_o != interp->callable_cache.isinstance) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CALL_ISINSTANCE - { - cls = stack_pointer[-1]; - instance = stack_pointer[-2]; - STAT_INC(CALL, hit); - PyObject *inst_o = PyStackRef_AsPyObjectBorrow(instance); - PyObject *cls_o = PyStackRef_AsPyObjectBorrow(cls); - _PyFrame_SetStackPointer(frame, stack_pointer); - int retval = PyObject_IsInstance(inst_o, cls_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (retval < 0) { - TRACING_JUMP_TO_LABEL(error); - } - (void)null; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(cls); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(instance); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callable); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = retval ? PyStackRef_True : PyStackRef_False; - assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_KW) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_KW; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_KW); - PREDICTED_TRACING_CALL_KW:; - _Py_CODEUNIT* const this_instr = next_instr - 4; - (void)this_instr; - opcode = CALL_KW; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - opcode = CALL_KW; - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef kwnames; - _PyStackRef res; - // _SPECIALIZE_CALL_KW - { - self_or_null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(CALL_KW); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 2 cache entries */ - // _MAYBE_EXPAND_METHOD_KW - { - if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - self_or_null = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - _PyStackRef temp = callable; - callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - // _DO_CALL_KW - { - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - if (Py_TYPE(callable_o) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) - { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, callable, locals, - arguments, positional_args, kwnames_o, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (new_frame == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); - frame->return_offset = 4u ; - TRACING_DISPATCH_INLINED(new_frame); - } - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = kwnames; - kwnames = PyStackRef_NULL; - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; - stack_pointer[-1] = kwnames; - PyStackRef_CLOSE(tmp); - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-2 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-3 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - if (opcode == INSTRUMENTED_CALL_KW) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); - if (res_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = kwnames; - kwnames = PyStackRef_NULL; - stack_pointer[-1] = kwnames; - PyStackRef_CLOSE(tmp); - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-2 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-3 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_KW_BOUND_METHOD) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_KW_BOUND_METHOD; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); - opcode = CALL_KW_BOUND_METHOD; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef null; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef kwnames; - _PyStackRef new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - } - // _CHECK_METHOD_VERSION_KW - { - null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (Py_TYPE(callable_o) != &PyMethod_Type) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - PyObject *func = ((PyMethodObject *)callable_o)->im_func; - if (!PyFunction_Check(func)) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - if (((PyFunctionObject *)func)->func_version != func_version) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - if (!PyStackRef_IsNull(null)) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - } - // _EXPAND_METHOD_KW - { - self_or_null = null; - assert(PyStackRef_IsNull(self_or_null)); - _PyStackRef callable_s = callable; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - assert(Py_TYPE(callable_o) == &PyMethod_Type); - self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - assert(PyStackRef_FunctionCheck(callable)); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callable_s); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - // flush - // _PY_FRAME_KW - { - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - assert(Py_TYPE(callable_o) == &PyFunction_Type); - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, callable, locals, - arguments, positional_args, kwnames_o, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - new_frame = PyStackRef_Wrap(temp); - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_KW_NON_PY) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_KW_NON_PY; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_KW_NON_PY); - opcode = CALL_KW_NON_PY; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - opcode = CALL_KW_NON_PY; - static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef kwnames; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CHECK_IS_NOT_PY_CALLABLE_KW - { - callable = stack_pointer[-3 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (PyFunction_Check(callable_o)) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - if (Py_TYPE(callable_o) == &PyMethod_Type) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - } - // _CALL_KW_NON_PY - { - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = stack_pointer[-2 - oparg]; - #if TIER_ONE - assert(opcode != INSTRUMENTED_CALL); - #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = kwnames; - kwnames = PyStackRef_NULL; - stack_pointer[-1] = kwnames; - PyStackRef_CLOSE(tmp); - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-2 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-3 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_KW_PY) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_KW_PY; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_KW_PY); - opcode = CALL_KW_PY; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef kwnames; - _PyStackRef new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - } - // _CHECK_FUNCTION_VERSION_KW - { - callable = stack_pointer[-3 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (!PyFunction_Check(callable_o)) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - PyFunctionObject *func = (PyFunctionObject *)callable_o; - if (func->func_version != func_version) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - } - // _CHECK_RECURSION_REMAINING - { - if (tstate->py_recursion_remaining <= 1) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - } - // _PY_FRAME_KW - { - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - assert(Py_TYPE(callable_o) == &PyFunction_Type); - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, callable, locals, - arguments, positional_args, kwnames_o, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - new_frame = PyStackRef_Wrap(temp); - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_LEN) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_LEN; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_LEN); - opcode = CALL_LEN; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef null; - _PyStackRef callable; - _PyStackRef arg; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _GUARD_NOS_NULL - { - null = stack_pointer[-2]; - if (!PyStackRef_IsNull(null)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _GUARD_CALLABLE_LEN - { - callable = stack_pointer[-3]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyInterpreterState *interp = tstate->interp; - if (callable_o != interp->callable_cache.len) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CALL_LEN - { - arg = stack_pointer[-1]; - (void)null; - STAT_INC(CALL, hit); - PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_ssize_t len_i = PyObject_Length(arg_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (len_i < 0) { - TRACING_JUMP_TO_LABEL(error); - } - PyObject *res_o = PyLong_FromSsize_t(len_i); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callable); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_LIST_APPEND) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_LIST_APPEND; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_LIST_APPEND); - opcode = CALL_LIST_APPEND; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef nos; - _PyStackRef self; - _PyStackRef arg; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _GUARD_CALLABLE_LIST_APPEND - { - callable = stack_pointer[-3]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyInterpreterState *interp = tstate->interp; - if (callable_o != interp->callable_cache.list_append) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _GUARD_NOS_NOT_NULL - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (o == NULL) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _GUARD_NOS_LIST - { - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyList_CheckExact(o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CALL_LIST_APPEND - { - arg = stack_pointer[-1]; - self = nos; - assert(oparg == 1); - PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); - if (!PyList_CheckExact(self_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (!LOCK_OBJECT(self_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - STAT_INC(CALL, hit); - int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); - UNLOCK_OBJECT(self_o); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(self); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - #if TIER_ONE - - assert(next_instr->op.code == POP_TOP); - SKIP_OVER(1); - #endif - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_METHOD_DESCRIPTOR_FAST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_METHOD_DESCRIPTOR_FAST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); - opcode = CALL_METHOD_DESCRIPTOR_FAST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_FAST - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - if (total_args == 0) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyMethodDef *meth = method->d_method; - if (meth->ml_flags != METH_FASTCALL) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); - assert(self != NULL); - if (!Py_IS_TYPE(self, method->d_common.d_type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - STAT_INC(CALL, hit); - int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFast cfunc = _PyCFunctionFast_CAST(meth->ml_meth); - PyObject *res_o = cfunc(self, (args_o + 1), nargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); - opcode = CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - if (total_args == 0) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyMethodDef *meth = method->d_method; - if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyTypeObject *d_type = method->d_common.d_type; - PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); - assert(self != NULL); - if (!Py_IS_TYPE(self, d_type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - STAT_INC(CALL, hit); - int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFastWithKeywords cfunc = - _PyCFunctionFastWithKeywords_CAST(meth->ml_meth); - PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_METHOD_DESCRIPTOR_NOARGS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); - opcode = CALL_METHOD_DESCRIPTOR_NOARGS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_NOARGS - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - assert(oparg == 0 || oparg == 1); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { - args--; - total_args++; - } - if (total_args != 1) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyMethodDef *meth = method->d_method; - _PyStackRef self_stackref = args[0]; - PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); - if (!Py_IS_TYPE(self, method->d_common.d_type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (meth->ml_flags != METH_NOARGS) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (_Py_ReachedRecursionLimit(tstate)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(self_stackref); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_METHOD_DESCRIPTOR_O) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_METHOD_DESCRIPTOR_O; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); - opcode = CALL_METHOD_DESCRIPTOR_O; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_O - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - if (total_args != 2) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyMethodDef *meth = method->d_method; - if (meth->ml_flags != METH_O) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (_Py_ReachedRecursionLimit(tstate)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - _PyStackRef arg_stackref = arguments[1]; - _PyStackRef self_stackref = arguments[0]; - if (!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), - method->d_common.d_type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, - PyStackRef_AsPyObjectBorrow(self_stackref), - PyStackRef_AsPyObjectBorrow(arg_stackref)); - stack_pointer = _PyFrame_GetStackPointer(frame); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_NON_PY_GENERAL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_NON_PY_GENERAL; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_NON_PY_GENERAL); - opcode = CALL_NON_PY_GENERAL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - opcode = CALL_NON_PY_GENERAL; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CHECK_IS_NOT_PY_CALLABLE - { - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (PyFunction_Check(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (Py_TYPE(callable_o) == &PyMethod_Type) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CALL_NON_PY_GENERAL - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - #if TIER_ONE - assert(opcode != INSTRUMENTED_CALL); - #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_PY_EXACT_ARGS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_PY_EXACT_ARGS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); - opcode = CALL_PY_EXACT_ARGS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_FUNCTION_VERSION - { - callable = stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (!PyFunction_Check(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyFunctionObject *func = (PyFunctionObject *)callable_o; - if (func->func_version != func_version) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_FUNCTION_EXACT_ARGS - { - self_or_null = stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - assert(PyFunction_Check(callable_o)); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - PyCodeObject *code = (PyCodeObject *)func->func_code; - if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_STACK_SPACE - { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - PyCodeObject *code = (PyCodeObject *)func->func_code; - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_RECURSION_REMAINING - { - if (tstate->py_recursion_remaining <= 1) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _INIT_CALL_PY_EXACT_ARGS - { - args = &stack_pointer[-oparg]; - int has_self = !PyStackRef_IsNull(self_or_null); - STAT_INC(CALL, hit); - _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); - _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; - pushed_frame->localsplus[0] = self_or_null; - for (int i = 0; i < oparg; i++) { - first_non_self_local[i] = args[i]; - } - new_frame = PyStackRef_Wrap(pushed_frame); - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_PY_GENERAL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_PY_GENERAL; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_PY_GENERAL); - opcode = CALL_PY_GENERAL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_FUNCTION_VERSION - { - callable = stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (!PyFunction_Check(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyFunctionObject *func = (PyFunctionObject *)callable_o; - if (func->func_version != func_version) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_RECURSION_REMAINING - { - if (tstate->py_recursion_remaining <= 1) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _PY_FRAME_GENERAL - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { - args--; - total_args++; - } - assert(Py_TYPE(callable_o) == &PyFunction_Type); - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, callable, locals, - args, total_args, NULL, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - new_frame = PyStackRef_Wrap(temp); - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_STR_1) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_STR_1; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_STR_1); - opcode = CALL_STR_1; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef null; - _PyStackRef callable; - _PyStackRef arg; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _GUARD_NOS_NULL - { - null = stack_pointer[-2]; - if (!PyStackRef_IsNull(null)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _GUARD_CALLABLE_STR_1 - { - callable = stack_pointer[-3]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (callable_o != (PyObject *)&PyUnicode_Type) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CALL_STR_1 - { - arg = stack_pointer[-1]; - PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); - STAT_INC(CALL, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Str(arg_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - (void)callable; - (void)null; - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_TUPLE_1) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_TUPLE_1; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_TUPLE_1); - opcode = CALL_TUPLE_1; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef null; - _PyStackRef callable; - _PyStackRef arg; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _GUARD_NOS_NULL - { - null = stack_pointer[-2]; - if (!PyStackRef_IsNull(null)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _GUARD_CALLABLE_TUPLE_1 - { - callable = stack_pointer[-3]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (callable_o != (PyObject *)&PyTuple_Type) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CALL_TUPLE_1 - { - arg = stack_pointer[-1]; - PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); - STAT_INC(CALL, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PySequence_Tuple(arg_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - (void)callable; - (void)null; - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_TYPE_1) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_TYPE_1; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_TYPE_1); - opcode = CALL_TYPE_1; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef null; - _PyStackRef callable; - _PyStackRef arg; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _GUARD_NOS_NULL - { - null = stack_pointer[-2]; - if (!PyStackRef_IsNull(null)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _GUARD_CALLABLE_TYPE_1 - { - callable = stack_pointer[-3]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (callable_o != (PyObject *)&PyType_Type) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CALL_TYPE_1 - { - arg = stack_pointer[-1]; - PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); - (void)callable; - (void)null; - STAT_INC(CALL, hit); - res = PyStackRef_FromPyObjectNew(Py_TYPE(arg_o)); - stack_pointer[-3] = res; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CHECK_EG_MATCH) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CHECK_EG_MATCH; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CHECK_EG_MATCH); - opcode = CHECK_EG_MATCH; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef exc_value_st; - _PyStackRef match_type_st; - _PyStackRef rest; - _PyStackRef match; - match_type_st = stack_pointer[-1]; - exc_value_st = stack_pointer[-2]; - PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); - PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = match_type_st; - match_type_st = PyStackRef_NULL; - stack_pointer[-1] = match_type_st; - PyStackRef_CLOSE(tmp); - tmp = exc_value_st; - exc_value_st = PyStackRef_NULL; - stack_pointer[-2] = exc_value_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - PyObject *match_o = NULL; - PyObject *rest_o = NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_ExceptionGroupMatch(frame, exc_value, match_type, - &match_o, &rest_o); - _PyStackRef tmp = match_type_st; - match_type_st = PyStackRef_NULL; - stack_pointer[-1] = match_type_st; - PyStackRef_CLOSE(tmp); - tmp = exc_value_st; - exc_value_st = PyStackRef_NULL; - stack_pointer[-2] = exc_value_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res < 0) { - TRACING_JUMP_TO_LABEL(error); - } - assert((match_o == NULL) == (rest_o == NULL)); - if (match_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - if (!Py_IsNone(match_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyErr_SetHandledException(match_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - rest = PyStackRef_FromPyObjectSteal(rest_o); - match = PyStackRef_FromPyObjectSteal(match_o); - stack_pointer[0] = rest; - stack_pointer[1] = match; - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CHECK_EXC_MATCH) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CHECK_EXC_MATCH; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CHECK_EXC_MATCH); - opcode = CHECK_EXC_MATCH; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef left; - _PyStackRef right; - _PyStackRef b; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyExceptionInstance_Check(left_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyEval_CheckExceptTypeValid(tstate, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = PyErr_GivenExceptionMatches(left_o, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(right); - stack_pointer = _PyFrame_GetStackPointer(frame); - b = res ? PyStackRef_True : PyStackRef_False; - stack_pointer[0] = b; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CLEANUP_THROW) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CLEANUP_THROW; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CLEANUP_THROW); - opcode = CLEANUP_THROW; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef sub_iter; - _PyStackRef last_sent_val; - _PyStackRef exc_value_st; - _PyStackRef none; - _PyStackRef value; - exc_value_st = stack_pointer[-1]; - last_sent_val = stack_pointer[-2]; - sub_iter = stack_pointer[-3]; - PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); - #if !_Py_TAIL_CALL_INTERP - assert(throwflag); - #endif - assert(exc_value && PyExceptionInstance_Check(exc_value)); - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = sub_iter; - sub_iter = value; - stack_pointer[-3] = sub_iter; - PyStackRef_CLOSE(tmp); - tmp = exc_value_st; - exc_value_st = PyStackRef_NULL; - stack_pointer[-1] = exc_value_st; - PyStackRef_CLOSE(tmp); - tmp = last_sent_val; - last_sent_val = PyStackRef_NULL; - stack_pointer[-2] = last_sent_val; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - none = PyStackRef_None; - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); - monitor_reraise(tstate, frame, this_instr); - TRACING_JUMP_TO_LABEL(exception_unwind); - } - stack_pointer[0] = none; - stack_pointer[1] = value; - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(COMPARE_OP) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = COMPARE_OP; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP); - PREDICTED_TRACING_COMPARE_OP:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - opcode = COMPARE_OP; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _SPECIALIZE_COMPARE_OP - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_CompareOp(left, right, next_instr, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(COMPARE_OP); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _COMPARE_OP - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert((oparg >> 5) <= Py_GE); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); - _PyStackRef tmp = right; - right = PyStackRef_NULL; - stack_pointer[-1] = right; - PyStackRef_CLOSE(tmp); - tmp = left; - left = PyStackRef_NULL; - stack_pointer[-2] = left; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - if (oparg & 16) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int res_bool = PyObject_IsTrue(res_o); - Py_DECREF(res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_bool < 0) { - TRACING_JUMP_TO_LABEL(error); - } - res = res_bool ? PyStackRef_True : PyStackRef_False; - } - else { - res = PyStackRef_FromPyObjectSteal(res_o); - } - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(COMPARE_OP_FLOAT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = COMPARE_OP_FLOAT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_FLOAT); - opcode = COMPARE_OP_FLOAT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - _PyStackRef value; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_TOS_FLOAT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyFloat_CheckExact(value_o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - JUMP_TO_PREDICTED(TRACING_COMPARE_OP); - } - } - // _GUARD_NOS_FLOAT - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!PyFloat_CheckExact(left_o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - JUMP_TO_PREDICTED(TRACING_COMPARE_OP); - } - } - /* Skip 1 cache entry */ - // _COMPARE_OP_FLOAT - { - right = value; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - STAT_INC(COMPARE_OP, hit); - double dleft = PyFloat_AS_DOUBLE(left_o); - double dright = PyFloat_AS_DOUBLE(right_o); - int sign_ish = COMPARISON_BIT(dleft, dright); - PyStackRef_CLOSE_SPECIALIZED(left, _PyFloat_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(right, _PyFloat_ExactDealloc); - res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(COMPARE_OP_INT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = COMPARE_OP_INT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_INT); - opcode = COMPARE_OP_INT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - _PyStackRef value; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_TOS_INT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - JUMP_TO_PREDICTED(TRACING_COMPARE_OP); - } - } - // _GUARD_NOS_INT - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!_PyLong_CheckExactAndCompact(left_o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - JUMP_TO_PREDICTED(TRACING_COMPARE_OP); - } - } - /* Skip 1 cache entry */ - // _COMPARE_OP_INT - { - right = value; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(_PyLong_IsCompact((PyLongObject *)left_o)); - assert(_PyLong_IsCompact((PyLongObject *)right_o)); - STAT_INC(COMPARE_OP, hit); - assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && - _PyLong_DigitCount((PyLongObject *)right_o) <= 1); - Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o); - Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); - int sign_ish = COMPARISON_BIT(ileft, iright); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(COMPARE_OP_STR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = COMPARE_OP_STR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_STR); - opcode = COMPARE_OP_STR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - _PyStackRef value; - _PyStackRef nos; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_TOS_UNICODE - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyUnicode_CheckExact(value_o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - JUMP_TO_PREDICTED(TRACING_COMPARE_OP); - } - } - // _GUARD_NOS_UNICODE - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyUnicode_CheckExact(o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - JUMP_TO_PREDICTED(TRACING_COMPARE_OP); - } - } - /* Skip 1 cache entry */ - // _COMPARE_OP_STR - { - right = value; - left = nos; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - STAT_INC(COMPARE_OP, hit); - int eq = _PyUnicode_Equal(left_o, right_o); - assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); - PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); - assert(eq == 0 || eq == 1); - assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); - assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); - res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False; - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CONTAINS_OP) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CONTAINS_OP; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(CONTAINS_OP); - PREDICTED_TRACING_CONTAINS_OP:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - opcode = CONTAINS_OP; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef right; - _PyStackRef left; - _PyStackRef b; - // _SPECIALIZE_CONTAINS_OP - { - right = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_ContainsOp(right, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(CONTAINS_OP); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _CONTAINS_OP - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = PySequence_Contains(right_o, left_o); - _PyStackRef tmp = right; - right = PyStackRef_NULL; - stack_pointer[-1] = right; - PyStackRef_CLOSE(tmp); - tmp = left; - left = PyStackRef_NULL; - stack_pointer[-2] = left; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res < 0) { - TRACING_JUMP_TO_LABEL(error); - } - b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; - } - stack_pointer[0] = b; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CONTAINS_OP_DICT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CONTAINS_OP_DICT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(CONTAINS_OP_DICT); - opcode = CONTAINS_OP_DICT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); - _PyStackRef tos; - _PyStackRef left; - _PyStackRef right; - _PyStackRef b; - // _GUARD_TOS_DICT - { - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyDict_CheckExact(o)) { - UPDATE_MISS_STATS(CONTAINS_OP); - assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); - JUMP_TO_PREDICTED(TRACING_CONTAINS_OP); - } - } - /* Skip 1 cache entry */ - // _CONTAINS_OP_DICT - { - right = tos; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyDict_CheckExact(right_o)); - STAT_INC(CONTAINS_OP, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = PyDict_Contains(right_o, left_o); - _PyStackRef tmp = right; - right = PyStackRef_NULL; - stack_pointer[-1] = right; - PyStackRef_CLOSE(tmp); - tmp = left; - left = PyStackRef_NULL; - stack_pointer[-2] = left; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res < 0) { - TRACING_JUMP_TO_LABEL(error); - } - b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; - } - stack_pointer[0] = b; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CONTAINS_OP_SET) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CONTAINS_OP_SET; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(CONTAINS_OP_SET); - opcode = CONTAINS_OP_SET; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); - _PyStackRef tos; - _PyStackRef left; - _PyStackRef right; - _PyStackRef b; - // _GUARD_TOS_ANY_SET - { - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyAnySet_CheckExact(o)) { - UPDATE_MISS_STATS(CONTAINS_OP); - assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); - JUMP_TO_PREDICTED(TRACING_CONTAINS_OP); - } - } - /* Skip 1 cache entry */ - // _CONTAINS_OP_SET - { - right = tos; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyAnySet_CheckExact(right_o)); - STAT_INC(CONTAINS_OP, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PySet_Contains((PySetObject *)right_o, left_o); - _PyStackRef tmp = right; - right = PyStackRef_NULL; - stack_pointer[-1] = right; - PyStackRef_CLOSE(tmp); - tmp = left; - left = PyStackRef_NULL; - stack_pointer[-2] = left; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res < 0) { - TRACING_JUMP_TO_LABEL(error); - } - b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; - } - stack_pointer[0] = b; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CONVERT_VALUE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CONVERT_VALUE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CONVERT_VALUE); - opcode = CONVERT_VALUE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - _PyStackRef result; - value = stack_pointer[-1]; - conversion_func conv_fn; - assert(oparg >= FVC_STR && oparg <= FVC_ASCII); - conv_fn = _PyEval_ConversionFuncs[oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (result_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - result = PyStackRef_FromPyObjectSteal(result_o); - stack_pointer[0] = result; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(COPY) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = COPY; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(COPY); - opcode = COPY; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef bottom; - _PyStackRef top; - bottom = stack_pointer[-1 - (oparg-1)]; - top = PyStackRef_DUP(bottom); - stack_pointer[0] = top; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(COPY_FREE_VARS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = COPY_FREE_VARS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(COPY_FREE_VARS); - opcode = COPY_FREE_VARS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - PyCodeObject *co = _PyFrame_GetCode(frame); - assert(PyStackRef_FunctionCheck(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - PyObject *closure = func->func_closure; - assert(oparg == co->co_nfreevars); - int offset = co->co_nlocalsplus - oparg; - for (int i = 0; i < oparg; ++i) { - PyObject *o = PyTuple_GET_ITEM(closure, i); - frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(DELETE_ATTR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = DELETE_ATTR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_ATTR); - opcode = DELETE_ATTR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef owner; - owner = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(DELETE_DEREF) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = DELETE_DEREF; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_DEREF); - opcode = DELETE_DEREF; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); - if (oldobj == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(oldobj); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(DELETE_FAST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = DELETE_FAST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_FAST); - opcode = DELETE_FAST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef v = GETLOCAL(oparg); - if (PyStackRef_IsNull(v)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, - UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = PyStackRef_NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(DELETE_GLOBAL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = DELETE_GLOBAL; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_GLOBAL); - opcode = DELETE_GLOBAL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyDict_Pop(GLOBALS(), name, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - if (err == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(DELETE_NAME) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = DELETE_NAME; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_NAME); - opcode = DELETE_NAME; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *ns = LOCALS(); - int err; - if (ns == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_SystemError, - "no locals when deleting %R", name); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_DelItem(ns, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, - name); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(DELETE_SUBSCR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = DELETE_SUBSCR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_SUBSCR); - opcode = DELETE_SUBSCR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef container; - _PyStackRef sub; - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), - PyStackRef_AsPyObjectBorrow(sub)); - _PyStackRef tmp = sub; - sub = PyStackRef_NULL; - stack_pointer[-1] = sub; - PyStackRef_CLOSE(tmp); - tmp = container; - container = PyStackRef_NULL; - stack_pointer[-2] = container; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(DICT_MERGE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = DICT_MERGE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DICT_MERGE); - opcode = DICT_MERGE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef callable; - _PyStackRef dict; - _PyStackRef update; - update = stack_pointer[-1]; - dict = stack_pointer[-2 - (oparg - 1)]; - callable = stack_pointer[-5 - (oparg - 1)]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); - PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyDict_MergeEx(dict_o, update_o, 2); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatKwargsError(tstate, callable_o, update_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(update); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(update); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(DICT_UPDATE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = DICT_UPDATE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DICT_UPDATE); - opcode = DICT_UPDATE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef dict; - _PyStackRef update; - update = stack_pointer[-1]; - dict = stack_pointer[-2 - (oparg - 1)]; - PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); - PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyDict_Update(dict_o, update_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object is not a mapping", - Py_TYPE(update_o)->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(update); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(update); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(END_ASYNC_FOR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = END_ASYNC_FOR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(END_ASYNC_FOR); - opcode = END_ASYNC_FOR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef awaitable_st; - _PyStackRef exc_st; - exc_st = stack_pointer[-1]; - awaitable_st = stack_pointer[-2]; - JUMPBY(0); - (void)oparg; - PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); - assert(exc && PyExceptionInstance_Check(exc)); - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = exc_st; - exc_st = PyStackRef_NULL; - stack_pointer[-1] = exc_st; - PyStackRef_CLOSE(tmp); - tmp = awaitable_st; - awaitable_st = PyStackRef_NULL; - stack_pointer[-2] = awaitable_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - } - else { - Py_INCREF(exc); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, this_instr); - TRACING_JUMP_TO_LABEL(exception_unwind); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(END_FOR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = END_FOR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(END_FOR); - opcode = END_FOR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - value = stack_pointer[-1]; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(END_SEND) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = END_SEND; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(END_SEND); - opcode = END_SEND; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef receiver; - _PyStackRef value; - _PyStackRef val; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; - val = value; - stack_pointer[-2] = val; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(receiver); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(ENTER_EXECUTOR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = ENTER_EXECUTOR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(ENTER_EXECUTOR); - opcode = ENTER_EXECUTOR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - opcode = ENTER_EXECUTOR; - #ifdef _Py_TIER2 - PyCodeObject *code = _PyFrame_GetCode(frame); - _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; - assert(executor->vm_data.index == INSTR_OFFSET() - 1); - assert(executor->vm_data.code == code); - assert(executor->vm_data.valid); - assert(tstate->current_executor == NULL); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - opcode = executor->vm_data.opcode; - oparg = (oparg & ~255) | executor->vm_data.oparg; - next_instr = this_instr; - if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { - PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - } - DISPATCH_GOTO(); - } - assert(executor != tstate->interp->cold_executor); - tstate->jit_exit = NULL; - if (IS_JIT_TRACING()) { - RECORD_TRACE_NO_DISPATCH(); - } - TIER1_TO_TIER2(executor); - #else - Py_FatalError("ENTER_EXECUTOR is not supported in this build"); - #endif /* _Py_TIER2 */ - TRACING_DISPATCH(); - } - - TRACING_TARGET(EXIT_INIT_CHECK) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = EXIT_INIT_CHECK; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(EXIT_INIT_CHECK); - opcode = EXIT_INIT_CHECK; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef should_be_none; - should_be_none = stack_pointer[-1]; - if (!PyStackRef_IsNone(should_be_none)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyErr_Format(PyExc_TypeError, - "__init__() should return None, not '%.200s'", - Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(EXTENDED_ARG) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = EXTENDED_ARG; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(EXTENDED_ARG); - opcode = EXTENDED_ARG; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - opcode = EXTENDED_ARG; - assert(oparg); - opcode = next_instr->op.code; - oparg = oparg << 8 | next_instr->op.arg; - PRE_DISPATCH_GOTO(); - DISPATCH_GOTO(); - } - - TRACING_TARGET(FORMAT_SIMPLE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = FORMAT_SIMPLE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(FORMAT_SIMPLE); - opcode = FORMAT_SIMPLE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyUnicode_CheckExact(value_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Format(value_o, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - else { - res = value; - stack_pointer += -1; - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(FORMAT_WITH_SPEC) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = FORMAT_WITH_SPEC; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(FORMAT_WITH_SPEC); - opcode = FORMAT_WITH_SPEC; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - _PyStackRef fmt_spec; - _PyStackRef res; - fmt_spec = stack_pointer[-1]; - value = stack_pointer[-2]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); - _PyStackRef tmp = fmt_spec; - fmt_spec = PyStackRef_NULL; - stack_pointer[-1] = fmt_spec; - PyStackRef_CLOSE(tmp); - tmp = value; - value = PyStackRef_NULL; - stack_pointer[-2] = value; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(FOR_ITER) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = FOR_ITER; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER); - PREDICTED_TRACING_FOR_ITER:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - opcode = FOR_ITER; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef iter; - _PyStackRef null_or_index; - _PyStackRef next; - // _SPECIALIZE_FOR_ITER - { - null_or_index = stack_pointer[-1]; - iter = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_ForIter(iter, null_or_index, next_instr, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(FOR_ITER); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _FOR_ITER - { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!PyStackRef_IsValid(item)) { - if (PyStackRef_IsError(item)) { - TRACING_JUMP_TO_LABEL(error); - } - JUMPBY(oparg + 1); - RECORD_JUMP_TAKEN(); - stack_pointer[-1] = null_or_index; - TRACING_DISPATCH(); - } - next = item; - } - stack_pointer[-1] = null_or_index; - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(FOR_ITER_GEN) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = FOR_ITER_GEN; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_GEN); - opcode = FOR_ITER_GEN; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - _PyStackRef iter; - _PyStackRef gen_frame; - _PyStackRef new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(TRACING_FOR_ITER); - } - } - // _FOR_ITER_GEN_FRAME - { - iter = stack_pointer[-2]; - PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(gen) != &PyGen_Type) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(TRACING_FOR_ITER); - } - #ifdef Py_GIL_DISABLED - if (!_PyObject_IsUniquelyReferenced((PyObject *)gen)) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(TRACING_FOR_ITER); - } - #endif - if (gen->gi_frame_state >= FRAME_EXECUTING) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(TRACING_FOR_ITER); - } - STAT_INC(FOR_ITER, hit); - _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; - _PyFrame_StackPush(pushed_frame, PyStackRef_None); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - pushed_frame->previous = frame; - frame->return_offset = (uint16_t)( 2u + oparg); - gen_frame = PyStackRef_Wrap(pushed_frame); - } - // _PUSH_FRAME - { - new_frame = gen_frame; - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(FOR_ITER_LIST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = FOR_ITER_LIST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_LIST); - opcode = FOR_ITER_LIST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - _PyStackRef iter; - _PyStackRef null_or_index; - _PyStackRef next; - /* Skip 1 cache entry */ - // _ITER_CHECK_LIST - { - null_or_index = stack_pointer[-1]; - iter = stack_pointer[-2]; - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(iter_o) != &PyList_Type) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(TRACING_FOR_ITER); - } - assert(PyStackRef_IsTaggedInt(null_or_index)); - #ifdef Py_GIL_DISABLED - if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(TRACING_FOR_ITER); - } - #endif - } - // _ITER_JUMP_LIST - { - #ifdef Py_GIL_DISABLED - - #else - PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(list_o) == &PyList_Type); - STAT_INC(FOR_ITER, hit); - if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { - null_or_index = PyStackRef_TagInt(-1); - JUMPBY(oparg + 1); - RECORD_JUMP_TAKEN(); - stack_pointer[-1] = null_or_index; - TRACING_DISPATCH(); - } - #endif - } - // _ITER_NEXT_LIST - { - PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); - assert(PyList_CheckExact(list_o)); - #ifdef Py_GIL_DISABLED - assert(_Py_IsOwnedByCurrentThread(list_o) || - _PyObject_GC_IS_SHARED(list_o)); - STAT_INC(FOR_ITER, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (result < 0) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(TRACING_FOR_ITER); - } - if (result == 0) { - null_or_index = PyStackRef_TagInt(-1); - JUMPBY(oparg + 1); - stack_pointer[-1] = null_or_index; - TRACING_DISPATCH(); - } - #else - next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index))); - #endif - null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); - } - stack_pointer[-1] = null_or_index; - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(FOR_ITER_RANGE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = FOR_ITER_RANGE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_RANGE); - opcode = FOR_ITER_RANGE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - _PyStackRef iter; - _PyStackRef next; - /* Skip 1 cache entry */ - // _ITER_CHECK_RANGE - { - iter = stack_pointer[-2]; - _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(r) != &PyRangeIter_Type) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(TRACING_FOR_ITER); - } - #ifdef Py_GIL_DISABLED - if (!_PyObject_IsUniquelyReferenced((PyObject *)r)) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(TRACING_FOR_ITER); - } - #endif - } - // _ITER_JUMP_RANGE - { - _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(r) == &PyRangeIter_Type); - #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced((PyObject *)r)); - #endif - STAT_INC(FOR_ITER, hit); - if (r->len <= 0) { - JUMPBY(oparg + 1); - RECORD_JUMP_TAKEN(); - TRACING_DISPATCH(); - } - } - // _ITER_NEXT_RANGE - { - _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(r) == &PyRangeIter_Type); - #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced((PyObject *)r)); - #endif - assert(r->len > 0); - long value = r->start; - r->start = value + r->step; - r->len--; - PyObject *res = PyLong_FromLong(value); - if (res == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - next = PyStackRef_FromPyObjectSteal(res); - } - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(FOR_ITER_TUPLE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = FOR_ITER_TUPLE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_TUPLE); - opcode = FOR_ITER_TUPLE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - _PyStackRef iter; - _PyStackRef null_or_index; - _PyStackRef next; - /* Skip 1 cache entry */ - // _ITER_CHECK_TUPLE - { - null_or_index = stack_pointer[-1]; - iter = stack_pointer[-2]; - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(iter_o) != &PyTuple_Type) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(TRACING_FOR_ITER); - } - assert(PyStackRef_IsTaggedInt(null_or_index)); - } - // _ITER_JUMP_TUPLE - { - PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); - (void)tuple_o; - assert(Py_TYPE(tuple_o) == &PyTuple_Type); - STAT_INC(FOR_ITER, hit); - if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { - null_or_index = PyStackRef_TagInt(-1); - JUMPBY(oparg + 1); - RECORD_JUMP_TAKEN(); - stack_pointer[-1] = null_or_index; - TRACING_DISPATCH(); - } - } - // _ITER_NEXT_TUPLE - { - PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(tuple_o) == &PyTuple_Type); - uintptr_t i = PyStackRef_UntagInt(null_or_index); - assert((size_t)i < (size_t)PyTuple_GET_SIZE(tuple_o)); - next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(tuple_o, i)); - null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); - } - stack_pointer[-1] = null_or_index; - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(GET_AITER) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = GET_AITER; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_AITER); - opcode = GET_AITER; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef obj; - _PyStackRef iter; - obj = stack_pointer[-1]; - unaryfunc getter = NULL; - PyObject *obj_o = PyStackRef_AsPyObjectBorrow(obj); - PyObject *iter_o; - PyTypeObject *type = Py_TYPE(obj_o); - if (type->tp_as_async != NULL) { - getter = type->tp_as_async->am_aiter; - } - if (getter == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, - "'async for' requires an object with " - "__aiter__ method, got %.100s", - type->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(obj); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - iter_o = (*getter)(obj_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(obj); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (iter_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - if (Py_TYPE(iter_o)->tp_as_async == NULL || - Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, - "'async for' received an object from __aiter__ " - "that does not implement __anext__: %.100s", - Py_TYPE(iter_o)->tp_name); - Py_DECREF(iter_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[0] = iter; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(GET_ANEXT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = GET_ANEXT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_ANEXT); - opcode = GET_ANEXT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef aiter; - _PyStackRef awaitable; - aiter = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (awaitable_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); - stack_pointer[0] = awaitable; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(GET_AWAITABLE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = GET_AWAITABLE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_AWAITABLE); - opcode = GET_AWAITABLE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef iterable; - _PyStackRef iter; - iterable = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (iter_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[0] = iter; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(GET_ITER) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = GET_ITER; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_ITER); - opcode = GET_ITER; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef iterable; - _PyStackRef iter; - _PyStackRef index_or_null; - iterable = stack_pointer[-1]; - #ifdef Py_STATS - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_GatherStats_GetIter(iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - #endif - - PyTypeObject *tp = PyStackRef_TYPE(iterable); - if (tp == &PyTuple_Type || tp == &PyList_Type) { - iter = iterable; - index_or_null = PyStackRef_TagInt(0); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (iter_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - iter = PyStackRef_FromPyObjectSteal(iter_o); - index_or_null = PyStackRef_NULL; - stack_pointer += 1; - } - stack_pointer[-1] = iter; - stack_pointer[0] = index_or_null; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(GET_LEN) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = GET_LEN; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_LEN); - opcode = GET_LEN; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef obj; - _PyStackRef len; - obj = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (len_i < 0) { - TRACING_JUMP_TO_LABEL(error); - } - PyObject *len_o = PyLong_FromSsize_t(len_i); - if (len_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - len = PyStackRef_FromPyObjectSteal(len_o); - stack_pointer[0] = len; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(GET_YIELD_FROM_ITER) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = GET_YIELD_FROM_ITER; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_YIELD_FROM_ITER); - opcode = GET_YIELD_FROM_ITER; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef iterable; - _PyStackRef iter; - iterable = stack_pointer[-1]; - PyObject *iterable_o = PyStackRef_AsPyObjectBorrow(iterable); - if (PyCoro_CheckExact(iterable_o)) { - if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_TypeError, - "cannot 'yield from' a coroutine object " - "in a non-coroutine generator"); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - iter = iterable; - } - else if (PyGen_CheckExact(iterable_o)) { - iter = iterable; - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = PyObject_GetIter(iterable_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (iter_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - iter = PyStackRef_FromPyObjectSteal(iter_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = iterable; - iterable = iter; - stack_pointer[-1] = iterable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - stack_pointer[-1] = iter; - TRACING_DISPATCH(); - } - - TRACING_TARGET(IMPORT_FROM) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = IMPORT_FROM; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IMPORT_FROM); - opcode = IMPORT_FROM; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef from; - _PyStackRef res; - from = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(IMPORT_NAME) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = IMPORT_NAME; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IMPORT_NAME); - opcode = IMPORT_NAME; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef level; - _PyStackRef fromlist; - _PyStackRef res; - fromlist = stack_pointer[-1]; - level = stack_pointer[-2]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyEval_ImportName(tstate, frame, name, - PyStackRef_AsPyObjectBorrow(fromlist), - PyStackRef_AsPyObjectBorrow(level)); - _PyStackRef tmp = fromlist; - fromlist = PyStackRef_NULL; - stack_pointer[-1] = fromlist; - PyStackRef_CLOSE(tmp); - tmp = level; - level = PyStackRef_NULL; - stack_pointer[-2] = level; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_CALL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_CALL; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(INSTRUMENTED_CALL); - opcode = INSTRUMENTED_CALL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - opcode = INSTRUMENTED_CALL; - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef func; - _PyStackRef maybe_self; - _PyStackRef *args; - _PyStackRef res; - /* Skip 3 cache entries */ - // _MAYBE_EXPAND_METHOD - { - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - self_or_null = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - _PyStackRef temp = callable; - callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - // _MONITOR_CALL - { - args = &stack_pointer[-oparg]; - maybe_self = self_or_null; - func = callable; - int is_meth = !PyStackRef_IsNull(maybe_self); - PyObject *function = PyStackRef_AsPyObjectBorrow(func); - PyObject *arg0; - if (is_meth) { - arg0 = PyStackRef_AsPyObjectBorrow(maybe_self); - } - else if (oparg) { - arg0 = PyStackRef_AsPyObjectBorrow(args[0]); - } - else { - arg0 = &_PyInstrumentation_MISSING; - } - stack_pointer[-2 - oparg] = func; - stack_pointer[-1 - oparg] = maybe_self; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg0 - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - } - // _DO_CALL - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - if (Py_TYPE(callable_o) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) - { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, callable, locals, - arguments, total_args, NULL, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (new_frame == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - frame->return_offset = 4u ; - TRACING_DISPATCH_INLINED(new_frame); - } - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - if (opcode == INSTRUMENTED_CALL) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); - if (res_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - } - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_CALL_FUNCTION_EX; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); - opcode = INSTRUMENTED_CALL_FUNCTION_EX; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - opcode = INSTRUMENTED_CALL_FUNCTION_EX; - _PyStackRef func; - _PyStackRef callargs; - _PyStackRef func_st; - _PyStackRef null; - _PyStackRef callargs_st; - _PyStackRef kwargs_st; - _PyStackRef result; - // _MAKE_CALLARGS_A_TUPLE - { - callargs = stack_pointer[-2]; - func = stack_pointer[-4]; - PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); - if (!PyTuple_CheckExact(callargs_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *tuple_o = PySequence_Tuple(callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (tuple_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - _PyStackRef temp = callargs; - callargs = PyStackRef_FromPyObjectSteal(tuple_o); - stack_pointer[-2] = callargs; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - // _DO_CALL_FUNCTION_EX - { - kwargs_st = stack_pointer[-1]; - callargs_st = callargs; - null = stack_pointer[-3]; - func_st = func; - (void)null; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); - EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); - PyObject *result_o; - assert(!_PyErr_Occurred(tstate)); - if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - assert(PyTuple_CheckExact(callargs)); - PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? - PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; - stack_pointer[-2] = callargs_st; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!PyFunction_Check(func) && !PyMethod_Check(func)) { - if (result_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(result_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - } - } - else { - if (Py_TYPE(func) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { - PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); - assert(PyTuple_CheckExact(callargs)); - PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); - int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( - tstate, func_st, locals, - nargs, callargs, kwargs, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (new_frame == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - assert( 1u == 1); - frame->return_offset = 1; - TRACING_DISPATCH_INLINED(new_frame); - } - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - assert(PyTuple_CheckExact(callargs)); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - stack_pointer[-2] = callargs_st; - _PyFrame_SetStackPointer(frame, stack_pointer); - result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(kwargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(func_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (result_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - result = PyStackRef_FromPyObjectSteal(result_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = result; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_CALL_KW) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_CALL_KW; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); - opcode = INSTRUMENTED_CALL_KW; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - opcode = INSTRUMENTED_CALL_KW; - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef kwnames; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _MAYBE_EXPAND_METHOD_KW - { - self_or_null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; - if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - self_or_null = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - _PyStackRef temp = callable; - callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - // _MONITOR_CALL_KW - { - args = &stack_pointer[-1 - oparg]; - int is_meth = !PyStackRef_IsNull(self_or_null); - PyObject *arg; - if (is_meth) { - arg = PyStackRef_AsPyObjectBorrow(self_or_null); - } - else if (args) { - arg = PyStackRef_AsPyObjectBorrow(args[0]); - } - else { - arg = &_PyInstrumentation_MISSING; - } - PyObject *function = PyStackRef_AsPyObjectBorrow(callable); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - } - // _DO_CALL_KW - { - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - if (Py_TYPE(callable_o) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) - { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, callable, locals, - arguments, positional_args, kwnames_o, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (new_frame == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); - frame->return_offset = 4u ; - TRACING_DISPATCH_INLINED(new_frame); - } - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = kwnames; - kwnames = PyStackRef_NULL; - stack_pointer[-1] = kwnames; - PyStackRef_CLOSE(tmp); - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-2 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-3 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - if (opcode == INSTRUMENTED_CALL_KW) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); - if (res_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = kwnames; - kwnames = PyStackRef_NULL; - stack_pointer[-1] = kwnames; - PyStackRef_CLOSE(tmp); - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-2 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-3 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_END_ASYNC_FOR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_END_ASYNC_FOR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_END_ASYNC_FOR); - opcode = INSTRUMENTED_END_ASYNC_FOR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef awaitable_st; - _PyStackRef exc_st; - // _MONITOR_END_ASYNC_FOR - { - assert((next_instr-oparg)->op.code == END_SEND || (next_instr-oparg)->op.code >= MIN_INSTRUMENTED_OPCODE); - INSTRUMENTED_JUMP(next_instr-oparg, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT); - } - // _END_ASYNC_FOR - { - exc_st = stack_pointer[-1]; - awaitable_st = stack_pointer[-2]; - JUMPBY(0); - (void)oparg; - PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); - assert(exc && PyExceptionInstance_Check(exc)); - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = exc_st; - exc_st = PyStackRef_NULL; - stack_pointer[-1] = exc_st; - PyStackRef_CLOSE(tmp); - tmp = awaitable_st; - awaitable_st = PyStackRef_NULL; - stack_pointer[-2] = awaitable_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - } - else { - Py_INCREF(exc); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, this_instr); - TRACING_JUMP_TO_LABEL(exception_unwind); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_END_FOR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_END_FOR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_END_FOR); - opcode = INSTRUMENTED_END_FOR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef receiver; - _PyStackRef value; - value = stack_pointer[-1]; - receiver = stack_pointer[-3]; - if (PyStackRef_GenCheck(receiver)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_END_SEND) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_END_SEND; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_END_SEND); - opcode = INSTRUMENTED_END_SEND; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef receiver; - _PyStackRef value; - _PyStackRef val; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; - PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); - if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - } - val = value; - stack_pointer[-2] = val; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(receiver); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_FOR_ITER) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_FOR_ITER; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); - opcode = INSTRUMENTED_FOR_ITER; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef iter; - _PyStackRef null_or_index; - _PyStackRef next; - /* Skip 1 cache entry */ - null_or_index = stack_pointer[-1]; - iter = stack_pointer[-2]; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!PyStackRef_IsValid(item)) { - if (PyStackRef_IsError(item)) { - TRACING_JUMP_TO_LABEL(error); - } - JUMPBY(oparg + 1); - stack_pointer[-1] = null_or_index; - TRACING_DISPATCH(); - } - next = item; - INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); - stack_pointer[-1] = null_or_index; - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_INSTRUCTION) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_INSTRUCTION; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); - opcode = INSTRUMENTED_INSTRUCTION; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - opcode = INSTRUMENTED_INSTRUCTION; - _PyFrame_SetStackPointer(frame, stack_pointer); - int next_opcode = _Py_call_instrumentation_instruction( - tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (next_opcode < 0) { - TRACING_JUMP_TO_LABEL(error); - } - next_instr = this_instr; - if (_PyOpcode_Caches[next_opcode]) { - PAUSE_ADAPTIVE_COUNTER(next_instr[1].counter); - } - assert(next_opcode > 0 && next_opcode < 256); - opcode = next_opcode; - DISPATCH_GOTO(); - } - - TRACING_TARGET(INSTRUMENTED_JUMP_BACKWARD) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_JUMP_BACKWARD; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); - opcode = INSTRUMENTED_JUMP_BACKWARD; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - /* Skip 1 cache entry */ - // _CHECK_PERIODIC - { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - // _MONITOR_JUMP_BACKWARD - { - INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_JUMP_FORWARD) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_JUMP_FORWARD; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); - opcode = INSTRUMENTED_JUMP_FORWARD; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_LINE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_LINE; - (void)(opcode); - #endif - _Py_CODEUNIT* const prev_instr = frame->instr_ptr; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_LINE); - opcode = INSTRUMENTED_LINE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - opcode = INSTRUMENTED_LINE; - int original_opcode = 0; - if (tstate->tracing) { - PyCodeObject *code = _PyFrame_GetCode(frame); - int index = (int)(this_instr - _PyFrame_GetBytecode(frame)); - original_opcode = code->_co_monitoring->lines->data[index*code->_co_monitoring->lines->bytes_per_entry]; - next_instr = this_instr; - } else { - _PyFrame_SetStackPointer(frame, stack_pointer); - original_opcode = _Py_call_instrumentation_line( - tstate, frame, this_instr, prev_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (original_opcode < 0) { - next_instr = this_instr+1; - TRACING_JUMP_TO_LABEL(error); - } - next_instr = frame->instr_ptr; - if (next_instr != this_instr) { - TRACING_DISPATCH(); - } - } - if (_PyOpcode_Caches[original_opcode]) { - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); - PAUSE_ADAPTIVE_COUNTER(cache->counter); - } - opcode = original_opcode; - DISPATCH_GOTO(); - } - - TRACING_TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_LOAD_SUPER_ATTR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); - opcode = INSTRUMENTED_LOAD_SUPER_ATTR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - opcode = INSTRUMENTED_LOAD_SUPER_ATTR; - _PyStackRef global_super_st; - _PyStackRef class_st; - _PyStackRef self_st; - _PyStackRef attr; - _PyStackRef *null; - /* Skip 1 cache entry */ - // _LOAD_SUPER_ATTR - { - self_st = stack_pointer[-1]; - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = self_st; - self_st = PyStackRef_NULL; - stack_pointer[-1] = self_st; - PyStackRef_CLOSE(tmp); - tmp = class_st; - class_st = PyStackRef_NULL; - stack_pointer[-2] = class_st; - PyStackRef_CLOSE(tmp); - tmp = global_super_st; - global_super_st = PyStackRef_NULL; - stack_pointer[-3] = global_super_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - } - PyObject *stack[] = {class, self}; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - if (super == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(super); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = self_st; - self_st = PyStackRef_NULL; - stack_pointer[-1] = self_st; - PyStackRef_CLOSE(tmp); - tmp = class_st; - class_st = PyStackRef_NULL; - stack_pointer[-2] = class_st; - PyStackRef_CLOSE(tmp); - tmp = global_super_st; - global_super_st = PyStackRef_NULL; - stack_pointer[-3] = global_super_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - if (super == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = PyObject_GetAttr(super, name); - Py_DECREF(super); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - attr = PyStackRef_FromPyObjectSteal(attr_o); - } - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[1]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer[0] = attr; - stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_NOT_TAKEN) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_NOT_TAKEN; - (void)(opcode); - #endif - _Py_CODEUNIT* const prev_instr = frame->instr_ptr; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_NOT_TAKEN); - opcode = INSTRUMENTED_NOT_TAKEN; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - (void)this_instr; - INSTRUMENTED_JUMP(prev_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_POP_ITER) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_POP_ITER; - (void)(opcode); - #endif - _Py_CODEUNIT* const prev_instr = frame->instr_ptr; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_POP_ITER); - opcode = INSTRUMENTED_POP_ITER; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef iter; - _PyStackRef index_or_null; - index_or_null = stack_pointer[-1]; - iter = stack_pointer[-2]; - (void)index_or_null; - INSTRUMENTED_JUMP(prev_instr, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iter); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_POP_JUMP_IF_FALSE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); - opcode = INSTRUMENTED_POP_JUMP_IF_FALSE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef cond; - /* Skip 1 cache entry */ - cond = stack_pointer[-1]; - assert(PyStackRef_BoolCheck(cond)); - int jump = PyStackRef_IsFalse(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); - if (jump) { - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_POP_JUMP_IF_NONE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); - opcode = INSTRUMENTED_POP_JUMP_IF_NONE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - /* Skip 1 cache entry */ - value = stack_pointer[-1]; - int jump = PyStackRef_IsNone(value); - RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); - if (jump) { - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); - } - else { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += 1; - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_POP_JUMP_IF_NOT_NONE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); - opcode = INSTRUMENTED_POP_JUMP_IF_NOT_NONE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - /* Skip 1 cache entry */ - value = stack_pointer[-1]; - int jump = !PyStackRef_IsNone(value); - RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); - if (jump) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); - } - else { - stack_pointer += -1; - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_POP_JUMP_IF_TRUE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); - opcode = INSTRUMENTED_POP_JUMP_IF_TRUE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef cond; - /* Skip 1 cache entry */ - cond = stack_pointer[-1]; - assert(PyStackRef_BoolCheck(cond)); - int jump = PyStackRef_IsTrue(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); - if (jump) { - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_RESUME) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_RESUME; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_RESUME); - opcode = INSTRUMENTED_RESUME; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - // _LOAD_BYTECODE - { - #ifdef Py_GIL_DISABLED - if (frame->tlbc_index != - ((_PyThreadStateImpl *)tstate)->tlbc_index) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_CODEUNIT *bytecode = - _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (bytecode == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); - frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; - frame->instr_ptr = bytecode + off; - next_instr = frame->instr_ptr; - TRACING_DISPATCH(); - } - #endif - } - // _MAYBE_INSTRUMENT - { - #ifdef Py_GIL_DISABLED - - int check_instrumentation = 1; - #else - int check_instrumentation = (tstate->tracing == 0); - #endif - if (check_instrumentation) { - uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - if (code_version != global_version) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - next_instr = this_instr; - TRACING_DISPATCH(); - } - } - } - // _CHECK_PERIODIC_IF_NOT_YIELD_FROM - { - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - } - // _MONITOR_RESUME - { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation( - tstate, oparg > 0, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - if (frame->instr_ptr != this_instr) { - next_instr = frame->instr_ptr; - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_RETURN_VALUE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_RETURN_VALUE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); - opcode = INSTRUMENTED_RETURN_VALUE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef val; - _PyStackRef retval; - _PyStackRef res; - // _RETURN_VALUE_EVENT - { - val = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - } - // _RETURN_VALUE - { - retval = val; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - _PyStackRef temp = PyStackRef_MakeHeapSafe(retval); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(STACK_LEVEL() == 0); - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - stack_pointer = _PyFrame_GetStackPointer(frame); - LOAD_IP(frame->return_offset); - #if TIER_TWO - frame->instr_ptr += frame->return_offset; - #endif - res = temp; - LLTRACE_RESUME_FRAME(); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_YIELD_VALUE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_YIELD_VALUE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); - opcode = INSTRUMENTED_YIELD_VALUE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef val; - _PyStackRef retval; - _PyStackRef value; - // _YIELD_VALUE_EVENT - { - val = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_YIELD, - frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - if (frame->instr_ptr != this_instr) { - next_instr = frame->instr_ptr; - TRACING_DISPATCH(); - } - } - // _YIELD_VALUE - { - retval = val; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); - assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); - assert(oparg == 0 || oparg == 1); - gen->gi_frame_state = FRAME_SUSPENDED + oparg; - _PyStackRef temp = retval; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; - assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - #if TIER_ONE - assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || - frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); - #endif - stack_pointer = _PyFrame_GetStackPointer(frame); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - #if TIER_TWO - frame->instr_ptr += (1 + INLINE_CACHE_ENTRIES_SEND); - #endif - value = PyStackRef_MakeHeapSafe(temp); - LLTRACE_RESUME_FRAME(); - } - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INTERPRETER_EXIT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INTERPRETER_EXIT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INTERPRETER_EXIT); - opcode = INTERPRETER_EXIT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef retval; - retval = stack_pointer[-1]; - assert(frame->owner == FRAME_OWNED_BY_INTERPRETER); - assert(_PyFrame_IsIncomplete(frame)); - tstate->current_frame = frame->previous; - assert(!_PyErr_Occurred(tstate)); - PyObject *result = PyStackRef_AsPyObjectSteal(retval); - #if !_Py_TAIL_CALL_INTERP - assert(frame == &entry.frame); - #endif - #ifdef _Py_TIER2 - _PyStackRef executor = frame->localsplus[0]; - assert(tstate->current_executor == NULL); - if (!PyStackRef_IsNull(executor)) { - tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(executor); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += 1; - } - #endif - LLTRACE_RESUME_FRAME(); - return result; - } - - TRACING_TARGET(IS_OP) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = IS_OP; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IS_OP); - opcode = IS_OP; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef left; - _PyStackRef right; - _PyStackRef b; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = right; - right = PyStackRef_NULL; - stack_pointer[-1] = right; - PyStackRef_CLOSE(tmp); - tmp = left; - left = PyStackRef_NULL; - stack_pointer[-2] = left; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - b = res ? PyStackRef_True : PyStackRef_False; - stack_pointer[0] = b; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(JUMP_BACKWARD) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = JUMP_BACKWARD; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(JUMP_BACKWARD); - PREDICTED_TRACING_JUMP_BACKWARD:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - opcode = JUMP_BACKWARD; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - /* Skip 1 cache entry */ - // _SPECIALIZE_JUMP_BACKWARD - { - #if ENABLE_SPECIALIZATION - if (this_instr->op.code == JUMP_BACKWARD) { - this_instr->op.code = (tstate->interp->jit && !PyStackRef_IsNull(frame->f_funcobj)) ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; - next_instr = this_instr; - DISPATCH_SAME_OPARG(); - } - #endif - } - // _CHECK_PERIODIC - { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - // _JUMP_BACKWARD_NO_INTERRUPT - { - #if TIER_ONE - assert(oparg <= INSTR_OFFSET()); - #endif - JUMPBY(-oparg); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(JUMP_BACKWARD_JIT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = JUMP_BACKWARD_JIT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(JUMP_BACKWARD_JIT); - opcode = JUMP_BACKWARD_JIT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(1 == 1, "incorrect cache size"); - /* Skip 1 cache entry */ - // _CHECK_PERIODIC - { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - // _JUMP_BACKWARD_NO_INTERRUPT - { - #if TIER_ONE - assert(oparg <= INSTR_OFFSET()); - #endif - JUMPBY(-oparg); - } - // _JIT - { - #ifdef _Py_TIER2 - _Py_BackoffCounter counter = this_instr[1].counter; - if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && - this_instr->op.code == JUMP_BACKWARD_JIT && - next_instr->op.code != ENTER_EXECUTOR) { - _Py_CODEUNIT *start = this_instr; - int curr_oparg = oparg; - while (curr_oparg > 255) { - curr_oparg >>= 8; - start--; - } - if (tstate->interp->jit_tracer_code_buffer == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - tstate->interp->jit_tracer_code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (tstate->interp->jit_tracer_code_buffer == NULL) { - TRACING_DISPATCH(); - } - } - _PyJIT_InitializeTracing(tstate, frame, this_instr, STACK_LEVEL(), 0, NULL); - ENTER_TRACING(); - TRACING_DISPATCH(); - } - else { - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - } - #endif - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(JUMP_BACKWARD_NO_INTERRUPT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = JUMP_BACKWARD_NO_INTERRUPT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); - opcode = JUMP_BACKWARD_NO_INTERRUPT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - #if TIER_ONE - assert(oparg <= INSTR_OFFSET()); - #endif - JUMPBY(-oparg); - TRACING_DISPATCH(); - } - - TRACING_TARGET(JUMP_BACKWARD_NO_JIT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = JUMP_BACKWARD_NO_JIT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(JUMP_BACKWARD_NO_JIT); - opcode = JUMP_BACKWARD_NO_JIT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(1 == 1, "incorrect cache size"); - /* Skip 1 cache entry */ - // _CHECK_PERIODIC - { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - // _JUMP_BACKWARD_NO_INTERRUPT - { - #if TIER_ONE - assert(oparg <= INSTR_OFFSET()); - #endif - JUMPBY(-oparg); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(JUMP_FORWARD) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = JUMP_FORWARD; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(JUMP_FORWARD); - opcode = JUMP_FORWARD; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - JUMPBY(oparg); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LIST_APPEND) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LIST_APPEND; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LIST_APPEND); - opcode = LIST_APPEND; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef list; - _PyStackRef v; - v = stack_pointer[-1]; - list = stack_pointer[-2 - (oparg-1)]; - int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), - PyStackRef_AsPyObjectSteal(v)); - if (err < 0) { - TRACING_JUMP_TO_LABEL(pop_1_error); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LIST_EXTEND) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LIST_EXTEND; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LIST_EXTEND); - opcode = LIST_EXTEND; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef list_st; - _PyStackRef iterable_st; - iterable_st = stack_pointer[-1]; - list_st = stack_pointer[-2 - (oparg-1)]; - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (none_val == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches && - (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) - { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Clear(tstate); - _PyErr_Format(tstate, PyExc_TypeError, - "Value after * must be an iterable, not %.200s", - Py_TYPE(iterable)->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - assert(Py_IsNone(none_val)); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR); - PREDICTED_TRACING_LOAD_ATTR:; - _Py_CODEUNIT* const this_instr = next_instr - 10; - (void)this_instr; - opcode = LOAD_ATTR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef owner; - _PyStackRef *attr; - _PyStackRef *self_or_null; - // _SPECIALIZE_LOAD_ATTR - { - owner = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadAttr(owner, next_instr, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(LOAD_ATTR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 8 cache entries */ - // _LOAD_ATTR - { - attr = &stack_pointer[-1]; - self_or_null = &stack_pointer[0]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - if (oparg & 1) { - *attr = PyStackRef_NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - int is_meth = _PyObject_GetMethodStackRef(tstate, PyStackRef_AsPyObjectBorrow(owner), name, attr); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (is_meth) { - assert(!PyStackRef_IsNull(*attr)); - self_or_null[0] = owner; - } - else { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(*attr)) { - TRACING_JUMP_TO_LABEL(error); - } - self_or_null[0] = PyStackRef_NULL; - stack_pointer += 1; - } - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - *attr = PyStackRef_FromPyObjectSteal(attr_o); - stack_pointer += 1; - } - } - stack_pointer += (oparg&1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_CLASS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_CLASS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_CLASS); - opcode = LOAD_ATTR_CLASS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef *null; - /* Skip 1 cache entry */ - // _CHECK_ATTR_CLASS - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - if (!PyType_Check(owner_o)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_CLASS - { - PyObject *descr = read_obj(&this_instr[6].cache); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - attr = PyStackRef_FromPyObjectNew(descr); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = owner; - owner = attr; - stack_pointer[-1] = owner; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[0]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_CLASS_WITH_METACLASS_CHECK; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); - opcode = LOAD_ATTR_CLASS_WITH_METACLASS_CHECK; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef *null; - /* Skip 1 cache entry */ - // _CHECK_ATTR_CLASS - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - if (!PyType_Check(owner_o)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _GUARD_TYPE_VERSION - { - uint32_t type_version = read_u32(&this_instr[4].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _LOAD_ATTR_CLASS - { - PyObject *descr = read_obj(&this_instr[6].cache); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - attr = PyStackRef_FromPyObjectNew(descr); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = owner; - owner = attr; - stack_pointer[-1] = owner; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[0]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); - opcode = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - /* Skip 1 cache entry */ - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - uint32_t func_version = read_u32(&this_instr[4].cache); - PyObject *getattribute = read_obj(&this_instr[6].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert((oparg & 1) == 0); - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - PyTypeObject *cls = Py_TYPE(owner_o); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); - PyFunctionObject *f = (PyFunctionObject *)getattribute; - assert(func_version != 0); - if (f->func_version != func_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - PyCodeObject *code = (PyCodeObject *)f->func_code; - assert(code->co_argcount == 2); - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - STAT_INC(LOAD_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( - tstate, PyStackRef_FromPyObjectNew(f), 2, frame); - new_frame->localsplus[0] = owner; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - new_frame->localsplus[1] = PyStackRef_FromPyObjectNew(name); - frame->return_offset = 10u ; - TRACING_DISPATCH_INLINED(new_frame); - } - - TRACING_TARGET(LOAD_ATTR_INSTANCE_VALUE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_INSTANCE_VALUE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); - opcode = LOAD_ATTR_INSTANCE_VALUE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef *null; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _CHECK_MANAGED_OBJECT_HAS_VALUES - { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_dictoffset < 0); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _LOAD_ATTR_INSTANCE_VALUE - { - uint16_t offset = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); - PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); - if (attr_o == NULL) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - #ifdef Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr); - if (!increfed) { - if (true) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - #else - attr = PyStackRef_FromPyObjectNew(attr_o); - #endif - STAT_INC(LOAD_ATTR, hit); - stack_pointer[-1] = attr; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - /* Skip 5 cache entries */ - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[0]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_METHOD_LAZY_DICT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); - opcode = LOAD_ATTR_METHOD_LAZY_DICT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _CHECK_ATTR_METHOD_LAZY_DICT - { - uint16_t dictoffset = read_u16(&this_instr[4].cache); - char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; - PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); - if (dict != NULL) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - /* Skip 1 cache entry */ - // _LOAD_ATTR_METHOD_LAZY_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - } - stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_METHOD_NO_DICT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_METHOD_NO_DICT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); - opcode = LOAD_ATTR_METHOD_NO_DICT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_METHOD_NO_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - } - stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_METHOD_WITH_VALUES; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); - opcode = LOAD_ATTR_METHOD_WITH_VALUES; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT - { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - PyDictValues *ivs = _PyObject_InlineValues(owner_o); - if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _GUARD_KEYS_VERSION - { - uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _LOAD_ATTR_METHOD_WITH_VALUES - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - } - stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_MODULE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_MODULE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_MODULE); - opcode = LOAD_ATTR_MODULE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef *null; - /* Skip 1 cache entry */ - // _LOAD_ATTR_MODULE - { - owner = stack_pointer[-1]; - uint32_t dict_version = read_u32(&this_instr[2].cache); - uint16_t index = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; - assert(dict != NULL); - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - assert(keys->dk_kind == DICT_KEYS_UNICODE); - assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(keys->dk_nentries)); - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(keys) + index; - PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); - if (attr_o == NULL) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - #ifdef Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); - if (!increfed) { - if (true) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - #else - attr = PyStackRef_FromPyObjectNew(attr_o); - #endif - STAT_INC(LOAD_ATTR, hit); - stack_pointer[-1] = attr; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - /* Skip 5 cache entries */ - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[0]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_NONDESCRIPTOR_NO_DICT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); - opcode = LOAD_ATTR_NONDESCRIPTOR_NO_DICT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); - assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - attr = PyStackRef_FromPyObjectNew(descr); - } - stack_pointer[0] = attr; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); - opcode = LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT - { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - PyDictValues *ivs = _PyObject_InlineValues(owner_o); - if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _GUARD_KEYS_VERSION - { - uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - attr = PyStackRef_FromPyObjectNew(descr); - } - stack_pointer[0] = attr; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_PROPERTY) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_PROPERTY; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); - opcode = LOAD_ATTR_PROPERTY; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_PROPERTY_FRAME - { - PyObject *fget = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); - assert(Py_IS_TYPE(fget, &PyFunction_Type)); - PyFunctionObject *f = (PyFunctionObject *)fget; - PyCodeObject *code = (PyCodeObject *)f->func_code; - if ((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - if (code->co_kwonlyargcount) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - if (code->co_argcount != 1) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - STAT_INC(LOAD_ATTR, hit); - _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); - pushed_frame->localsplus[0] = owner; - new_frame = PyStackRef_Wrap(pushed_frame); - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_SLOT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_SLOT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_SLOT); - opcode = LOAD_ATTR_SLOT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef *null; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _LOAD_ATTR_SLOT - { - uint16_t index = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - PyObject **addr = (PyObject **)((char *)owner_o + index); - PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); - if (attr_o == NULL) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - #ifdef Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); - if (!increfed) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - #else - attr = PyStackRef_FromPyObjectNew(attr_o); - #endif - STAT_INC(LOAD_ATTR, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = owner; - owner = attr; - stack_pointer[-1] = owner; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - /* Skip 5 cache entries */ - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[0]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_WITH_HINT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_WITH_HINT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); - opcode = LOAD_ATTR_WITH_HINT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef *null; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _LOAD_ATTR_WITH_HINT - { - uint16_t hint = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - if (dict == NULL) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - PyDictKeysObject *dk = FT_ATOMIC_LOAD_PTR(dict->ma_keys); - assert(PyDict_CheckExact((PyObject *)dict)); - #ifdef Py_GIL_DISABLED - if (!_Py_IsOwnedByCurrentThread((PyObject *)dict) && !_PyObject_GC_IS_SHARED(dict)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - #endif - PyObject *attr_o; - if (hint >= (size_t)FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_nentries)) { - if (true) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - if (dk->dk_kind != DICT_KEYS_UNICODE) { - if (true) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dk) + hint; - if (FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_key) != name) { - if (true) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - attr_o = FT_ATOMIC_LOAD_PTR(ep->me_value); - if (attr_o == NULL) { - if (true) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - STAT_INC(LOAD_ATTR, hit); - #ifdef Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); - if (!increfed) { - if (true) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - #else - attr = PyStackRef_FromPyObjectNew(attr_o); - #endif - stack_pointer[-1] = attr; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - /* Skip 5 cache entries */ - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[0]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_BUILD_CLASS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_BUILD_CLASS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_BUILD_CLASS); - opcode = LOAD_BUILD_CLASS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef bc; - PyObject *bc_o; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - if (bc_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_NameError, - "__build_class__ not found"); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - bc = PyStackRef_FromPyObjectSteal(bc_o); - stack_pointer[0] = bc; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_COMMON_CONSTANT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_COMMON_CONSTANT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); - opcode = LOAD_COMMON_CONSTANT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - assert(oparg < NUM_COMMON_CONSTANTS); - value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_CONST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_CONST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_CONST); - opcode = LOAD_CONST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); - value = PyStackRef_FromPyObjectBorrow(obj); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_DEREF) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_DEREF; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_DEREF); - opcode = LOAD_DEREF; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - _PyFrame_SetStackPointer(frame, stack_pointer); - value = _PyCell_GetStackRef(cell); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(value)) { - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_FAST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_FAST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST); - opcode = LOAD_FAST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_DUP(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_FAST_AND_CLEAR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_FAST_AND_CLEAR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); - opcode = LOAD_FAST_AND_CLEAR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - value = GETLOCAL(oparg); - GETLOCAL(oparg) = PyStackRef_NULL; - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_FAST_BORROW) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_FAST_BORROW; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_BORROW); - opcode = LOAD_FAST_BORROW; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_FAST_BORROW_LOAD_FAST_BORROW) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_FAST_BORROW_LOAD_FAST_BORROW; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_BORROW_LOAD_FAST_BORROW); - opcode = LOAD_FAST_BORROW_LOAD_FAST_BORROW; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value1; - _PyStackRef value2; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - value1 = PyStackRef_Borrow(GETLOCAL(oparg1)); - value2 = PyStackRef_Borrow(GETLOCAL(oparg2)); - stack_pointer[0] = value1; - stack_pointer[1] = value2; - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_FAST_CHECK) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_FAST_CHECK; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_CHECK); - opcode = LOAD_FAST_CHECK; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - _PyStackRef value_s = GETLOCAL(oparg); - if (PyStackRef_IsNull(value_s)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, - UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - value = PyStackRef_DUP(value_s); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_FAST_LOAD_FAST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_FAST_LOAD_FAST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); - opcode = LOAD_FAST_LOAD_FAST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value1; - _PyStackRef value2; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - value1 = PyStackRef_DUP(GETLOCAL(oparg1)); - value2 = PyStackRef_DUP(GETLOCAL(oparg2)); - stack_pointer[0] = value1; - stack_pointer[1] = value2; - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_FROM_DICT_OR_DEREF) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_FROM_DICT_OR_DEREF; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); - opcode = LOAD_FROM_DICT_OR_DEREF; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef class_dict_st; - _PyStackRef value; - class_dict_st = stack_pointer[-1]; - PyObject *value_o; - PyObject *name; - PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); - assert(class_dict); - assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); - name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(class_dict, name, &value_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - if (!value_o) { - PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - value_o = PyCell_GetRef(cell); - if (value_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(class_dict_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - value = PyStackRef_FromPyObjectSteal(value_o); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_FROM_DICT_OR_GLOBALS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_FROM_DICT_OR_GLOBALS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); - opcode = LOAD_FROM_DICT_OR_GLOBALS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef mod_or_class_dict; - _PyStackRef v; - mod_or_class_dict = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *v_o; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(mod_or_class_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - if (v_o == NULL) { - if (PyDict_CheckExact(GLOBALS()) - && PyDict_CheckExact(BUILTINS())) - { - _PyFrame_SetStackPointer(frame, stack_pointer); - v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), - (PyDictObject *)BUILTINS(), - name); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (v_o == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_JUMP_TO_LABEL(error); - } - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(GLOBALS(), name, &v_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - if (v_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(BUILTINS(), name, &v_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - if (v_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - } - } - } - v = PyStackRef_FromPyObjectSteal(v_o); - stack_pointer[0] = v; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_GLOBAL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_GLOBAL; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL); - PREDICTED_TRACING_LOAD_GLOBAL:; - _Py_CODEUNIT* const this_instr = next_instr - 5; - (void)this_instr; - opcode = LOAD_GLOBAL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef *res; - _PyStackRef *null; - // _SPECIALIZE_LOAD_GLOBAL - { - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(LOAD_GLOBAL); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 1 cache entry */ - /* Skip 1 cache entry */ - /* Skip 1 cache entry */ - // _LOAD_GLOBAL - { - res = &stack_pointer[0]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(*res)) { - TRACING_JUMP_TO_LABEL(error); - } - } - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[1]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_GLOBAL_BUILTIN) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_GLOBAL_BUILTIN; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); - opcode = LOAD_GLOBAL_BUILTIN; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); - _PyStackRef res; - _PyStackRef *null; - /* Skip 1 cache entry */ - // _GUARD_GLOBALS_VERSION - { - uint16_t version = read_u16(&this_instr[2].cache); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - if (!PyDict_CheckExact(dict)) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); - } - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); - } - assert(DK_IS_UNICODE(keys)); - } - // _LOAD_GLOBAL_BUILTINS - { - uint16_t version = read_u16(&this_instr[3].cache); - uint16_t index = read_u16(&this_instr[4].cache); - PyDictObject *dict = (PyDictObject *)BUILTINS(); - if (!PyDict_CheckExact(dict)) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); - } - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); - } - assert(DK_IS_UNICODE(keys)); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); - PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - if (res_o == NULL) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); - } - #if Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - if (!increfed) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); - } - #else - res = PyStackRef_FromPyObjectNew(res_o); - #endif - STAT_INC(LOAD_GLOBAL, hit); - } - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[1]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer[0] = res; - stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_GLOBAL_MODULE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_GLOBAL_MODULE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); - opcode = LOAD_GLOBAL_MODULE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); - _PyStackRef res; - _PyStackRef *null; - /* Skip 1 cache entry */ - // _NOP - { - } - // _LOAD_GLOBAL_MODULE - { - uint16_t version = read_u16(&this_instr[2].cache); - uint16_t index = read_u16(&this_instr[4].cache); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - if (!PyDict_CheckExact(dict)) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); - } - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); - } - assert(DK_IS_UNICODE(keys)); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); - assert(index < DK_SIZE(keys)); - PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - if (res_o == NULL) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); - } - #if Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - if (!increfed) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); - } - #else - res = PyStackRef_FromPyObjectNew(res_o); - #endif - STAT_INC(LOAD_GLOBAL, hit); - } - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[1]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer[0] = res; - stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_LOCALS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_LOCALS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_LOCALS); - opcode = LOAD_LOCALS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef locals; - PyObject *l = LOCALS(); - if (l == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_SystemError, - "no locals found"); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - locals = PyStackRef_FromPyObjectNew(l); - stack_pointer[0] = locals; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_NAME) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_NAME; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_NAME); - opcode = LOAD_NAME; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef v; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *v_o = _PyEval_LoadName(tstate, frame, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (v_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - v = PyStackRef_FromPyObjectSteal(v_o); - stack_pointer[0] = v; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_SMALL_INT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_SMALL_INT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_SMALL_INT); - opcode = LOAD_SMALL_INT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - assert(oparg < _PY_NSMALLPOSINTS); - PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; - value = PyStackRef_FromPyObjectBorrow(obj); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_SPECIAL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_SPECIAL; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_SPECIAL); - opcode = LOAD_SPECIAL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef self; - _PyStackRef *method_and_self; - // _INSERT_NULL - { - self = stack_pointer[-1]; - method_and_self = &stack_pointer[-1]; - method_and_self[1] = self; - method_and_self[0] = PyStackRef_NULL; - } - // _LOAD_SPECIAL - { - method_and_self = &stack_pointer[-1]; - PyObject *name = _Py_SpecialMethods[oparg].name; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyObject_LookupSpecialMethod(name, method_and_self); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err <= 0) { - if (err == 0) { - PyObject *owner = PyStackRef_AsPyObjectBorrow(method_and_self[1]); - _PyFrame_SetStackPointer(frame, stack_pointer); - const char *errfmt = _PyEval_SpecialMethodCanSuggest(owner, oparg) - ? _Py_SpecialMethods[oparg].error_suggestion - : _Py_SpecialMethods[oparg].error; - stack_pointer = _PyFrame_GetStackPointer(frame); - assert(!_PyErr_Occurred(tstate)); - assert(errfmt != NULL); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, errfmt, owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_SUPER_ATTR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_SUPER_ATTR; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR); - PREDICTED_TRACING_LOAD_SUPER_ATTR:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - opcode = LOAD_SUPER_ATTR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - opcode = LOAD_SUPER_ATTR; - _PyStackRef global_super_st; - _PyStackRef class_st; - _PyStackRef self_st; - _PyStackRef attr; - _PyStackRef *null; - // _SPECIALIZE_LOAD_SUPER_ATTR - { - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - int load_method = oparg & 1; - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _LOAD_SUPER_ATTR - { - self_st = stack_pointer[-1]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = self_st; - self_st = PyStackRef_NULL; - stack_pointer[-1] = self_st; - PyStackRef_CLOSE(tmp); - tmp = class_st; - class_st = PyStackRef_NULL; - stack_pointer[-2] = class_st; - PyStackRef_CLOSE(tmp); - tmp = global_super_st; - global_super_st = PyStackRef_NULL; - stack_pointer[-3] = global_super_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - } - PyObject *stack[] = {class, self}; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - if (super == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(super); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = self_st; - self_st = PyStackRef_NULL; - stack_pointer[-1] = self_st; - PyStackRef_CLOSE(tmp); - tmp = class_st; - class_st = PyStackRef_NULL; - stack_pointer[-2] = class_st; - PyStackRef_CLOSE(tmp); - tmp = global_super_st; - global_super_st = PyStackRef_NULL; - stack_pointer[-3] = global_super_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - if (super == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = PyObject_GetAttr(super, name); - Py_DECREF(super); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - attr = PyStackRef_FromPyObjectSteal(attr_o); - } - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[1]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer[0] = attr; - stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_SUPER_ATTR_ATTR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_SUPER_ATTR_ATTR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); - opcode = LOAD_SUPER_ATTR_ATTR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); - _PyStackRef global_super_st; - _PyStackRef class_st; - _PyStackRef self_st; - _PyStackRef attr_st; - /* Skip 1 cache entry */ - self_st = stack_pointer[-1]; - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - assert(!(oparg & 1)); - if (global_super != (PyObject *)&PySuper_Type) { - UPDATE_MISS_STATS(LOAD_SUPER_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_SUPER_ATTR); - } - if (!PyType_Check(class)) { - UPDATE_MISS_STATS(LOAD_SUPER_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_SUPER_ATTR); - } - STAT_INC(LOAD_SUPER_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); - _PyStackRef tmp = self_st; - self_st = PyStackRef_NULL; - stack_pointer[-1] = self_st; - PyStackRef_CLOSE(tmp); - tmp = class_st; - class_st = PyStackRef_NULL; - stack_pointer[-2] = class_st; - PyStackRef_CLOSE(tmp); - tmp = global_super_st; - global_super_st = PyStackRef_NULL; - stack_pointer[-3] = global_super_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - if (attr == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - attr_st = PyStackRef_FromPyObjectSteal(attr); - stack_pointer[0] = attr_st; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_SUPER_ATTR_METHOD) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_SUPER_ATTR_METHOD; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); - opcode = LOAD_SUPER_ATTR_METHOD; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); - _PyStackRef global_super_st; - _PyStackRef class_st; - _PyStackRef self_st; - _PyStackRef attr; - _PyStackRef self_or_null; - /* Skip 1 cache entry */ - self_st = stack_pointer[-1]; - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - assert(oparg & 1); - if (global_super != (PyObject *)&PySuper_Type) { - UPDATE_MISS_STATS(LOAD_SUPER_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_SUPER_ATTR); - } - if (!PyType_Check(class)) { - UPDATE_MISS_STATS(LOAD_SUPER_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_SUPER_ATTR); - } - STAT_INC(LOAD_SUPER_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - PyTypeObject *cls = (PyTypeObject *)class; - int method_found = 0; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = _PySuper_Lookup(cls, self, name, - Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - if (method_found) { - self_or_null = self_st; - } else { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(self_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - self_or_null = PyStackRef_NULL; - stack_pointer += 1; - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = global_super_st; - global_super_st = self_or_null; - stack_pointer[-2] = global_super_st; - PyStackRef_CLOSE(tmp); - tmp = class_st; - class_st = PyStackRef_NULL; - stack_pointer[-1] = class_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - attr = PyStackRef_FromPyObjectSteal(attr_o); - stack_pointer[0] = attr; - stack_pointer[1] = self_or_null; - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(MAKE_CELL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = MAKE_CELL; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MAKE_CELL); - opcode = MAKE_CELL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - PyObject *cell = PyCell_New(initial); - if (cell == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = PyStackRef_FromPyObjectSteal(cell); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(MAKE_FUNCTION) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = MAKE_FUNCTION; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MAKE_FUNCTION); - opcode = MAKE_FUNCTION; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef codeobj_st; - _PyStackRef func; - codeobj_st = stack_pointer[-1]; - PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyFunctionObject *func_obj = (PyFunctionObject *) - PyFunction_New(codeobj, GLOBALS()); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(codeobj_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (func_obj == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - _PyFunction_SetVersion( - func_obj, ((PyCodeObject *)codeobj)->co_version); - func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); - stack_pointer[0] = func; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(MAP_ADD) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = MAP_ADD; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MAP_ADD); - opcode = MAP_ADD; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef dict_st; - _PyStackRef key; - _PyStackRef value; - value = stack_pointer[-1]; - key = stack_pointer[-2]; - dict_st = stack_pointer[-3 - (oparg - 1)]; - PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - assert(PyDict_CheckExact(dict)); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyDict_SetItem_Take2( - (PyDictObject *)dict, - PyStackRef_AsPyObjectSteal(key), - PyStackRef_AsPyObjectSteal(value) - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(pop_2_error); - } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(MATCH_CLASS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = MATCH_CLASS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_CLASS); - opcode = MATCH_CLASS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef subject; - _PyStackRef type; - _PyStackRef names; - _PyStackRef attrs; - names = stack_pointer[-1]; - type = stack_pointer[-2]; - subject = stack_pointer[-3]; - assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names))); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attrs_o = _PyEval_MatchClass(tstate, - PyStackRef_AsPyObjectBorrow(subject), - PyStackRef_AsPyObjectBorrow(type), oparg, - PyStackRef_AsPyObjectBorrow(names)); - _PyStackRef tmp = names; - names = PyStackRef_NULL; - stack_pointer[-1] = names; - PyStackRef_CLOSE(tmp); - tmp = type; - type = PyStackRef_NULL; - stack_pointer[-2] = type; - PyStackRef_CLOSE(tmp); - tmp = subject; - subject = PyStackRef_NULL; - stack_pointer[-3] = subject; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - if (attrs_o) { - assert(PyTuple_CheckExact(attrs_o)); - attrs = PyStackRef_FromPyObjectSteal(attrs_o); - } - else { - if (_PyErr_Occurred(tstate)) { - TRACING_JUMP_TO_LABEL(error); - } - attrs = PyStackRef_None; - } - stack_pointer[0] = attrs; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(MATCH_KEYS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = MATCH_KEYS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_KEYS); - opcode = MATCH_KEYS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef subject; - _PyStackRef keys; - _PyStackRef values_or_none; - keys = stack_pointer[-1]; - subject = stack_pointer[-2]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, - PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (values_or_none_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); - stack_pointer[0] = values_or_none; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(MATCH_MAPPING) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = MATCH_MAPPING; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_MAPPING); - opcode = MATCH_MAPPING; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef subject; - _PyStackRef res; - subject = stack_pointer[-1]; - int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; - res = match ? PyStackRef_True : PyStackRef_False; - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(MATCH_SEQUENCE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = MATCH_SEQUENCE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_SEQUENCE); - opcode = MATCH_SEQUENCE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef subject; - _PyStackRef res; - subject = stack_pointer[-1]; - int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; - res = match ? PyStackRef_True : PyStackRef_False; - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(NOP) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = NOP; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(NOP); - opcode = NOP; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - TRACING_DISPATCH(); - } - - TRACING_TARGET(NOT_TAKEN) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = NOT_TAKEN; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(NOT_TAKEN); - opcode = NOT_TAKEN; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - TRACING_DISPATCH(); - } - - TRACING_TARGET(POP_EXCEPT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = POP_EXCEPT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(POP_EXCEPT); - opcode = POP_EXCEPT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef exc_value; - exc_value = stack_pointer[-1]; - _PyErr_StackItem *exc_info = tstate->exc_info; - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_XSETREF(exc_info->exc_value, - PyStackRef_IsNone(exc_value) - ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(POP_ITER) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = POP_ITER; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(POP_ITER); - opcode = POP_ITER; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef iter; - _PyStackRef index_or_null; - index_or_null = stack_pointer[-1]; - iter = stack_pointer[-2]; - (void)index_or_null; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iter); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(POP_JUMP_IF_FALSE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = POP_JUMP_IF_FALSE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_FALSE); - opcode = POP_JUMP_IF_FALSE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef cond; - /* Skip 1 cache entry */ - cond = stack_pointer[-1]; - assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsFalse(cond); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(POP_JUMP_IF_NONE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = POP_JUMP_IF_NONE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_NONE); - opcode = POP_JUMP_IF_NONE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - _PyStackRef b; - _PyStackRef cond; - /* Skip 1 cache entry */ - // _IS_NONE - { - value = stack_pointer[-1]; - if (PyStackRef_IsNone(value)) { - b = PyStackRef_True; - } - else { - b = PyStackRef_False; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = value; - value = b; - stack_pointer[-1] = value; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - // _POP_JUMP_IF_TRUE - { - cond = b; - assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsTrue(cond); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(POP_JUMP_IF_NOT_NONE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = POP_JUMP_IF_NOT_NONE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); - opcode = POP_JUMP_IF_NOT_NONE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - _PyStackRef b; - _PyStackRef cond; - /* Skip 1 cache entry */ - // _IS_NONE - { - value = stack_pointer[-1]; - if (PyStackRef_IsNone(value)) { - b = PyStackRef_True; - } - else { - b = PyStackRef_False; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = value; - value = b; - stack_pointer[-1] = value; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - // _POP_JUMP_IF_FALSE - { - cond = b; - assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsFalse(cond); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(POP_JUMP_IF_TRUE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = POP_JUMP_IF_TRUE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_TRUE); - opcode = POP_JUMP_IF_TRUE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef cond; - /* Skip 1 cache entry */ - cond = stack_pointer[-1]; - assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsTrue(cond); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(POP_TOP) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = POP_TOP; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(POP_TOP); - opcode = POP_TOP; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - value = stack_pointer[-1]; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(PUSH_EXC_INFO) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = PUSH_EXC_INFO; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(PUSH_EXC_INFO); - opcode = PUSH_EXC_INFO; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef exc; - _PyStackRef prev_exc; - _PyStackRef new_exc; - exc = stack_pointer[-1]; - _PyErr_StackItem *exc_info = tstate->exc_info; - if (exc_info->exc_value != NULL) { - prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value); - } - else { - prev_exc = PyStackRef_None; - } - assert(PyStackRef_ExceptionInstanceCheck(exc)); - exc_info->exc_value = PyStackRef_AsPyObjectNew(exc); - new_exc = exc; - stack_pointer[-1] = prev_exc; - stack_pointer[0] = new_exc; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(PUSH_NULL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = PUSH_NULL; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(PUSH_NULL); - opcode = PUSH_NULL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef res; - res = PyStackRef_NULL; - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(RAISE_VARARGS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = RAISE_VARARGS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RAISE_VARARGS); - opcode = RAISE_VARARGS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef *args; - args = &stack_pointer[-oparg]; - assert(oparg < 3); - PyObject *cause = oparg == 2 ? PyStackRef_AsPyObjectSteal(args[1]) : NULL; - PyObject *exc = oparg > 0 ? PyStackRef_AsPyObjectSteal(args[0]) : NULL; - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = do_raise(tstate, exc, cause); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - assert(oparg == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_reraise(tstate, frame, this_instr); - TRACING_JUMP_TO_LABEL(exception_unwind); - } - TRACING_JUMP_TO_LABEL(error); - } - - TRACING_TARGET(RERAISE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = RERAISE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RERAISE); - opcode = RERAISE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef *values; - _PyStackRef exc_st; - exc_st = stack_pointer[-1]; - values = &stack_pointer[-1 - oparg]; - PyObject *exc = PyStackRef_AsPyObjectSteal(exc_st); - assert(oparg >= 0 && oparg <= 2); - if (oparg) { - frame->instr_ptr = _PyFrame_GetBytecode(frame) + PyStackRef_UntagInt(values[0]); - } - assert(exc && PyExceptionInstance_Check(exc)); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, this_instr); - TRACING_JUMP_TO_LABEL(exception_unwind); - } - - TRACING_TARGET(RESERVED) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = RESERVED; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RESERVED); - opcode = RESERVED; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - assert(0 && "Executing RESERVED instruction."); - Py_FatalError("Executing RESERVED instruction."); - TRACING_DISPATCH(); - } - - TRACING_TARGET(RESUME) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = RESUME; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RESUME); - PREDICTED_TRACING_RESUME:; - _Py_CODEUNIT* const this_instr = next_instr - 1; - (void)this_instr; - opcode = RESUME; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - // _LOAD_BYTECODE - { - #ifdef Py_GIL_DISABLED - if (frame->tlbc_index != - ((_PyThreadStateImpl *)tstate)->tlbc_index) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_CODEUNIT *bytecode = - _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (bytecode == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); - frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; - frame->instr_ptr = bytecode + off; - next_instr = frame->instr_ptr; - TRACING_DISPATCH(); - } - #endif - } - // _MAYBE_INSTRUMENT - { - #ifdef Py_GIL_DISABLED - - int check_instrumentation = 1; - #else - int check_instrumentation = (tstate->tracing == 0); - #endif - if (check_instrumentation) { - uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - if (code_version != global_version) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - next_instr = this_instr; - TRACING_DISPATCH(); - } - } - } - // _QUICKEN_RESUME - { - #if ENABLE_SPECIALIZATION_FT - if (tstate->tracing == 0 && this_instr->op.code == RESUME) { - FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); - } - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _CHECK_PERIODIC_IF_NOT_YIELD_FROM - { - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(RESUME_CHECK) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = RESUME_CHECK; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RESUME_CHECK); - opcode = RESUME_CHECK; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(0 == 0, "incorrect cache size"); - #if defined(__EMSCRIPTEN__) - if (_Py_emscripten_signal_clock == 0) { - UPDATE_MISS_STATS(RESUME); - assert(_PyOpcode_Deopt[opcode] == (RESUME)); - JUMP_TO_PREDICTED(TRACING_RESUME); - } - _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; - #endif - uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); - uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - assert((version & _PY_EVAL_EVENTS_MASK) == 0); - if (eval_breaker != version) { - UPDATE_MISS_STATS(RESUME); - assert(_PyOpcode_Deopt[opcode] == (RESUME)); - JUMP_TO_PREDICTED(TRACING_RESUME); - } - #ifdef Py_GIL_DISABLED - if (frame->tlbc_index != - ((_PyThreadStateImpl *)tstate)->tlbc_index) { - UPDATE_MISS_STATS(RESUME); - assert(_PyOpcode_Deopt[opcode] == (RESUME)); - JUMP_TO_PREDICTED(TRACING_RESUME); - } - #endif - TRACING_DISPATCH(); - } - - TRACING_TARGET(RETURN_GENERATOR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = RETURN_GENERATOR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RETURN_GENERATOR); - opcode = RETURN_GENERATOR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef res; - assert(PyStackRef_FunctionCheck(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (gen == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - LOAD_IP(frame->return_offset); - #if TIER_TWO - frame->instr_ptr += (frame->return_offset); - #endif - stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); - LLTRACE_RESUME_FRAME(); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(RETURN_VALUE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = RETURN_VALUE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RETURN_VALUE); - opcode = RETURN_VALUE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef retval; - _PyStackRef res; - retval = stack_pointer[-1]; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - _PyStackRef temp = PyStackRef_MakeHeapSafe(retval); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(STACK_LEVEL() == 0); - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - stack_pointer = _PyFrame_GetStackPointer(frame); - LOAD_IP(frame->return_offset); - #if TIER_TWO - frame->instr_ptr += frame->return_offset; - #endif - res = temp; - LLTRACE_RESUME_FRAME(); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(SEND) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = SEND; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(SEND); - PREDICTED_TRACING_SEND:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - opcode = SEND; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef receiver; - _PyStackRef v; - _PyStackRef retval; - // _SPECIALIZE_SEND - { - receiver = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_Send(receiver, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(SEND); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _SEND - { - v = stack_pointer[-1]; - PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); - PyObject *retval_o; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - if ((tstate->interp->eval_frame == NULL) && - (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && - ((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING) - { - PyGenObject *gen = (PyGenObject *)receiver_o; - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - _PyFrame_StackPush(gen_frame, PyStackRef_MakeHeapSafe(v)); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - assert( 2u + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)( 2u + oparg); - assert(gen_frame->previous == NULL); - gen_frame->previous = frame; - TRACING_DISPATCH_INLINED(gen_frame); - } - if (PyStackRef_IsNone(v) && PyIter_Check(receiver_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - retval_o = PyObject_CallMethodOneArg(receiver_o, - &_Py_ID(send), - PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - if (retval_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_MonitorRaise(tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyGen_FetchStopIterationValue(&retval_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err == 0) { - assert(retval_o != NULL); - JUMPBY(oparg); - } - else { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); - retval = PyStackRef_FromPyObjectSteal(retval_o); - } - stack_pointer[0] = retval; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(SEND_GEN) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = SEND_GEN; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(SEND_GEN); - opcode = SEND_GEN; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); - _PyStackRef receiver; - _PyStackRef v; - _PyStackRef gen_frame; - _PyStackRef new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(SEND); - assert(_PyOpcode_Deopt[opcode] == (SEND)); - JUMP_TO_PREDICTED(TRACING_SEND); - } - } - // _SEND_GEN_FRAME - { - v = stack_pointer[-1]; - receiver = stack_pointer[-2]; - PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); - if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { - UPDATE_MISS_STATS(SEND); - assert(_PyOpcode_Deopt[opcode] == (SEND)); - JUMP_TO_PREDICTED(TRACING_SEND); - } - if (gen->gi_frame_state >= FRAME_EXECUTING) { - UPDATE_MISS_STATS(SEND); - assert(_PyOpcode_Deopt[opcode] == (SEND)); - JUMP_TO_PREDICTED(TRACING_SEND); - } - STAT_INC(SEND, hit); - _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; - _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - assert( 2u + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)( 2u + oparg); - pushed_frame->previous = frame; - gen_frame = PyStackRef_Wrap(pushed_frame); - } - // _PUSH_FRAME - { - new_frame = gen_frame; - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(SETUP_ANNOTATIONS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = SETUP_ANNOTATIONS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SETUP_ANNOTATIONS); - opcode = SETUP_ANNOTATIONS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - PyObject *ann_dict; - if (LOCALS() == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_SystemError, - "no locals found when setting up annotations"); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - if (ann_dict == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - ann_dict = PyDict_New(); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (ann_dict == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), - ann_dict); - Py_DECREF(ann_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(ann_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(SET_ADD) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = SET_ADD; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SET_ADD); - opcode = SET_ADD; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef set; - _PyStackRef v; - v = stack_pointer[-1]; - set = stack_pointer[-2 - (oparg-1)]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PySet_AddTakeRef((PySetObject *)PyStackRef_AsPyObjectBorrow(set), - PyStackRef_AsPyObjectSteal(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(pop_1_error); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(SET_FUNCTION_ATTRIBUTE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = SET_FUNCTION_ATTRIBUTE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); - opcode = SET_FUNCTION_ATTRIBUTE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef attr_st; - _PyStackRef func_in; - _PyStackRef func_out; - func_in = stack_pointer[-1]; - attr_st = stack_pointer[-2]; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_in); - PyObject *attr = PyStackRef_AsPyObjectSteal(attr_st); - func_out = func_in; - assert(PyFunction_Check(func)); - size_t offset = _Py_FunctionAttributeOffsets[oparg]; - assert(offset != 0); - PyObject **ptr = (PyObject **)(((char *)func) + offset); - assert(*ptr == NULL); - *ptr = attr; - stack_pointer[-2] = func_out; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(SET_UPDATE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = SET_UPDATE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SET_UPDATE); - opcode = SET_UPDATE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef set; - _PyStackRef iterable; - iterable = stack_pointer[-1]; - set = stack_pointer[-2 - (oparg-1)]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), - PyStackRef_AsPyObjectBorrow(iterable)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_ATTR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_ATTR; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR); - PREDICTED_TRACING_STORE_ATTR:; - _Py_CODEUNIT* const this_instr = next_instr - 5; - (void)this_instr; - opcode = STORE_ATTR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef owner; - _PyStackRef v; - // _SPECIALIZE_STORE_ATTR - { - owner = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_StoreAttr(owner, next_instr, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(STORE_ATTR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 3 cache entries */ - // _STORE_ATTR - { - v = stack_pointer[-2]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), - name, PyStackRef_AsPyObjectBorrow(v)); - _PyStackRef tmp = owner; - owner = PyStackRef_NULL; - stack_pointer[-1] = owner; - PyStackRef_CLOSE(tmp); - tmp = v; - v = PyStackRef_NULL; - stack_pointer[-2] = v; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_ATTR_INSTANCE_VALUE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_ATTR_INSTANCE_VALUE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); - opcode = STORE_ATTR_INSTANCE_VALUE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef value; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION_AND_LOCK - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(type_version != 0); - if (!LOCK_OBJECT(owner_o)) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - PyTypeObject *tp = Py_TYPE(owner_o); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UNLOCK_OBJECT(owner_o); - if (true) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - } - } - // _GUARD_DORV_NO_DICT - { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_dictoffset < 0); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (_PyObject_GetManagedDict(owner_o) || - !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { - UNLOCK_OBJECT(owner_o); - if (true) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - } - } - // _STORE_ATTR_INSTANCE_VALUE - { - value = stack_pointer[-2]; - uint16_t offset = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - STAT_INC(STORE_ATTR, hit); - assert(_PyObject_GetManagedDict(owner_o) == NULL); - PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); - PyObject *old_value = *value_ptr; - FT_ATOMIC_STORE_PTR_RELEASE(*value_ptr, PyStackRef_AsPyObjectSteal(value)); - if (old_value == NULL) { - PyDictValues *values = _PyObject_InlineValues(owner_o); - Py_ssize_t index = value_ptr - values->values; - _PyDictValues_AddToInsertionOrder(values, index); - } - UNLOCK_OBJECT(owner_o); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - Py_XDECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_ATTR_SLOT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_ATTR_SLOT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_SLOT); - opcode = STORE_ATTR_SLOT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef value; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - } - // _STORE_ATTR_SLOT - { - value = stack_pointer[-2]; - uint16_t index = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - if (!LOCK_OBJECT(owner_o)) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - char *addr = (char *)owner_o + index; - STAT_INC(STORE_ATTR, hit); - PyObject *old_value = *(PyObject **)addr; - FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); - UNLOCK_OBJECT(owner_o); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - Py_XDECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_ATTR_WITH_HINT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_ATTR_WITH_HINT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); - opcode = STORE_ATTR_WITH_HINT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef value; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - } - // _STORE_ATTR_WITH_HINT - { - value = stack_pointer[-2]; - uint16_t hint = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - if (dict == NULL) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - if (!LOCK_OBJECT(dict)) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - assert(PyDict_CheckExact((PyObject *)dict)); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (hint >= (size_t)dict->ma_keys->dk_nentries || - !DK_IS_UNICODE(dict->ma_keys)) { - UNLOCK_OBJECT(dict); - if (true) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - } - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - if (ep->me_key != name) { - UNLOCK_OBJECT(dict); - if (true) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - } - PyObject *old_value = ep->me_value; - if (old_value == NULL) { - UNLOCK_OBJECT(dict); - if (true) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value)); - UNLOCK_OBJECT(dict); - STAT_INC(STORE_ATTR, hit); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - Py_XDECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_DEREF) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_DEREF; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_DEREF); - opcode = STORE_DEREF; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef v; - v = stack_pointer[-1]; - PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_FAST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_FAST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST); - opcode = STORE_FAST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - value = stack_pointer[-1]; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_FAST_LOAD_FAST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_FAST_LOAD_FAST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); - opcode = STORE_FAST_LOAD_FAST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value1; - _PyStackRef value2; - value1 = stack_pointer[-1]; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - _PyStackRef tmp = GETLOCAL(oparg1); - GETLOCAL(oparg1) = value1; - value2 = PyStackRef_DUP(GETLOCAL(oparg2)); - stack_pointer[-1] = value2; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_FAST_STORE_FAST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_FAST_STORE_FAST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST_STORE_FAST); - opcode = STORE_FAST_STORE_FAST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value2; - _PyStackRef value1; - value1 = stack_pointer[-1]; - value2 = stack_pointer[-2]; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - _PyStackRef tmp = GETLOCAL(oparg1); - GETLOCAL(oparg1) = value1; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - tmp = GETLOCAL(oparg2); - GETLOCAL(oparg2) = value2; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_GLOBAL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_GLOBAL; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_GLOBAL); - opcode = STORE_GLOBAL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef v; - v = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_NAME) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_NAME; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_NAME); - opcode = STORE_NAME; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef v; - v = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *ns = LOCALS(); - int err; - if (ns == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_SystemError, - "no locals found when storing %R", name); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - if (PyDict_CheckExact(ns)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_SLICE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_SLICE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_SLICE); - opcode = STORE_SLICE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef v; - _PyStackRef container; - _PyStackRef start; - _PyStackRef stop; - // _SPECIALIZE_STORE_SLICE - { - #if ENABLE_SPECIALIZATION - OPCODE_DEFERRED_INC(STORE_SLICE); - #endif /* ENABLE_SPECIALIZATION */ - } - // _STORE_SLICE - { - stop = stack_pointer[-1]; - start = stack_pointer[-2]; - container = stack_pointer[-3]; - v = stack_pointer[-4]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), - PyStackRef_AsPyObjectSteal(stop)); - stack_pointer = _PyFrame_GetStackPointer(frame); - int err; - if (slice == NULL) { - err = 1; - } - else { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); - Py_DECREF(slice); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += 2; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = container; - container = PyStackRef_NULL; - stack_pointer[-3] = container; - PyStackRef_CLOSE(tmp); - tmp = v; - v = PyStackRef_NULL; - stack_pointer[-4] = v; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -4; - assert(WITHIN_STACK_BOUNDS()); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_SUBSCR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_SUBSCR; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR); - PREDICTED_TRACING_STORE_SUBSCR:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - opcode = STORE_SUBSCR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef container; - _PyStackRef sub; - _PyStackRef v; - // _SPECIALIZE_STORE_SUBSCR - { - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_StoreSubscr(container, sub, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(STORE_SUBSCR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _STORE_SUBSCR - { - v = stack_pointer[-3]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); - _PyStackRef tmp = sub; - sub = PyStackRef_NULL; - stack_pointer[-1] = sub; - PyStackRef_CLOSE(tmp); - tmp = container; - container = PyStackRef_NULL; - stack_pointer[-2] = container; - PyStackRef_CLOSE(tmp); - tmp = v; - v = PyStackRef_NULL; - stack_pointer[-3] = v; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_SUBSCR_DICT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_SUBSCR_DICT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR_DICT); - opcode = STORE_SUBSCR_DICT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); - _PyStackRef nos; - _PyStackRef value; - _PyStackRef dict_st; - _PyStackRef sub; - // _GUARD_NOS_DICT - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyDict_CheckExact(o)) { - UPDATE_MISS_STATS(STORE_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); - } - } - /* Skip 1 cache entry */ - // _STORE_SUBSCR_DICT - { - sub = stack_pointer[-1]; - dict_st = nos; - value = stack_pointer[-3]; - PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - assert(PyDict_CheckExact(dict)); - STAT_INC(STORE_SUBSCR, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyDict_SetItem_Take2((PyDictObject *)dict, - PyStackRef_AsPyObjectSteal(sub), - PyStackRef_AsPyObjectSteal(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(dict_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_SUBSCR_LIST_INT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_SUBSCR_LIST_INT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); - opcode = STORE_SUBSCR_LIST_INT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); - _PyStackRef value; - _PyStackRef nos; - _PyStackRef list_st; - _PyStackRef sub_st; - // _GUARD_TOS_INT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { - UPDATE_MISS_STATS(STORE_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); - } - } - // _GUARD_NOS_LIST - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyList_CheckExact(o)) { - UPDATE_MISS_STATS(STORE_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); - } - } - /* Skip 1 cache entry */ - // _STORE_SUBSCR_LIST_INT - { - sub_st = value; - list_st = nos; - value = stack_pointer[-3]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - assert(PyLong_CheckExact(sub)); - assert(PyList_CheckExact(list)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UPDATE_MISS_STATS(STORE_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); - } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (!LOCK_OBJECT(list)) { - UPDATE_MISS_STATS(STORE_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); - } - if (index >= PyList_GET_SIZE(list)) { - UNLOCK_OBJECT(list); - if (true) { - UPDATE_MISS_STATS(STORE_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); - } - } - STAT_INC(STORE_SUBSCR, hit); - PyObject *old_value = PyList_GET_ITEM(list, index); - FT_ATOMIC_STORE_PTR_RELEASE(_PyList_ITEMS(list)[index], - PyStackRef_AsPyObjectSteal(value)); - assert(old_value != NULL); - UNLOCK_OBJECT(list); - PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(list_st); - Py_DECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(SWAP) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = SWAP; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SWAP); - opcode = SWAP; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef bottom; - _PyStackRef top; - top = stack_pointer[-1]; - bottom = stack_pointer[-2 - (oparg-2)]; - _PyStackRef temp = bottom; - bottom = top; - top = temp; - stack_pointer[-2 - (oparg-2)] = bottom; - stack_pointer[-1] = top; - TRACING_DISPATCH(); - } - - TRACING_TARGET(TO_BOOL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = TO_BOOL; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL); - PREDICTED_TRACING_TO_BOOL:; - _Py_CODEUNIT* const this_instr = next_instr - 4; - (void)this_instr; - opcode = TO_BOOL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - _PyStackRef res; - // _SPECIALIZE_TO_BOOL - { - value = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_ToBool(value, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(TO_BOOL); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 2 cache entries */ - // _TO_BOOL - { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - res = err ? PyStackRef_True : PyStackRef_False; - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(TO_BOOL_ALWAYS_TRUE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = TO_BOOL_ALWAYS_TRUE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); - opcode = TO_BOOL_ALWAYS_TRUE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef value; - _PyStackRef res; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(TO_BOOL); - assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - JUMP_TO_PREDICTED(TRACING_TO_BOOL); - } - } - // _REPLACE_WITH_TRUE - { - value = owner; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_True; - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(TO_BOOL_BOOL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = TO_BOOL_BOOL; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_BOOL); - opcode = TO_BOOL_BOOL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef value; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - value = stack_pointer[-1]; - if (!PyStackRef_BoolCheck(value)) { - UPDATE_MISS_STATS(TO_BOOL); - assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - JUMP_TO_PREDICTED(TRACING_TO_BOOL); - } - STAT_INC(TO_BOOL, hit); - TRACING_DISPATCH(); - } - - TRACING_TARGET(TO_BOOL_INT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = TO_BOOL_INT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_INT); - opcode = TO_BOOL_INT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef value; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyLong_CheckExact(value_o)) { - UPDATE_MISS_STATS(TO_BOOL); - assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - JUMP_TO_PREDICTED(TRACING_TO_BOOL); - } - STAT_INC(TO_BOOL, hit); - if (_PyLong_IsZero((PyLongObject *)value_o)) { - assert(_Py_IsImmortal(value_o)); - res = PyStackRef_False; - } - else { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_True; - stack_pointer += 1; - } - stack_pointer[-1] = res; - TRACING_DISPATCH(); - } - - TRACING_TARGET(TO_BOOL_LIST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = TO_BOOL_LIST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_LIST); - opcode = TO_BOOL_LIST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef tos; - _PyStackRef value; - _PyStackRef res; - // _GUARD_TOS_LIST - { - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyList_CheckExact(o)) { - UPDATE_MISS_STATS(TO_BOOL); - assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - JUMP_TO_PREDICTED(TRACING_TO_BOOL); - } - } - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _TO_BOOL_LIST - { - value = tos; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - assert(PyList_CheckExact(value_o)); - STAT_INC(TO_BOOL, hit); - res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = value; - value = res; - stack_pointer[-1] = value; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(TO_BOOL_NONE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = TO_BOOL_NONE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_NONE); - opcode = TO_BOOL_NONE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef value; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - value = stack_pointer[-1]; - if (!PyStackRef_IsNone(value)) { - UPDATE_MISS_STATS(TO_BOOL); - assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - JUMP_TO_PREDICTED(TRACING_TO_BOOL); - } - STAT_INC(TO_BOOL, hit); - res = PyStackRef_False; - stack_pointer[-1] = res; - TRACING_DISPATCH(); - } - - TRACING_TARGET(TO_BOOL_STR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = TO_BOOL_STR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_STR); - opcode = TO_BOOL_STR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef value; - _PyStackRef res; - // _GUARD_TOS_UNICODE - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyUnicode_CheckExact(value_o)) { - UPDATE_MISS_STATS(TO_BOOL); - assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - JUMP_TO_PREDICTED(TRACING_TO_BOOL); - } - } - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _TO_BOOL_STR - { - STAT_INC(TO_BOOL, hit); - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (value_o == &_Py_STR(empty)) { - assert(_Py_IsImmortal(value_o)); - res = PyStackRef_False; - } - else { - assert(Py_SIZE(value_o)); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_True; - stack_pointer += 1; - } - } - stack_pointer[-1] = res; - TRACING_DISPATCH(); - } - - TRACING_TARGET(UNARY_INVERT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = UNARY_INVERT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_INVERT); - opcode = UNARY_INVERT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(UNARY_NEGATIVE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = UNARY_NEGATIVE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_NEGATIVE); - opcode = UNARY_NEGATIVE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(UNARY_NOT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = UNARY_NOT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_NOT); - opcode = UNARY_NOT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - assert(PyStackRef_BoolCheck(value)); - res = PyStackRef_IsFalse(value) - ? PyStackRef_True : PyStackRef_False; - stack_pointer[-1] = res; - TRACING_DISPATCH(); - } - - TRACING_TARGET(UNPACK_EX) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = UNPACK_EX; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNPACK_EX); - opcode = UNPACK_EX; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef seq; - _PyStackRef *top; - seq = stack_pointer[-1]; - top = &stack_pointer[(oparg & 0xFF) + (oparg >> 8)]; - PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg & 0xFF, oparg >> 8, top); - Py_DECREF(seq_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res == 0) { - TRACING_JUMP_TO_LABEL(error); - } - stack_pointer += 1 + (oparg & 0xFF) + (oparg >> 8); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(UNPACK_SEQUENCE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = UNPACK_SEQUENCE; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE); - PREDICTED_TRACING_UNPACK_SEQUENCE:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - opcode = UNPACK_SEQUENCE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef seq; - _PyStackRef *top; - // _SPECIALIZE_UNPACK_SEQUENCE - { - seq = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_UnpackSequence(seq, next_instr, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); - } - OPCODE_DEFERRED_INC(UNPACK_SEQUENCE); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - (void)seq; - (void)counter; - } - // _UNPACK_SEQUENCE - { - top = &stack_pointer[-1 + oparg]; - PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg, -1, top); - Py_DECREF(seq_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res == 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - stack_pointer += oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(UNPACK_SEQUENCE_LIST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = UNPACK_SEQUENCE_LIST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); - opcode = UNPACK_SEQUENCE_LIST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - _PyStackRef tos; - _PyStackRef seq; - _PyStackRef *values; - // _GUARD_TOS_LIST - { - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyList_CheckExact(o)) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); - } - } - /* Skip 1 cache entry */ - // _UNPACK_SEQUENCE_LIST - { - seq = tos; - values = &stack_pointer[-1]; - PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - assert(PyList_CheckExact(seq_o)); - if (!LOCK_OBJECT(seq_o)) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); - } - if (PyList_GET_SIZE(seq_o) != oparg) { - UNLOCK_OBJECT(seq_o); - if (true) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); - } - } - STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyList_ITEMS(seq_o); - for (int i = oparg; --i >= 0; ) { - *values++ = PyStackRef_FromPyObjectNew(items[i]); - } - UNLOCK_OBJECT(seq_o); - stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(UNPACK_SEQUENCE_TUPLE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = UNPACK_SEQUENCE_TUPLE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); - opcode = UNPACK_SEQUENCE_TUPLE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - _PyStackRef tos; - _PyStackRef seq; - _PyStackRef *values; - // _GUARD_TOS_TUPLE - { - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyTuple_CheckExact(o)) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); - } - } - /* Skip 1 cache entry */ - // _UNPACK_SEQUENCE_TUPLE - { - seq = tos; - values = &stack_pointer[-1]; - PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - assert(PyTuple_CheckExact(seq_o)); - if (PyTuple_GET_SIZE(seq_o) != oparg) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); - } - STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyTuple_ITEMS(seq_o); - for (int i = oparg; --i >= 0; ) { - *values++ = PyStackRef_FromPyObjectNew(items[i]); - } - stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = UNPACK_SEQUENCE_TWO_TUPLE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); - opcode = UNPACK_SEQUENCE_TWO_TUPLE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - _PyStackRef tos; - _PyStackRef seq; - _PyStackRef val1; - _PyStackRef val0; - // _GUARD_TOS_TUPLE - { - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyTuple_CheckExact(o)) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); - } - } - /* Skip 1 cache entry */ - // _UNPACK_SEQUENCE_TWO_TUPLE - { - seq = tos; - assert(oparg == 2); - PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - assert(PyTuple_CheckExact(seq_o)); - if (PyTuple_GET_SIZE(seq_o) != 2) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); - } - STAT_INC(UNPACK_SEQUENCE, hit); - val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); - val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); - stack_pointer[-1] = val1; - stack_pointer[0] = val0; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(WITH_EXCEPT_START) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = WITH_EXCEPT_START; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(WITH_EXCEPT_START); - opcode = WITH_EXCEPT_START; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef exit_func; - _PyStackRef exit_self; - _PyStackRef lasti; - _PyStackRef val; - _PyStackRef res; - val = stack_pointer[-1]; - lasti = stack_pointer[-3]; - exit_self = stack_pointer[-4]; - exit_func = stack_pointer[-5]; - PyObject *exc, *tb; - PyObject *val_o = PyStackRef_AsPyObjectBorrow(val); - PyObject *exit_func_o = PyStackRef_AsPyObjectBorrow(exit_func); - assert(val_o && PyExceptionInstance_Check(val_o)); - exc = PyExceptionInstance_Class(val_o); - PyObject *original_tb = tb = PyException_GetTraceback(val_o); - if (tb == NULL) { - tb = Py_None; - } - assert(PyStackRef_IsTaggedInt(lasti)); - (void)lasti; - PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; - int has_self = !PyStackRef_IsNull(exit_self); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, - (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); - Py_XDECREF(original_tb); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(YIELD_VALUE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = YIELD_VALUE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(YIELD_VALUE); - opcode = YIELD_VALUE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - _PyStackRef retval; - _PyStackRef value; - retval = stack_pointer[-1]; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); - assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); - assert(oparg == 0 || oparg == 1); - gen->gi_frame_state = FRAME_SUSPENDED + oparg; - _PyStackRef temp = retval; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; - assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - #if TIER_ONE - assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || - frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); - #endif - stack_pointer = _PyFrame_GetStackPointer(frame); - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - #if TIER_TWO - frame->instr_ptr += (1 + INLINE_CACHE_ENTRIES_SEND); - #endif - value = PyStackRef_MakeHeapSafe(temp); - LLTRACE_RESUME_FRAME(); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - #endif /* END TRACING INSTRUCTIONS */ /* END INSTRUCTIONS */ #if !_Py_TAIL_CALL_INTERP diff --git a/Python/generated_tracer_cases.c.h b/Python/generated_tracer_cases.c.h new file mode 100644 index 00000000000000..1871af274084fc --- /dev/null +++ b/Python/generated_tracer_cases.c.h @@ -0,0 +1,14100 @@ +// This file is generated by Tools/cases_generator/tracer_generator.py +// from: +// Python/bytecodes.c +// Do not edit! + #define TIER_ONE 1 + #define TRACING_JIT 1 + #ifdef _Py_TIER2 /* BEGIN TRACING INSTRUCTIONS */ + + + TRACING_TARGET(BINARY_OP) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP); + PREDICTED_TRACING_BINARY_OP:; + _Py_CODEUNIT* const this_instr = next_instr - 6; + (void)this_instr; + opcode = BINARY_OP; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef lhs; + _PyStackRef rhs; + _PyStackRef res; + // _SPECIALIZE_BINARY_OP + { + rhs = stack_pointer[-1]; + lhs = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(BINARY_OP); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + assert(NB_ADD <= oparg); + assert(oparg <= NB_OPARG_LAST); + } + /* Skip 4 cache entries */ + // _BINARY_OP + { + PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); + PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); + assert(_PyEval_BinaryOps[oparg]); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = lhs; + lhs = res; + stack_pointer[-2] = lhs; + PyStackRef_CLOSE(tmp); + tmp = rhs; + rhs = PyStackRef_NULL; + stack_pointer[-1] = rhs; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_ADD_FLOAT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_ADD_FLOAT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); + opcode = BINARY_OP_ADD_FLOAT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_TOS_FLOAT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + // _GUARD_NOS_FLOAT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_ADD_FLOAT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval + + ((PyFloatObject *)right_o)->ob_fval; + res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (PyStackRef_IsNull(res)) { + TRACING_JUMP_TO_LABEL(pop_2_error); + } + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_ADD_INT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_ADD_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_ADD_INT); + opcode = BINARY_OP_ADD_INT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + // _GUARD_NOS_INT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!_PyLong_CheckExactAndCompact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_ADD_INT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + STAT_INC(BINARY_OP, hit); + res = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_ADD_UNICODE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_ADD_UNICODE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); + opcode = BINARY_OP_ADD_UNICODE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_TOS_UNICODE + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + // _GUARD_NOS_UNICODE + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_ADD_UNICODE + { + right = value; + left = nos; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyUnicode_CheckExact(left_o)); + assert(PyUnicode_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyUnicode_Concat(left_o, right_o); + PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(pop_2_error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_EXTEND) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_EXTEND; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_EXTEND); + opcode = BINARY_OP_EXTEND; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + /* Skip 1 cache entry */ + // _GUARD_BINARY_OP_EXTEND + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *descr = read_obj(&this_instr[2].cache); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; + assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); + assert(d && d->guard); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = d->guard(left_o, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!res) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + /* Skip -4 cache entry */ + // _BINARY_OP_EXTEND + { + PyObject *descr = read_obj(&this_instr[2].cache); + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); + _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; + STAT_INC(BINARY_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = d->action(left_o, right_o); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_INPLACE_ADD_UNICODE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); + opcode = BINARY_OP_INPLACE_ADD_UNICODE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef left; + _PyStackRef right; + // _GUARD_TOS_UNICODE + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + // _GUARD_NOS_UNICODE + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_INPLACE_ADD_UNICODE + { + right = value; + left = nos; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + assert(PyUnicode_CheckExact(left_o)); + assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(right))); + int next_oparg; + #if TIER_ONE + assert(next_instr->op.code == STORE_FAST); + next_oparg = next_instr->op.arg; + #else + next_oparg = (int)CURRENT_OPERAND0(); + #endif + _PyStackRef *target_local = &GETLOCAL(next_oparg); + assert(PyUnicode_CheckExact(left_o)); + if (PyStackRef_AsPyObjectBorrow(*target_local) != left_o) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + STAT_INC(BINARY_OP, hit); + assert(Py_REFCNT(left_o) >= 2 || !PyStackRef_IsHeapSafe(left)); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); + PyObject *right_o = PyStackRef_AsPyObjectSteal(right); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyUnicode_Append(&temp, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + *target_local = PyStackRef_FromPyObjectSteal(temp); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsNull(*target_local)) { + TRACING_JUMP_TO_LABEL(error); + } + #if TIER_ONE + + assert(next_instr->op.code == STORE_FAST); + SKIP_OVER(1); + #endif + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_MULTIPLY_FLOAT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_MULTIPLY_FLOAT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); + opcode = BINARY_OP_MULTIPLY_FLOAT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_TOS_FLOAT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + // _GUARD_NOS_FLOAT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_MULTIPLY_FLOAT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval * + ((PyFloatObject *)right_o)->ob_fval; + res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (PyStackRef_IsNull(res)) { + TRACING_JUMP_TO_LABEL(pop_2_error); + } + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_MULTIPLY_INT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_MULTIPLY_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); + opcode = BINARY_OP_MULTIPLY_INT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + // _GUARD_NOS_INT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!_PyLong_CheckExactAndCompact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_MULTIPLY_INT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + STAT_INC(BINARY_OP, hit); + res = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_SUBSCR_DICT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBSCR_DICT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_DICT); + opcode = BINARY_OP_SUBSCR_DICT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef nos; + _PyStackRef dict_st; + _PyStackRef sub_st; + _PyStackRef res; + // _GUARD_NOS_DICT + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyDict_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBSCR_DICT + { + sub_st = stack_pointer[-1]; + dict_st = nos; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + assert(PyDict_CheckExact(dict)); + STAT_INC(BINARY_OP, hit); + PyObject *res_o; + _PyFrame_SetStackPointer(frame, stack_pointer); + int rc = PyDict_GetItemRef(dict, sub, &res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (rc == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetKeyError(sub); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = sub_st; + sub_st = PyStackRef_NULL; + stack_pointer[-1] = sub_st; + PyStackRef_CLOSE(tmp); + tmp = dict_st; + dict_st = PyStackRef_NULL; + stack_pointer[-2] = dict_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (rc <= 0) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_SUBSCR_GETITEM) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBSCR_GETITEM; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_GETITEM); + opcode = BINARY_OP_SUBSCR_GETITEM; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef container; + _PyStackRef getitem; + _PyStackRef sub; + _PyStackRef new_frame; + /* Skip 5 cache entries */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + // _BINARY_OP_SUBSCR_CHECK_FUNC + { + container = stack_pointer[-2]; + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); + if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; + PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); + if (getitem_o == NULL) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + assert(PyFunction_Check(getitem_o)); + uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); + if (((PyFunctionObject *)getitem_o)->func_version != cached_version) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); + assert(code->co_argcount == 2); + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + getitem = PyStackRef_FromPyObjectNew(getitem_o); + STAT_INC(BINARY_OP, hit); + } + // _BINARY_OP_SUBSCR_INIT_CALL + { + sub = stack_pointer[-1]; + _PyInterpreterFrame* pushed_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); + pushed_frame->localsplus[0] = container; + pushed_frame->localsplus[1] = sub; + frame->return_offset = 6u ; + new_frame = PyStackRef_Wrap(pushed_frame); + } + // _PUSH_FRAME + { + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_SUBSCR_LIST_INT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBSCR_LIST_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_INT); + opcode = BINARY_OP_SUBSCR_LIST_INT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef list_st; + _PyStackRef sub_st; + _PyStackRef res; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + // _GUARD_NOS_LIST + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyList_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBSCR_LIST_INT + { + sub_st = value; + list_st = nos; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + assert(PyLong_CheckExact(sub)); + assert(PyList_CheckExact(list)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + #ifdef Py_GIL_DISABLED + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + STAT_INC(BINARY_OP, hit); + res = PyStackRef_FromPyObjectSteal(res_o); + #else + if (index >= PyList_GET_SIZE(list)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyList_GET_ITEM(list, index); + assert(res_o != NULL); + res = PyStackRef_FromPyObjectNew(res_o); + #endif + STAT_INC(BINARY_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = list_st; + list_st = res; + stack_pointer[-2] = list_st; + PyStackRef_CLOSE(tmp); + tmp = sub_st; + sub_st = PyStackRef_NULL; + stack_pointer[-1] = sub_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_SUBSCR_LIST_SLICE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBSCR_LIST_SLICE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_SLICE); + opcode = BINARY_OP_SUBSCR_LIST_SLICE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef nos; + _PyStackRef list_st; + _PyStackRef sub_st; + _PyStackRef res; + // _GUARD_TOS_SLICE + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PySlice_Check(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + // _GUARD_NOS_LIST + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyList_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBSCR_LIST_SLICE + { + sub_st = tos; + list_st = nos; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + assert(PySlice_Check(sub)); + assert(PyList_CheckExact(list)); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyList_SliceSubscript(list, sub); + stack_pointer = _PyFrame_GetStackPointer(frame); + STAT_INC(BINARY_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = sub_st; + sub_st = PyStackRef_NULL; + stack_pointer[-1] = sub_st; + PyStackRef_CLOSE(tmp); + tmp = list_st; + list_st = PyStackRef_NULL; + stack_pointer[-2] = list_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_SUBSCR_STR_INT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBSCR_STR_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_STR_INT); + opcode = BINARY_OP_SUBSCR_STR_INT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef str_st; + _PyStackRef sub_st; + _PyStackRef res; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + // _GUARD_NOS_UNICODE + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBSCR_STR_INT + { + sub_st = value; + str_st = nos; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); + assert(PyLong_CheckExact(sub)); + assert(PyUnicode_CheckExact(str)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (PyUnicode_GET_LENGTH(str) <= index) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + Py_UCS4 c = PyUnicode_READ_CHAR(str, index); + if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + STAT_INC(BINARY_OP, hit); + PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(str_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_FromPyObjectBorrow(res_o); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_SUBSCR_TUPLE_INT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBSCR_TUPLE_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBSCR_TUPLE_INT); + opcode = BINARY_OP_SUBSCR_TUPLE_INT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef tuple_st; + _PyStackRef sub_st; + _PyStackRef res; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + // _GUARD_NOS_TUPLE + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyTuple_CheckExact(o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBSCR_TUPLE_INT + { + sub_st = value; + tuple_st = nos; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); + assert(PyLong_CheckExact(sub)); + assert(PyTuple_CheckExact(tuple)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (index >= PyTuple_GET_SIZE(tuple)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + STAT_INC(BINARY_OP, hit); + PyObject *res_o = PyTuple_GET_ITEM(tuple, index); + assert(res_o != NULL); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + res = PyStackRef_FromPyObjectNew(res_o); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = tuple_st; + tuple_st = res; + stack_pointer[-1] = tuple_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_SUBTRACT_FLOAT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBTRACT_FLOAT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); + opcode = BINARY_OP_SUBTRACT_FLOAT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_TOS_FLOAT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + // _GUARD_NOS_FLOAT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBTRACT_FLOAT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyFloat_CheckExact(left_o)); + assert(PyFloat_CheckExact(right_o)); + STAT_INC(BINARY_OP, hit); + double dres = + ((PyFloatObject *)left_o)->ob_fval - + ((PyFloatObject *)right_o)->ob_fval; + res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); + if (PyStackRef_IsNull(res)) { + TRACING_JUMP_TO_LABEL(pop_2_error); + } + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_OP_SUBTRACT_INT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_OP_SUBTRACT_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 6; + INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); + opcode = BINARY_OP_SUBTRACT_INT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + // _GUARD_NOS_INT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!_PyLong_CheckExactAndCompact(left_o)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + } + /* Skip 5 cache entries */ + // _BINARY_OP_SUBTRACT_INT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyLong_CheckExact(left_o)); + assert(PyLong_CheckExact(right_o)); + assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); + STAT_INC(BINARY_OP, hit); + res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); + if (PyStackRef_IsNull(res)) { + UPDATE_MISS_STATS(BINARY_OP); + assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + JUMP_TO_PREDICTED(TRACING_BINARY_OP); + } + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BINARY_SLICE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BINARY_SLICE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BINARY_SLICE); + opcode = BINARY_SLICE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef container; + _PyStackRef start; + _PyStackRef stop; + _PyStackRef res; + // _SPECIALIZE_BINARY_SLICE + { + #if ENABLE_SPECIALIZATION + OPCODE_DEFERRED_INC(BINARY_SLICE); + #endif /* ENABLE_SPECIALIZATION */ + } + // _BINARY_SLICE + { + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), + PyStackRef_AsPyObjectSteal(stop)); + stack_pointer = _PyFrame_GetStackPointer(frame); + PyObject *res_o; + if (slice == NULL) { + res_o = NULL; + } + else { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); + Py_DECREF(slice); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += 2; + } + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(container); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BUILD_INTERPOLATION) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_INTERPOLATION; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_INTERPOLATION); + opcode = BUILD_INTERPOLATION; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + _PyStackRef str; + _PyStackRef *format; + _PyStackRef interpolation; + format = &stack_pointer[-(oparg & 1)]; + str = stack_pointer[-1 - (oparg & 1)]; + value = stack_pointer[-2 - (oparg & 1)]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + PyObject *str_o = PyStackRef_AsPyObjectBorrow(str); + int conversion = oparg >> 2; + PyObject *format_o; + if (oparg & 1) { + format_o = PyStackRef_AsPyObjectBorrow(format[0]); + } + else { + format_o = &_Py_STR(empty); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *interpolation_o = _PyInterpolation_Build(value_o, str_o, conversion, format_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (oparg & 1) { + stack_pointer += -(oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(format[0]); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + stack_pointer += -(oparg & 1); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(str); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (interpolation_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + interpolation = PyStackRef_FromPyObjectSteal(interpolation_o); + stack_pointer[0] = interpolation; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BUILD_LIST) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_LIST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_LIST); + opcode = BUILD_LIST; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef *values; + _PyStackRef list; + values = &stack_pointer[-oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *list_o = _PyList_FromStackRefStealOnSuccess(values, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (list_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + list = PyStackRef_FromPyObjectStealMortal(list_o); + stack_pointer[-oparg] = list; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BUILD_MAP) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_MAP; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_MAP); + opcode = BUILD_MAP; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef *values; + _PyStackRef map; + values = &stack_pointer[-oparg*2]; + STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); + if (CONVERSION_FAILED(values_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg*2; --_i >= 0;) { + tmp = values[_i]; + values[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg*2; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *map_o = _PyDict_FromItems( + values_o, 2, + values_o+1, 2, + oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg*2; --_i >= 0;) { + tmp = values[_i]; + values[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg*2; + assert(WITHIN_STACK_BOUNDS()); + if (map_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + map = PyStackRef_FromPyObjectStealMortal(map_o); + stack_pointer[0] = map; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BUILD_SET) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_SET; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_SET); + opcode = BUILD_SET; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef *values; + _PyStackRef set; + values = &stack_pointer[-oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *set_o = PySet_New(NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (set_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = values[_i]; + values[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + int err = 0; + for (Py_ssize_t i = 0; i < oparg; i++) { + _PyStackRef value = values[i]; + values[i] = PyStackRef_NULL; + if (err == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = _PySet_AddTakeRef((PySetObject *)set_o, PyStackRef_AsPyObjectSteal(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + if (err) { + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(set_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + set = PyStackRef_FromPyObjectStealMortal(set_o); + stack_pointer[-oparg] = set; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BUILD_SLICE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_SLICE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_SLICE); + opcode = BUILD_SLICE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef *args; + _PyStackRef slice; + args = &stack_pointer[-oparg]; + PyObject *start_o = PyStackRef_AsPyObjectBorrow(args[0]); + PyObject *stop_o = PyStackRef_AsPyObjectBorrow(args[1]); + PyObject *step_o = oparg == 3 ? PyStackRef_AsPyObjectBorrow(args[2]) : NULL; + PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + if (slice_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + slice = PyStackRef_FromPyObjectStealMortal(slice_o); + stack_pointer[0] = slice; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BUILD_STRING) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_STRING; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_STRING); + opcode = BUILD_STRING; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef *pieces; + _PyStackRef str; + pieces = &stack_pointer[-oparg]; + STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); + if (CONVERSION_FAILED(pieces_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = pieces[_i]; + pieces[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); + STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = pieces[_i]; + pieces[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + if (str_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + str = PyStackRef_FromPyObjectSteal(str_o); + stack_pointer[0] = str; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BUILD_TEMPLATE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_TEMPLATE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_TEMPLATE); + opcode = BUILD_TEMPLATE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef strings; + _PyStackRef interpolations; + _PyStackRef template; + interpolations = stack_pointer[-1]; + strings = stack_pointer[-2]; + PyObject *strings_o = PyStackRef_AsPyObjectBorrow(strings); + PyObject *interpolations_o = PyStackRef_AsPyObjectBorrow(interpolations); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *template_o = _PyTemplate_Build(strings_o, interpolations_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(interpolations); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(strings); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (template_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + template = PyStackRef_FromPyObjectSteal(template_o); + stack_pointer[0] = template; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(BUILD_TUPLE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = BUILD_TUPLE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(BUILD_TUPLE); + opcode = BUILD_TUPLE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef *values; + _PyStackRef tup; + values = &stack_pointer[-oparg]; + PyObject *tup_o = _PyTuple_FromStackRefStealOnSuccess(values, oparg); + if (tup_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + tup = PyStackRef_FromPyObjectStealMortal(tup_o); + stack_pointer[-oparg] = tup; + stack_pointer += 1 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CACHE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CACHE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CACHE); + opcode = CACHE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + assert(0 && "Executing a cache."); + Py_FatalError("Executing a cache."); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL); + PREDICTED_TRACING_CALL:; + _Py_CODEUNIT* const this_instr = next_instr - 4; + (void)this_instr; + opcode = CALL; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + opcode = CALL; + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + // _SPECIALIZE_CALL + { + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_Call(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(CALL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 2 cache entries */ + // _MAYBE_EXPAND_METHOD + { + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + self_or_null = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(method); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + // _DO_CALL + { + args = &stack_pointer[-oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, total_args, NULL, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (new_frame == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + frame->return_offset = 4u ; + TRACING_DISPATCH_INLINED(new_frame); + } + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); + if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + } + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_ALLOC_AND_ENTER_INIT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_ALLOC_AND_ENTER_INIT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); + opcode = CALL_ALLOC_AND_ENTER_INIT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef init; + _PyStackRef self; + _PyStackRef *args; + _PyStackRef init_frame; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CHECK_AND_ALLOCATE_OBJECT + { + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyStackRef_IsNull(self_or_null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (!PyType_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyTypeObject *tp = (PyTypeObject *)callable_o; + if (FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + assert(tp->tp_new == PyBaseObject_Type.tp_new); + assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); + assert(tp->tp_alloc == PyType_GenericAlloc); + PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; + PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); + PyCodeObject *code = (PyCodeObject *)init_func->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *self_o = PyType_GenericAlloc(tp, 0); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (self_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + self_or_null = PyStackRef_FromPyObjectSteal(self_o); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(init_func); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _CREATE_INIT_FRAME + { + args = &stack_pointer[-oparg]; + self = self_or_null; + init = callable; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( + tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); + assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); + shim->localsplus[0] = PyStackRef_DUP(self); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, init, NULL, args-1, oparg+1, NULL, shim); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FrameClearAndPop(tstate, shim); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; + tstate->py_recursion_remaining--; + init_frame = PyStackRef_Wrap(temp); + } + // _PUSH_FRAME + { + new_frame = init_frame; + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_BOUND_METHOD_EXACT_ARGS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); + opcode = CALL_BOUND_METHOD_EXACT_ARGS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS + { + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _INIT_CALL_BOUND_METHOD_EXACT_ARGS + { + self_or_null = null; + assert(PyStackRef_IsNull(self_or_null)); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + STAT_INC(CALL, hit); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // flush + // _CHECK_FUNCTION_VERSION + { + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CHECK_FUNCTION_EXACT_ARGS + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CHECK_STACK_SPACE + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _INIT_CALL_PY_EXACT_ARGS + { + args = &stack_pointer[-oparg]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_BOUND_METHOD_GENERAL) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_BOUND_METHOD_GENERAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); + opcode = CALL_BOUND_METHOD_GENERAL; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CHECK_METHOD_VERSION + { + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + if (!PyFunction_Check(func)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _EXPAND_METHOD + { + self_or_null = null; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyStackRef_IsNull(self_or_null)); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(callable)); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // flush + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _PY_FRAME_GENERAL + { + args = &stack_pointer[-oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + args, total_args, NULL, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_BUILTIN_CLASS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_BUILTIN_CLASS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_CLASS); + opcode = CALL_BUILTIN_CLASS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_CLASS + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyType_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyTypeObject *tp = (PyTypeObject *)callable_o; + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + if (tp->tp_vectorcall == NULL) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + STAT_INC(CALL, hit); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_BUILTIN_FAST) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_BUILTIN_FAST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST); + opcode = CALL_BUILTIN_FAST; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_FAST + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + if (!PyCFunction_CheckExact(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunctionFast_CAST(cfunc)( + PyCFunction_GET_SELF(callable_o), + args_o, + total_args); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_BUILTIN_FAST_WITH_KEYWORDS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); + opcode = CALL_BUILTIN_FAST_WITH_KEYWORDS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_FAST_WITH_KEYWORDS + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + if (!PyCFunction_CheckExact(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCFunctionFastWithKeywords cfunc = + _PyCFunctionFastWithKeywords_CAST(PyCFunction_GET_FUNCTION(callable_o)); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_BUILTIN_O) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_BUILTIN_O; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BUILTIN_O); + opcode = CALL_BUILTIN_O; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_BUILTIN_O + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + if (total_args != 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (!PyCFunction_CheckExact(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (_Py_ReachedRecursionLimit(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); + _PyStackRef arg = args[0]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg)); + stack_pointer = _PyFrame_GetStackPointer(frame); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_FUNCTION_EX) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_FUNCTION_EX; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_FUNCTION_EX); + opcode = CALL_FUNCTION_EX; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + opcode = CALL_FUNCTION_EX; + _PyStackRef func; + _PyStackRef callargs; + _PyStackRef func_st; + _PyStackRef null; + _PyStackRef callargs_st; + _PyStackRef kwargs_st; + _PyStackRef result; + // _MAKE_CALLARGS_A_TUPLE + { + callargs = stack_pointer[-2]; + func = stack_pointer[-4]; + PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); + if (!PyTuple_CheckExact(callargs_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *tuple_o = PySequence_Tuple(callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (tuple_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + _PyStackRef temp = callargs; + callargs = PyStackRef_FromPyObjectSteal(tuple_o); + stack_pointer[-2] = callargs; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + // _DO_CALL_FUNCTION_EX + { + kwargs_st = stack_pointer[-1]; + callargs_st = callargs; + null = stack_pointer[-3]; + func_st = func; + (void)null; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); + PyObject *result_o; + assert(!_PyErr_Occurred(tstate)); + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + assert(PyTuple_CheckExact(callargs)); + PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? + PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; + stack_pointer[-2] = callargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + result_o = PyObject_Call(func, callargs, kwargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!PyFunction_Check(func) && !PyMethod_Check(func)) { + if (result_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(result_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + } + } + else { + if (Py_TYPE(func) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( + tstate, func_st, locals, + nargs, callargs, kwargs, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (new_frame == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + assert( 1u == 1); + frame->return_offset = 1; + TRACING_DISPATCH_INLINED(new_frame); + } + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + stack_pointer[-2] = callargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + result_o = PyObject_Call(func, callargs, kwargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(kwargs_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callargs_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(func_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (result_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + result = PyStackRef_FromPyObjectSteal(result_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = result; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_INTRINSIC_1) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_INTRINSIC_1; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_INTRINSIC_1); + opcode = CALL_INTRINSIC_1; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + assert(oparg <= MAX_INTRINSIC_1); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_INTRINSIC_2) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_INTRINSIC_2; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CALL_INTRINSIC_2); + opcode = CALL_INTRINSIC_2; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value2_st; + _PyStackRef value1_st; + _PyStackRef res; + value1_st = stack_pointer[-1]; + value2_st = stack_pointer[-2]; + assert(oparg <= MAX_INTRINSIC_2); + PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); + PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); + _PyStackRef tmp = value1_st; + value1_st = PyStackRef_NULL; + stack_pointer[-1] = value1_st; + PyStackRef_CLOSE(tmp); + tmp = value2_st; + value2_st = PyStackRef_NULL; + stack_pointer[-2] = value2_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_ISINSTANCE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_ISINSTANCE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_ISINSTANCE); + opcode = CALL_ISINSTANCE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef null; + _PyStackRef callable; + _PyStackRef instance; + _PyStackRef cls; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_THIRD_NULL + { + null = stack_pointer[-3]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _GUARD_CALLABLE_ISINSTANCE + { + callable = stack_pointer[-4]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyInterpreterState *interp = tstate->interp; + if (callable_o != interp->callable_cache.isinstance) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CALL_ISINSTANCE + { + cls = stack_pointer[-1]; + instance = stack_pointer[-2]; + STAT_INC(CALL, hit); + PyObject *inst_o = PyStackRef_AsPyObjectBorrow(instance); + PyObject *cls_o = PyStackRef_AsPyObjectBorrow(cls); + _PyFrame_SetStackPointer(frame, stack_pointer); + int retval = PyObject_IsInstance(inst_o, cls_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (retval < 0) { + TRACING_JUMP_TO_LABEL(error); + } + (void)null; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(cls); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(instance); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = retval ? PyStackRef_True : PyStackRef_False; + assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_KW) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_KW; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW); + PREDICTED_TRACING_CALL_KW:; + _Py_CODEUNIT* const this_instr = next_instr - 4; + (void)this_instr; + opcode = CALL_KW; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + opcode = CALL_KW; + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef res; + // _SPECIALIZE_CALL_KW + { + self_or_null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(CALL_KW); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 2 cache entries */ + // _MAYBE_EXPAND_METHOD_KW + { + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + self_or_null = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(method); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + // _DO_CALL_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(kwnames); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (new_frame == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); + frame->return_offset = 4u ; + TRACING_DISPATCH_INLINED(new_frame); + } + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = kwnames; + kwnames = PyStackRef_NULL; + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + stack_pointer[-1] = kwnames; + PyStackRef_CLOSE(tmp); + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-2 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-3 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL_KW) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); + if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = kwnames; + kwnames = PyStackRef_NULL; + stack_pointer[-1] = kwnames; + PyStackRef_CLOSE(tmp); + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-2 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-3 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_KW_BOUND_METHOD) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_KW_BOUND_METHOD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); + opcode = CALL_KW_BOUND_METHOD; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef null; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(TRACING_CALL_KW); + } + } + // _CHECK_METHOD_VERSION_KW + { + null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (Py_TYPE(callable_o) != &PyMethod_Type) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(TRACING_CALL_KW); + } + PyObject *func = ((PyMethodObject *)callable_o)->im_func; + if (!PyFunction_Check(func)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(TRACING_CALL_KW); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(TRACING_CALL_KW); + } + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(TRACING_CALL_KW); + } + } + // _EXPAND_METHOD_KW + { + self_or_null = null; + assert(PyStackRef_IsNull(self_or_null)); + _PyStackRef callable_s = callable; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(Py_TYPE(callable_o) == &PyMethod_Type); + self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); + callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); + assert(PyStackRef_FunctionCheck(callable)); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable_s); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // flush + // _PY_FRAME_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(kwnames); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_KW_NON_PY) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_KW_NON_PY; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_NON_PY); + opcode = CALL_KW_NON_PY; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + opcode = CALL_KW_NON_PY; + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CHECK_IS_NOT_PY_CALLABLE_KW + { + callable = stack_pointer[-3 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(TRACING_CALL_KW); + } + if (Py_TYPE(callable_o) == &PyMethod_Type) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(TRACING_CALL_KW); + } + } + // _CALL_KW_NON_PY + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + #if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); + #endif + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = kwnames; + kwnames = PyStackRef_NULL; + stack_pointer[-1] = kwnames; + PyStackRef_CLOSE(tmp); + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-2 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-3 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(kwnames); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_KW_PY) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_KW_PY; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_KW_PY); + opcode = CALL_KW_PY; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(TRACING_CALL_KW); + } + } + // _CHECK_FUNCTION_VERSION_KW + { + callable = stack_pointer[-3 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(TRACING_CALL_KW); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(TRACING_CALL_KW); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(TRACING_CALL_KW); + } + } + // _PY_FRAME_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + self_or_null = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(kwnames); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_LEN) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_LEN; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_LEN); + opcode = CALL_LEN; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef null; + _PyStackRef callable; + _PyStackRef arg; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_NOS_NULL + { + null = stack_pointer[-2]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _GUARD_CALLABLE_LEN + { + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyInterpreterState *interp = tstate->interp; + if (callable_o != interp->callable_cache.len) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CALL_LEN + { + arg = stack_pointer[-1]; + (void)null; + STAT_INC(CALL, hit); + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_ssize_t len_i = PyObject_Length(arg_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (len_i < 0) { + TRACING_JUMP_TO_LABEL(error); + } + PyObject *res_o = PyLong_FromSsize_t(len_i); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_LIST_APPEND) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_LIST_APPEND; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_LIST_APPEND); + opcode = CALL_LIST_APPEND; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef nos; + _PyStackRef self; + _PyStackRef arg; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_CALLABLE_LIST_APPEND + { + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyInterpreterState *interp = tstate->interp; + if (callable_o != interp->callable_cache.list_append) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _GUARD_NOS_NOT_NULL + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (o == NULL) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _GUARD_NOS_LIST + { + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyList_CheckExact(o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CALL_LIST_APPEND + { + arg = stack_pointer[-1]; + self = nos; + assert(oparg == 1); + PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); + if (!PyList_CheckExact(self_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (!LOCK_OBJECT(self_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + STAT_INC(CALL, hit); + int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); + UNLOCK_OBJECT(self_o); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(self); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + #if TIER_ONE + + assert(next_instr->op.code == POP_TOP); + SKIP_OVER(1); + #endif + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_METHOD_DESCRIPTOR_FAST) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_METHOD_DESCRIPTOR_FAST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); + opcode = CALL_METHOD_DESCRIPTOR_FAST; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_FAST + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + if (total_args == 0) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyMethodDef *meth = method->d_method; + if (meth->ml_flags != METH_FASTCALL) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); + assert(self != NULL); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + STAT_INC(CALL, hit); + int nargs = total_args - 1; + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCFunctionFast cfunc = _PyCFunctionFast_CAST(meth->ml_meth); + PyObject *res_o = cfunc(self, (args_o + 1), nargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); + opcode = CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + if (total_args == 0) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyMethodDef *meth = method->d_method; + if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyTypeObject *d_type = method->d_common.d_type; + PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); + assert(self != NULL); + if (!Py_IS_TYPE(self, d_type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + STAT_INC(CALL, hit); + int nargs = total_args - 1; + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCFunctionFastWithKeywords cfunc = + _PyCFunctionFastWithKeywords_CAST(meth->ml_meth); + PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_METHOD_DESCRIPTOR_NOARGS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); + opcode = CALL_METHOD_DESCRIPTOR_NOARGS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_NOARGS + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + assert(oparg == 0 || oparg == 1); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + if (total_args != 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyMethodDef *meth = method->d_method; + _PyStackRef self_stackref = args[0]; + PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); + if (!Py_IS_TYPE(self, method->d_common.d_type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (meth->ml_flags != METH_NOARGS) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (_Py_ReachedRecursionLimit(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(self_stackref); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_METHOD_DESCRIPTOR_O) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_METHOD_DESCRIPTOR_O; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); + opcode = CALL_METHOD_DESCRIPTOR_O; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CALL_METHOD_DESCRIPTOR_O + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; + if (total_args != 2) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyMethodDef *meth = method->d_method; + if (meth->ml_flags != METH_O) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (_Py_ReachedRecursionLimit(tstate)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + _PyStackRef arg_stackref = arguments[1]; + _PyStackRef self_stackref = arguments[0]; + if (!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), + method->d_common.d_type)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, + PyStackRef_AsPyObjectBorrow(self_stackref), + PyStackRef_AsPyObjectBorrow(arg_stackref)); + stack_pointer = _PyFrame_GetStackPointer(frame); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_NON_PY_GENERAL) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_NON_PY_GENERAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_NON_PY_GENERAL); + opcode = CALL_NON_PY_GENERAL; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + opcode = CALL_NON_PY_GENERAL; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CHECK_IS_NOT_PY_CALLABLE + { + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + if (Py_TYPE(callable_o) == &PyMethod_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CALL_NON_PY_GENERAL + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + #if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); + #endif + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_PY_EXACT_ARGS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_PY_EXACT_ARGS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); + opcode = CALL_PY_EXACT_ARGS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CHECK_FUNCTION_VERSION + { + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CHECK_FUNCTION_EXACT_ARGS + { + self_or_null = stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + assert(PyFunction_Check(callable_o)); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CHECK_STACK_SPACE + { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyFunctionObject *func = (PyFunctionObject *)callable_o; + PyCodeObject *code = (PyCodeObject *)func->func_code; + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _INIT_CALL_PY_EXACT_ARGS + { + args = &stack_pointer[-oparg]; + int has_self = !PyStackRef_IsNull(self_or_null); + STAT_INC(CALL, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); + _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; + pushed_frame->localsplus[0] = self_or_null; + for (int i = 0; i < oparg; i++) { + first_non_self_local[i] = args[i]; + } + new_frame = PyStackRef_Wrap(pushed_frame); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_PY_GENERAL) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_PY_GENERAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_PY_GENERAL); + opcode = CALL_PY_GENERAL; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CHECK_FUNCTION_VERSION + { + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = read_u32(&this_instr[2].cache); + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (!PyFunction_Check(callable_o)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + PyFunctionObject *func = (PyFunctionObject *)callable_o; + if (func->func_version != func_version) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _PY_FRAME_GENERAL + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + if (!PyStackRef_IsNull(self_or_null)) { + args--; + total_args++; + } + assert(Py_TYPE(callable_o) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( + tstate, callable, locals, + args, total_args, NULL, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (temp == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + new_frame = PyStackRef_Wrap(temp); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_STR_1) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_STR_1; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_STR_1); + opcode = CALL_STR_1; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef null; + _PyStackRef callable; + _PyStackRef arg; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_NOS_NULL + { + null = stack_pointer[-2]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _GUARD_CALLABLE_STR_1 + { + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (callable_o != (PyObject *)&PyUnicode_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CALL_STR_1 + { + arg = stack_pointer[-1]; + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Str(arg_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + (void)callable; + (void)null; + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_TUPLE_1) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_TUPLE_1; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_TUPLE_1); + opcode = CALL_TUPLE_1; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef null; + _PyStackRef callable; + _PyStackRef arg; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_NOS_NULL + { + null = stack_pointer[-2]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _GUARD_CALLABLE_TUPLE_1 + { + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (callable_o != (PyObject *)&PyTuple_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CALL_TUPLE_1 + { + arg = stack_pointer[-1]; + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); + STAT_INC(CALL, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PySequence_Tuple(arg_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + (void)callable; + (void)null; + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CALL_TYPE_1) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CALL_TYPE_1; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_TYPE_1); + opcode = CALL_TYPE_1; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + _PyStackRef null; + _PyStackRef callable; + _PyStackRef arg; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _GUARD_NOS_NULL + { + null = stack_pointer[-2]; + if (!PyStackRef_IsNull(null)) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _GUARD_CALLABLE_TYPE_1 + { + callable = stack_pointer[-3]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + if (callable_o != (PyObject *)&PyType_Type) { + UPDATE_MISS_STATS(CALL); + assert(_PyOpcode_Deopt[opcode] == (CALL)); + JUMP_TO_PREDICTED(TRACING_CALL); + } + } + // _CALL_TYPE_1 + { + arg = stack_pointer[-1]; + PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); + assert(oparg == 1); + (void)callable; + (void)null; + STAT_INC(CALL, hit); + res = PyStackRef_FromPyObjectNew(Py_TYPE(arg_o)); + stack_pointer[-3] = res; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(CHECK_EG_MATCH) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CHECK_EG_MATCH; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CHECK_EG_MATCH); + opcode = CHECK_EG_MATCH; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef exc_value_st; + _PyStackRef match_type_st; + _PyStackRef rest; + _PyStackRef match; + match_type_st = stack_pointer[-1]; + exc_value_st = stack_pointer[-2]; + PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); + PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = match_type_st; + match_type_st = PyStackRef_NULL; + stack_pointer[-1] = match_type_st; + PyStackRef_CLOSE(tmp); + tmp = exc_value_st; + exc_value_st = PyStackRef_NULL; + stack_pointer[-2] = exc_value_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + PyObject *match_o = NULL; + PyObject *rest_o = NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PyEval_ExceptionGroupMatch(frame, exc_value, match_type, + &match_o, &rest_o); + _PyStackRef tmp = match_type_st; + match_type_st = PyStackRef_NULL; + stack_pointer[-1] = match_type_st; + PyStackRef_CLOSE(tmp); + tmp = exc_value_st; + exc_value_st = PyStackRef_NULL; + stack_pointer[-2] = exc_value_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res < 0) { + TRACING_JUMP_TO_LABEL(error); + } + assert((match_o == NULL) == (rest_o == NULL)); + if (match_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + if (!Py_IsNone(match_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyErr_SetHandledException(match_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + rest = PyStackRef_FromPyObjectSteal(rest_o); + match = PyStackRef_FromPyObjectSteal(match_o); + stack_pointer[0] = rest; + stack_pointer[1] = match; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CHECK_EXC_MATCH) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CHECK_EXC_MATCH; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CHECK_EXC_MATCH); + opcode = CHECK_EXC_MATCH; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyExceptionInstance_Check(left_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyEval_CheckExceptTypeValid(tstate, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = PyErr_GivenExceptionMatches(left_o, right_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(right); + stack_pointer = _PyFrame_GetStackPointer(frame); + b = res ? PyStackRef_True : PyStackRef_False; + stack_pointer[0] = b; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CLEANUP_THROW) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CLEANUP_THROW; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CLEANUP_THROW); + opcode = CLEANUP_THROW; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef sub_iter; + _PyStackRef last_sent_val; + _PyStackRef exc_value_st; + _PyStackRef none; + _PyStackRef value; + exc_value_st = stack_pointer[-1]; + last_sent_val = stack_pointer[-2]; + sub_iter = stack_pointer[-3]; + PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); + #if !_Py_TAIL_CALL_INTERP + assert(throwflag); + #endif + assert(exc_value && PyExceptionInstance_Check(exc_value)); + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = sub_iter; + sub_iter = value; + stack_pointer[-3] = sub_iter; + PyStackRef_CLOSE(tmp); + tmp = exc_value_st; + exc_value_st = PyStackRef_NULL; + stack_pointer[-1] = exc_value_st; + PyStackRef_CLOSE(tmp); + tmp = last_sent_val; + last_sent_val = PyStackRef_NULL; + stack_pointer[-2] = last_sent_val; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + none = PyStackRef_None; + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); + monitor_reraise(tstate, frame, this_instr); + TRACING_JUMP_TO_LABEL(exception_unwind); + } + stack_pointer[0] = none; + stack_pointer[1] = value; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(COMPARE_OP) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = COMPARE_OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP); + PREDICTED_TRACING_COMPARE_OP:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + opcode = COMPARE_OP; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _SPECIALIZE_COMPARE_OP + { + right = stack_pointer[-1]; + left = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_CompareOp(left, right, next_instr, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(COMPARE_OP); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _COMPARE_OP + { + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert((oparg >> 5) <= Py_GE); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + if (oparg & 16) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int res_bool = PyObject_IsTrue(res_o); + Py_DECREF(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_bool < 0) { + TRACING_JUMP_TO_LABEL(error); + } + res = res_bool ? PyStackRef_True : PyStackRef_False; + } + else { + res = PyStackRef_FromPyObjectSteal(res_o); + } + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(COMPARE_OP_FLOAT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = COMPARE_OP_FLOAT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_FLOAT); + opcode = COMPARE_OP_FLOAT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_TOS_FLOAT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyFloat_CheckExact(value_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(TRACING_COMPARE_OP); + } + } + // _GUARD_NOS_FLOAT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!PyFloat_CheckExact(left_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(TRACING_COMPARE_OP); + } + } + /* Skip 1 cache entry */ + // _COMPARE_OP_FLOAT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(COMPARE_OP, hit); + double dleft = PyFloat_AS_DOUBLE(left_o); + double dright = PyFloat_AS_DOUBLE(right_o); + int sign_ish = COMPARISON_BIT(dleft, dright); + PyStackRef_CLOSE_SPECIALIZED(left, _PyFloat_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyFloat_ExactDealloc); + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(COMPARE_OP_INT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = COMPARE_OP_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_INT); + opcode = COMPARE_OP_INT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + _PyStackRef value; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(TRACING_COMPARE_OP); + } + } + // _GUARD_NOS_INT + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + if (!_PyLong_CheckExactAndCompact(left_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(TRACING_COMPARE_OP); + } + } + /* Skip 1 cache entry */ + // _COMPARE_OP_INT + { + right = value; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(_PyLong_IsCompact((PyLongObject *)left_o)); + assert(_PyLong_IsCompact((PyLongObject *)right_o)); + STAT_INC(COMPARE_OP, hit); + assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && + _PyLong_DigitCount((PyLongObject *)right_o) <= 1); + Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o); + Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); + int sign_ish = COMPARISON_BIT(ileft, iright); + PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); + res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(COMPARE_OP_STR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = COMPARE_OP_STR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(COMPARE_OP_STR); + opcode = COMPARE_OP_STR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef left; + _PyStackRef right; + _PyStackRef res; + // _GUARD_TOS_UNICODE + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(TRACING_COMPARE_OP); + } + } + // _GUARD_NOS_UNICODE + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyUnicode_CheckExact(o)) { + UPDATE_MISS_STATS(COMPARE_OP); + assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + JUMP_TO_PREDICTED(TRACING_COMPARE_OP); + } + } + /* Skip 1 cache entry */ + // _COMPARE_OP_STR + { + right = value; + left = nos; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + STAT_INC(COMPARE_OP, hit); + int eq = _PyUnicode_Equal(left_o, right_o); + assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); + PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); + PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); + assert(eq == 0 || eq == 1); + assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); + assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); + res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[-2] = res; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CONTAINS_OP) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CONTAINS_OP; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP); + PREDICTED_TRACING_CONTAINS_OP:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + opcode = CONTAINS_OP; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef right; + _PyStackRef left; + _PyStackRef b; + // _SPECIALIZE_CONTAINS_OP + { + right = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_ContainsOp(right, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(CONTAINS_OP); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _CONTAINS_OP + { + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = PySequence_Contains(right_o, left_o); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res < 0) { + TRACING_JUMP_TO_LABEL(error); + } + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[0] = b; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CONTAINS_OP_DICT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CONTAINS_OP_DICT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP_DICT); + opcode = CONTAINS_OP_DICT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + // _GUARD_TOS_DICT + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyDict_CheckExact(o)) { + UPDATE_MISS_STATS(CONTAINS_OP); + assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); + JUMP_TO_PREDICTED(TRACING_CONTAINS_OP); + } + } + /* Skip 1 cache entry */ + // _CONTAINS_OP_DICT + { + right = tos; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyDict_CheckExact(right_o)); + STAT_INC(CONTAINS_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = PyDict_Contains(right_o, left_o); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res < 0) { + TRACING_JUMP_TO_LABEL(error); + } + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[0] = b; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CONTAINS_OP_SET) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CONTAINS_OP_SET; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(CONTAINS_OP_SET); + opcode = CONTAINS_OP_SET; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + // _GUARD_TOS_ANY_SET + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyAnySet_CheckExact(o)) { + UPDATE_MISS_STATS(CONTAINS_OP); + assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); + JUMP_TO_PREDICTED(TRACING_CONTAINS_OP); + } + } + /* Skip 1 cache entry */ + // _CONTAINS_OP_SET + { + right = tos; + left = stack_pointer[-2]; + PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); + PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); + assert(PyAnySet_CheckExact(right_o)); + STAT_INC(CONTAINS_OP, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PySet_Contains((PySetObject *)right_o, left_o); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res < 0) { + TRACING_JUMP_TO_LABEL(error); + } + b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[0] = b; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(CONVERT_VALUE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = CONVERT_VALUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(CONVERT_VALUE); + opcode = CONVERT_VALUE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + _PyStackRef result; + value = stack_pointer[-1]; + conversion_func conv_fn; + assert(oparg >= FVC_STR && oparg <= FVC_ASCII); + conv_fn = _PyEval_ConversionFuncs[oparg]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (result_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + result = PyStackRef_FromPyObjectSteal(result_o); + stack_pointer[0] = result; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(COPY) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = COPY; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(COPY); + opcode = COPY; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef bottom; + _PyStackRef top; + bottom = stack_pointer[-1 - (oparg-1)]; + top = PyStackRef_DUP(bottom); + stack_pointer[0] = top; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(COPY_FREE_VARS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = COPY_FREE_VARS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(COPY_FREE_VARS); + opcode = COPY_FREE_VARS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + PyCodeObject *co = _PyFrame_GetCode(frame); + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + PyObject *closure = func->func_closure; + assert(oparg == co->co_nfreevars); + int offset = co->co_nlocalsplus - oparg; + for (int i = 0; i < oparg; ++i) { + PyObject *o = PyTuple_GET_ITEM(closure, i); + frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(DELETE_ATTR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = DELETE_ATTR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_ATTR); + opcode = DELETE_ATTR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef owner; + owner = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(DELETE_DEREF) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = DELETE_DEREF; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_DEREF); + opcode = DELETE_DEREF; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); + if (oldobj == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(oldobj); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(DELETE_FAST) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = DELETE_FAST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_FAST); + opcode = DELETE_FAST; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef v = GETLOCAL(oparg); + if (PyStackRef_IsNull(v)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = PyStackRef_NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(DELETE_GLOBAL) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = DELETE_GLOBAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_GLOBAL); + opcode = DELETE_GLOBAL; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyDict_Pop(GLOBALS(), name, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + TRACING_JUMP_TO_LABEL(error); + } + if (err == 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(DELETE_NAME) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = DELETE_NAME; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_NAME); + opcode = DELETE_NAME; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *ns = LOCALS(); + int err; + if (ns == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_SystemError, + "no locals when deleting %R", name); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_DelItem(ns, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, + name); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(DELETE_SUBSCR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = DELETE_SUBSCR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DELETE_SUBSCR); + opcode = DELETE_SUBSCR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef container; + _PyStackRef sub; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), + PyStackRef_AsPyObjectBorrow(sub)); + _PyStackRef tmp = sub; + sub = PyStackRef_NULL; + stack_pointer[-1] = sub; + PyStackRef_CLOSE(tmp); + tmp = container; + container = PyStackRef_NULL; + stack_pointer[-2] = container; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(DICT_MERGE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = DICT_MERGE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DICT_MERGE); + opcode = DICT_MERGE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef callable; + _PyStackRef dict; + _PyStackRef update; + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + callable = stack_pointer[-5 - (oparg - 1)]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyDict_MergeEx(dict_o, update_o, 2); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatKwargsError(tstate, callable_o, update_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(update); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(update); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(DICT_UPDATE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = DICT_UPDATE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(DICT_UPDATE); + opcode = DICT_UPDATE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef dict; + _PyStackRef update; + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); + PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyDict_Update(dict_o, update_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object is not a mapping", + Py_TYPE(update_o)->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(update); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(update); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(END_ASYNC_FOR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = END_ASYNC_FOR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(END_ASYNC_FOR); + opcode = END_ASYNC_FOR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef awaitable_st; + _PyStackRef exc_st; + exc_st = stack_pointer[-1]; + awaitable_st = stack_pointer[-2]; + JUMPBY(0); + (void)oparg; + PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); + assert(exc && PyExceptionInstance_Check(exc)); + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = exc_st; + exc_st = PyStackRef_NULL; + stack_pointer[-1] = exc_st; + PyStackRef_CLOSE(tmp); + tmp = awaitable_st; + awaitable_st = PyStackRef_NULL; + stack_pointer[-2] = awaitable_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + } + else { + Py_INCREF(exc); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetRaisedException(tstate, exc); + monitor_reraise(tstate, frame, this_instr); + TRACING_JUMP_TO_LABEL(exception_unwind); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(END_FOR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = END_FOR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(END_FOR); + opcode = END_FOR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + value = stack_pointer[-1]; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(END_SEND) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = END_SEND; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(END_SEND); + opcode = END_SEND; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef receiver; + _PyStackRef value; + _PyStackRef val; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + val = value; + stack_pointer[-2] = val; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(receiver); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(ENTER_EXECUTOR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = ENTER_EXECUTOR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(ENTER_EXECUTOR); + opcode = ENTER_EXECUTOR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + opcode = ENTER_EXECUTOR; + #ifdef _Py_TIER2 + PyCodeObject *code = _PyFrame_GetCode(frame); + _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; + assert(executor->vm_data.index == INSTR_OFFSET() - 1); + assert(executor->vm_data.code == code); + assert(executor->vm_data.valid); + assert(tstate->current_executor == NULL); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + opcode = executor->vm_data.opcode; + oparg = (oparg & ~255) | executor->vm_data.oparg; + next_instr = this_instr; + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); + } + DISPATCH_GOTO(); + } + assert(executor != tstate->interp->cold_executor); + tstate->jit_exit = NULL; + #if TRACING_JIT + RECORD_TRACE_NO_DISPATCH(); + #endif + TIER1_TO_TIER2(executor); + #else + Py_FatalError("ENTER_EXECUTOR is not supported in this build"); + #endif /* _Py_TIER2 */ + TRACING_DISPATCH(); + } + + TRACING_TARGET(EXIT_INIT_CHECK) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = EXIT_INIT_CHECK; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(EXIT_INIT_CHECK); + opcode = EXIT_INIT_CHECK; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef should_be_none; + should_be_none = stack_pointer[-1]; + if (!PyStackRef_IsNone(should_be_none)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyErr_Format(PyExc_TypeError, + "__init__() should return None, not '%.200s'", + Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(EXTENDED_ARG) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = EXTENDED_ARG; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(EXTENDED_ARG); + opcode = EXTENDED_ARG; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + opcode = EXTENDED_ARG; + assert(oparg); + opcode = next_instr->op.code; + oparg = oparg << 8 | next_instr->op.arg; + PRE_DISPATCH_GOTO(); + DISPATCH_GOTO(); + } + + TRACING_TARGET(FORMAT_SIMPLE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = FORMAT_SIMPLE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(FORMAT_SIMPLE); + opcode = FORMAT_SIMPLE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Format(value_o, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + else { + res = value; + stack_pointer += -1; + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(FORMAT_WITH_SPEC) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = FORMAT_WITH_SPEC; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(FORMAT_WITH_SPEC); + opcode = FORMAT_WITH_SPEC; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + _PyStackRef fmt_spec; + _PyStackRef res; + fmt_spec = stack_pointer[-1]; + value = stack_pointer[-2]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); + _PyStackRef tmp = fmt_spec; + fmt_spec = PyStackRef_NULL; + stack_pointer[-1] = fmt_spec; + PyStackRef_CLOSE(tmp); + tmp = value; + value = PyStackRef_NULL; + stack_pointer[-2] = value; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(FOR_ITER) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = FOR_ITER; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER); + PREDICTED_TRACING_FOR_ITER:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + opcode = FOR_ITER; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef iter; + _PyStackRef null_or_index; + _PyStackRef next; + // _SPECIALIZE_FOR_ITER + { + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_ForIter(iter, null_or_index, next_instr, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(FOR_ITER); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _FOR_ITER + { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!PyStackRef_IsValid(item)) { + if (PyStackRef_IsError(item)) { + TRACING_JUMP_TO_LABEL(error); + } + JUMPBY(oparg + 1); + RECORD_JUMP_TAKEN(); + stack_pointer[-1] = null_or_index; + TRACING_DISPATCH(); + } + next = item; + } + stack_pointer[-1] = null_or_index; + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(FOR_ITER_GEN) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = FOR_ITER_GEN; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_GEN); + opcode = FOR_ITER_GEN; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef gen_frame; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(TRACING_FOR_ITER); + } + } + // _FOR_ITER_GEN_FRAME + { + iter = stack_pointer[-2]; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(gen) != &PyGen_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(TRACING_FOR_ITER); + } + #ifdef Py_GIL_DISABLED + if (!_PyObject_IsUniquelyReferenced((PyObject *)gen)) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(TRACING_FOR_ITER); + } + #endif + if (gen->gi_frame_state >= FRAME_EXECUTING) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(TRACING_FOR_ITER); + } + STAT_INC(FOR_ITER, hit); + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + _PyFrame_StackPush(pushed_frame, PyStackRef_None); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + pushed_frame->previous = frame; + frame->return_offset = (uint16_t)( 2u + oparg); + gen_frame = PyStackRef_Wrap(pushed_frame); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(FOR_ITER_LIST) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = FOR_ITER_LIST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_LIST); + opcode = FOR_ITER_LIST; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef null_or_index; + _PyStackRef next; + /* Skip 1 cache entry */ + // _ITER_CHECK_LIST + { + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o) != &PyList_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(TRACING_FOR_ITER); + } + assert(PyStackRef_IsTaggedInt(null_or_index)); + #ifdef Py_GIL_DISABLED + if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(TRACING_FOR_ITER); + } + #endif + } + // _ITER_JUMP_LIST + { + #ifdef Py_GIL_DISABLED + + #else + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(list_o) == &PyList_Type); + STAT_INC(FOR_ITER, hit); + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { + null_or_index = PyStackRef_TagInt(-1); + JUMPBY(oparg + 1); + RECORD_JUMP_TAKEN(); + stack_pointer[-1] = null_or_index; + TRACING_DISPATCH(); + } + #endif + } + // _ITER_NEXT_LIST + { + PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); + assert(PyList_CheckExact(list_o)); + #ifdef Py_GIL_DISABLED + assert(_Py_IsOwnedByCurrentThread(list_o) || + _PyObject_GC_IS_SHARED(list_o)); + STAT_INC(FOR_ITER, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (result < 0) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(TRACING_FOR_ITER); + } + if (result == 0) { + null_or_index = PyStackRef_TagInt(-1); + JUMPBY(oparg + 1); + stack_pointer[-1] = null_or_index; + TRACING_DISPATCH(); + } + #else + next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index))); + #endif + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); + } + stack_pointer[-1] = null_or_index; + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(FOR_ITER_RANGE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = FOR_ITER_RANGE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_RANGE); + opcode = FOR_ITER_RANGE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef next; + /* Skip 1 cache entry */ + // _ITER_CHECK_RANGE + { + iter = stack_pointer[-2]; + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(r) != &PyRangeIter_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(TRACING_FOR_ITER); + } + #ifdef Py_GIL_DISABLED + if (!_PyObject_IsUniquelyReferenced((PyObject *)r)) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(TRACING_FOR_ITER); + } + #endif + } + // _ITER_JUMP_RANGE + { + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(r) == &PyRangeIter_Type); + #ifdef Py_GIL_DISABLED + assert(_PyObject_IsUniquelyReferenced((PyObject *)r)); + #endif + STAT_INC(FOR_ITER, hit); + if (r->len <= 0) { + JUMPBY(oparg + 1); + RECORD_JUMP_TAKEN(); + TRACING_DISPATCH(); + } + } + // _ITER_NEXT_RANGE + { + _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(r) == &PyRangeIter_Type); + #ifdef Py_GIL_DISABLED + assert(_PyObject_IsUniquelyReferenced((PyObject *)r)); + #endif + assert(r->len > 0); + long value = r->start; + r->start = value + r->step; + r->len--; + PyObject *res = PyLong_FromLong(value); + if (res == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + next = PyStackRef_FromPyObjectSteal(res); + } + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(FOR_ITER_TUPLE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = FOR_ITER_TUPLE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(FOR_ITER_TUPLE); + opcode = FOR_ITER_TUPLE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); + _PyStackRef iter; + _PyStackRef null_or_index; + _PyStackRef next; + /* Skip 1 cache entry */ + // _ITER_CHECK_TUPLE + { + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; + PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + if (Py_TYPE(iter_o) != &PyTuple_Type) { + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + JUMP_TO_PREDICTED(TRACING_FOR_ITER); + } + assert(PyStackRef_IsTaggedInt(null_or_index)); + } + // _ITER_JUMP_TUPLE + { + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + (void)tuple_o; + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + STAT_INC(FOR_ITER, hit); + if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { + null_or_index = PyStackRef_TagInt(-1); + JUMPBY(oparg + 1); + RECORD_JUMP_TAKEN(); + stack_pointer[-1] = null_or_index; + TRACING_DISPATCH(); + } + } + // _ITER_NEXT_TUPLE + { + PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); + assert(Py_TYPE(tuple_o) == &PyTuple_Type); + uintptr_t i = PyStackRef_UntagInt(null_or_index); + assert((size_t)i < (size_t)PyTuple_GET_SIZE(tuple_o)); + next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(tuple_o, i)); + null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); + } + stack_pointer[-1] = null_or_index; + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(GET_AITER) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = GET_AITER; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_AITER); + opcode = GET_AITER; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef obj; + _PyStackRef iter; + obj = stack_pointer[-1]; + unaryfunc getter = NULL; + PyObject *obj_o = PyStackRef_AsPyObjectBorrow(obj); + PyObject *iter_o; + PyTypeObject *type = Py_TYPE(obj_o); + if (type->tp_as_async != NULL) { + getter = type->tp_as_async->am_aiter; + } + if (getter == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' requires an object with " + "__aiter__ method, got %.100s", + type->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(obj); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + iter_o = (*getter)(obj_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(obj); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (iter_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + if (Py_TYPE(iter_o)->tp_as_async == NULL || + Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' received an object from __aiter__ " + "that does not implement __anext__: %.100s", + Py_TYPE(iter_o)->tp_name); + Py_DECREF(iter_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + stack_pointer[0] = iter; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(GET_ANEXT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = GET_ANEXT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_ANEXT); + opcode = GET_ANEXT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef aiter; + _PyStackRef awaitable; + aiter = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (awaitable_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); + stack_pointer[0] = awaitable; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(GET_AWAITABLE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = GET_AWAITABLE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_AWAITABLE); + opcode = GET_AWAITABLE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef iterable; + _PyStackRef iter; + iterable = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (iter_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + stack_pointer[0] = iter; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(GET_ITER) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = GET_ITER; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_ITER); + opcode = GET_ITER; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef iterable; + _PyStackRef iter; + _PyStackRef index_or_null; + iterable = stack_pointer[-1]; + #ifdef Py_STATS + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_GatherStats_GetIter(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + #endif + + PyTypeObject *tp = PyStackRef_TYPE(iterable); + if (tp == &PyTuple_Type || tp == &PyList_Type) { + iter = iterable; + index_or_null = PyStackRef_TagInt(0); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (iter_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + index_or_null = PyStackRef_NULL; + stack_pointer += 1; + } + stack_pointer[-1] = iter; + stack_pointer[0] = index_or_null; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(GET_LEN) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = GET_LEN; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_LEN); + opcode = GET_LEN; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef obj; + _PyStackRef len; + obj = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (len_i < 0) { + TRACING_JUMP_TO_LABEL(error); + } + PyObject *len_o = PyLong_FromSsize_t(len_i); + if (len_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + len = PyStackRef_FromPyObjectSteal(len_o); + stack_pointer[0] = len; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(GET_YIELD_FROM_ITER) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = GET_YIELD_FROM_ITER; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(GET_YIELD_FROM_ITER); + opcode = GET_YIELD_FROM_ITER; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef iterable; + _PyStackRef iter; + iterable = stack_pointer[-1]; + PyObject *iterable_o = PyStackRef_AsPyObjectBorrow(iterable); + if (PyCoro_CheckExact(iterable_o)) { + if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetString(tstate, PyExc_TypeError, + "cannot 'yield from' a coroutine object " + "in a non-coroutine generator"); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + iter = iterable; + } + else if (PyGen_CheckExact(iterable_o)) { + iter = iterable; + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *iter_o = PyObject_GetIter(iterable_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (iter_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + iter = PyStackRef_FromPyObjectSteal(iter_o); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = iterable; + iterable = iter; + stack_pointer[-1] = iterable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + stack_pointer[-1] = iter; + TRACING_DISPATCH(); + } + + TRACING_TARGET(IMPORT_FROM) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = IMPORT_FROM; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IMPORT_FROM); + opcode = IMPORT_FROM; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef from; + _PyStackRef res; + from = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(IMPORT_NAME) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = IMPORT_NAME; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IMPORT_NAME); + opcode = IMPORT_NAME; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef level; + _PyStackRef fromlist; + _PyStackRef res; + fromlist = stack_pointer[-1]; + level = stack_pointer[-2]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyEval_ImportName(tstate, frame, name, + PyStackRef_AsPyObjectBorrow(fromlist), + PyStackRef_AsPyObjectBorrow(level)); + _PyStackRef tmp = fromlist; + fromlist = PyStackRef_NULL; + stack_pointer[-1] = fromlist; + PyStackRef_CLOSE(tmp); + tmp = level; + level = PyStackRef_NULL; + stack_pointer[-2] = level; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_CALL) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_CALL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(INSTRUMENTED_CALL); + opcode = INSTRUMENTED_CALL; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + opcode = INSTRUMENTED_CALL; + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef func; + _PyStackRef maybe_self; + _PyStackRef *args; + _PyStackRef res; + /* Skip 3 cache entries */ + // _MAYBE_EXPAND_METHOD + { + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + self_or_null = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(method); + stack_pointer[-2 - oparg] = callable; + stack_pointer[-1 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + // _MONITOR_CALL + { + args = &stack_pointer[-oparg]; + maybe_self = self_or_null; + func = callable; + int is_meth = !PyStackRef_IsNull(maybe_self); + PyObject *function = PyStackRef_AsPyObjectBorrow(func); + PyObject *arg0; + if (is_meth) { + arg0 = PyStackRef_AsPyObjectBorrow(maybe_self); + } + else if (oparg) { + arg0 = PyStackRef_AsPyObjectBorrow(args[0]); + } + else { + arg0 = &_PyInstrumentation_MISSING; + } + stack_pointer[-2 - oparg] = func; + stack_pointer[-1 - oparg] = maybe_self; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg0 + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + } + // _DO_CALL + { + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, total_args, NULL, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (new_frame == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + frame->return_offset = 4u ; + TRACING_DISPATCH_INLINED(new_frame); + } + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); + if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + } + assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp; + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-1 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-2 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_CALL_FUNCTION_EX; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); + opcode = INSTRUMENTED_CALL_FUNCTION_EX; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + opcode = INSTRUMENTED_CALL_FUNCTION_EX; + _PyStackRef func; + _PyStackRef callargs; + _PyStackRef func_st; + _PyStackRef null; + _PyStackRef callargs_st; + _PyStackRef kwargs_st; + _PyStackRef result; + // _MAKE_CALLARGS_A_TUPLE + { + callargs = stack_pointer[-2]; + func = stack_pointer[-4]; + PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); + if (!PyTuple_CheckExact(callargs_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *tuple_o = PySequence_Tuple(callargs_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (tuple_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + _PyStackRef temp = callargs; + callargs = PyStackRef_FromPyObjectSteal(tuple_o); + stack_pointer[-2] = callargs; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + // _DO_CALL_FUNCTION_EX + { + kwargs_st = stack_pointer[-1]; + callargs_st = callargs; + null = stack_pointer[-3]; + func_st = func; + (void)null; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); + PyObject *result_o; + assert(!_PyErr_Occurred(tstate)); + if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + assert(PyTuple_CheckExact(callargs)); + PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? + PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; + stack_pointer[-2] = callargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + result_o = PyObject_Call(func, callargs, kwargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!PyFunction_Check(func) && !PyMethod_Check(func)) { + if (result_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, func, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(result_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + } + } + else { + if (Py_TYPE(func) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { + PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); + int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( + tstate, func_st, locals, + nargs, callargs, kwargs, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (new_frame == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + assert( 1u == 1); + frame->return_offset = 1; + TRACING_DISPATCH_INLINED(new_frame); + } + PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); + assert(PyTuple_CheckExact(callargs)); + PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + stack_pointer[-2] = callargs_st; + _PyFrame_SetStackPointer(frame, stack_pointer); + result_o = PyObject_Call(func, callargs, kwargs); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(kwargs_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(callargs_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(func_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (result_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + result = PyStackRef_FromPyObjectSteal(result_o); + } + // _CHECK_PERIODIC_AT_END + { + stack_pointer[0] = result; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_CALL_KW) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_CALL_KW; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); + opcode = INSTRUMENTED_CALL_KW; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + opcode = INSTRUMENTED_CALL_KW; + _PyStackRef callable; + _PyStackRef self_or_null; + _PyStackRef *args; + _PyStackRef kwnames; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _MAYBE_EXPAND_METHOD_KW + { + self_or_null = stack_pointer[-2 - oparg]; + callable = stack_pointer[-3 - oparg]; + if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *self = ((PyMethodObject *)callable_o)->im_self; + self_or_null = PyStackRef_FromPyObjectNew(self); + PyObject *method = ((PyMethodObject *)callable_o)->im_func; + _PyStackRef temp = callable; + callable = PyStackRef_FromPyObjectNew(method); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(temp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + // _MONITOR_CALL_KW + { + args = &stack_pointer[-1 - oparg]; + int is_meth = !PyStackRef_IsNull(self_or_null); + PyObject *arg; + if (is_meth) { + arg = PyStackRef_AsPyObjectBorrow(self_or_null); + } + else if (args) { + arg = PyStackRef_AsPyObjectBorrow(args[0]); + } + else { + arg = &_PyInstrumentation_MISSING; + } + PyObject *function = PyStackRef_AsPyObjectBorrow(callable); + stack_pointer[-3 - oparg] = callable; + stack_pointer[-2 - oparg] = self_or_null; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, function, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + } + // _DO_CALL_KW + { + kwnames = stack_pointer[-1]; + args = &stack_pointer[-1 - oparg]; + PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); + PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); + int total_args = oparg; + _PyStackRef *arguments = args; + if (!PyStackRef_IsNull(self_or_null)) { + arguments--; + total_args++; + } + int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); + if (Py_TYPE(callable_o) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, callable, locals, + arguments, positional_args, kwnames_o, frame + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(kwnames); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (new_frame == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); + frame->return_offset = 4u ; + TRACING_DISPATCH_INLINED(new_frame); + } + STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); + if (CONVERSION_FAILED(args_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = kwnames; + kwnames = PyStackRef_NULL; + stack_pointer[-1] = kwnames; + PyStackRef_CLOSE(tmp); + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-2 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-3 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall( + callable_o, args_o, + positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); + if (opcode == INSTRUMENTED_CALL_KW) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); + if (res_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable_o, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(res_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = kwnames; + kwnames = PyStackRef_NULL; + stack_pointer[-1] = kwnames; + PyStackRef_CLOSE(tmp); + for (int _i = oparg; --_i >= 0;) { + tmp = args[_i]; + args[_i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } + tmp = self_or_null; + self_or_null = PyStackRef_NULL; + stack_pointer[-2 - oparg] = self_or_null; + PyStackRef_XCLOSE(tmp); + tmp = callable; + callable = PyStackRef_NULL; + stack_pointer[-3 - oparg] = callable; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3 - oparg; + assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_END_ASYNC_FOR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_END_ASYNC_FOR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_ASYNC_FOR); + opcode = INSTRUMENTED_END_ASYNC_FOR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef awaitable_st; + _PyStackRef exc_st; + // _MONITOR_END_ASYNC_FOR + { + assert((next_instr-oparg)->op.code == END_SEND || (next_instr-oparg)->op.code >= MIN_INSTRUMENTED_OPCODE); + INSTRUMENTED_JUMP(next_instr-oparg, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT); + } + // _END_ASYNC_FOR + { + exc_st = stack_pointer[-1]; + awaitable_st = stack_pointer[-2]; + JUMPBY(0); + (void)oparg; + PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); + assert(exc && PyExceptionInstance_Check(exc)); + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = exc_st; + exc_st = PyStackRef_NULL; + stack_pointer[-1] = exc_st; + PyStackRef_CLOSE(tmp); + tmp = awaitable_st; + awaitable_st = PyStackRef_NULL; + stack_pointer[-2] = awaitable_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + } + else { + Py_INCREF(exc); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetRaisedException(tstate, exc); + monitor_reraise(tstate, frame, this_instr); + TRACING_JUMP_TO_LABEL(exception_unwind); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_END_FOR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_END_FOR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_FOR); + opcode = INSTRUMENTED_END_FOR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef receiver; + _PyStackRef value; + value = stack_pointer[-1]; + receiver = stack_pointer[-3]; + if (PyStackRef_GenCheck(receiver)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_END_SEND) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_END_SEND; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_END_SEND); + opcode = INSTRUMENTED_END_SEND; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef receiver; + _PyStackRef value; + _PyStackRef val; + value = stack_pointer[-1]; + receiver = stack_pointer[-2]; + PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); + if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + } + val = value; + stack_pointer[-2] = val; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(receiver); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_FOR_ITER) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_FOR_ITER; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); + opcode = INSTRUMENTED_FOR_ITER; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef iter; + _PyStackRef null_or_index; + _PyStackRef next; + /* Skip 1 cache entry */ + null_or_index = stack_pointer[-1]; + iter = stack_pointer[-2]; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (!PyStackRef_IsValid(item)) { + if (PyStackRef_IsError(item)) { + TRACING_JUMP_TO_LABEL(error); + } + JUMPBY(oparg + 1); + stack_pointer[-1] = null_or_index; + TRACING_DISPATCH(); + } + next = item; + INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); + stack_pointer[-1] = null_or_index; + stack_pointer[0] = next; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_INSTRUCTION) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_INSTRUCTION; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); + opcode = INSTRUMENTED_INSTRUCTION; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + opcode = INSTRUMENTED_INSTRUCTION; + _PyFrame_SetStackPointer(frame, stack_pointer); + int next_opcode = _Py_call_instrumentation_instruction( + tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (next_opcode < 0) { + TRACING_JUMP_TO_LABEL(error); + } + next_instr = this_instr; + if (_PyOpcode_Caches[next_opcode]) { + PAUSE_ADAPTIVE_COUNTER(next_instr[1].counter); + } + assert(next_opcode > 0 && next_opcode < 256); + opcode = next_opcode; + DISPATCH_GOTO(); + } + + TRACING_TARGET(INSTRUMENTED_JUMP_BACKWARD) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_JUMP_BACKWARD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); + opcode = INSTRUMENTED_JUMP_BACKWARD; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + /* Skip 1 cache entry */ + // _CHECK_PERIODIC + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + // _MONITOR_JUMP_BACKWARD + { + INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_JUMP_FORWARD) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_JUMP_FORWARD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); + opcode = INSTRUMENTED_JUMP_FORWARD; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_LINE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_LINE; + (void)(opcode); + #endif + _Py_CODEUNIT* const prev_instr = frame->instr_ptr; + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_LINE); + opcode = INSTRUMENTED_LINE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + opcode = INSTRUMENTED_LINE; + int original_opcode = 0; + if (tstate->tracing) { + PyCodeObject *code = _PyFrame_GetCode(frame); + int index = (int)(this_instr - _PyFrame_GetBytecode(frame)); + original_opcode = code->_co_monitoring->lines->data[index*code->_co_monitoring->lines->bytes_per_entry]; + next_instr = this_instr; + } else { + _PyFrame_SetStackPointer(frame, stack_pointer); + original_opcode = _Py_call_instrumentation_line( + tstate, frame, this_instr, prev_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (original_opcode < 0) { + next_instr = this_instr+1; + TRACING_JUMP_TO_LABEL(error); + } + next_instr = frame->instr_ptr; + if (next_instr != this_instr) { + TRACING_DISPATCH(); + } + } + if (_PyOpcode_Caches[original_opcode]) { + _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); + PAUSE_ADAPTIVE_COUNTER(cache->counter); + } + opcode = original_opcode; + DISPATCH_GOTO(); + } + + TRACING_TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_LOAD_SUPER_ATTR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); + opcode = INSTRUMENTED_LOAD_SUPER_ATTR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + opcode = INSTRUMENTED_LOAD_SUPER_ATTR; + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _LOAD_SUPER_ATTR + { + self_st = stack_pointer[-1]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + } + PyObject *stack[] = {class, self}; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + if (super == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(super); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + if (super == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = PyObject_GetAttr(super, name); + Py_DECREF(super); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + attr = PyStackRef_FromPyObjectSteal(attr_o); + } + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[1]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer[0] = attr; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_NOT_TAKEN) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_NOT_TAKEN; + (void)(opcode); + #endif + _Py_CODEUNIT* const prev_instr = frame->instr_ptr; + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_NOT_TAKEN); + opcode = INSTRUMENTED_NOT_TAKEN; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + (void)this_instr; + INSTRUMENTED_JUMP(prev_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_POP_ITER) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_POP_ITER; + (void)(opcode); + #endif + _Py_CODEUNIT* const prev_instr = frame->instr_ptr; + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_POP_ITER); + opcode = INSTRUMENTED_POP_ITER; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef iter; + _PyStackRef index_or_null; + index_or_null = stack_pointer[-1]; + iter = stack_pointer[-2]; + (void)index_or_null; + INSTRUMENTED_JUMP(prev_instr, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iter); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_POP_JUMP_IF_FALSE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); + opcode = INSTRUMENTED_POP_JUMP_IF_FALSE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef cond; + /* Skip 1 cache entry */ + cond = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(cond)); + int jump = PyStackRef_IsFalse(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_POP_JUMP_IF_NONE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); + opcode = INSTRUMENTED_POP_JUMP_IF_NONE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + /* Skip 1 cache entry */ + value = stack_pointer[-1]; + int jump = PyStackRef_IsNone(value); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } + else { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += 1; + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_POP_JUMP_IF_NOT_NONE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); + opcode = INSTRUMENTED_POP_JUMP_IF_NOT_NONE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + /* Skip 1 cache entry */ + value = stack_pointer[-1]; + int jump = !PyStackRef_IsNone(value); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } + else { + stack_pointer += -1; + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_POP_JUMP_IF_TRUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); + opcode = INSTRUMENTED_POP_JUMP_IF_TRUE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef cond; + /* Skip 1 cache entry */ + cond = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(cond)); + int jump = PyStackRef_IsTrue(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); + if (jump) { + INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_RESUME) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_RESUME; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RESUME); + opcode = INSTRUMENTED_RESUME; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + // _LOAD_BYTECODE + { + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_CODEUNIT *bytecode = + _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (bytecode == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); + frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; + frame->instr_ptr = bytecode + off; + next_instr = frame->instr_ptr; + TRACING_DISPATCH(); + } + #endif + } + // _MAYBE_INSTRUMENT + { + #ifdef Py_GIL_DISABLED + + int check_instrumentation = 1; + #else + int check_instrumentation = (tstate->tracing == 0); + #endif + if (check_instrumentation) { + uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + if (code_version != global_version) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + next_instr = this_instr; + TRACING_DISPATCH(); + } + } + } + // _CHECK_PERIODIC_IF_NOT_YIELD_FROM + { + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + } + // _MONITOR_RESUME + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation( + tstate, oparg > 0, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + if (frame->instr_ptr != this_instr) { + next_instr = frame->instr_ptr; + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_RETURN_VALUE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_RETURN_VALUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); + opcode = INSTRUMENTED_RETURN_VALUE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef val; + _PyStackRef retval; + _PyStackRef res; + // _RETURN_VALUE_EVENT + { + val = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_RETURN, + frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + } + // _RETURN_VALUE + { + retval = val; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + _PyStackRef temp = PyStackRef_MakeHeapSafe(retval); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(STACK_LEVEL() == 0); + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_IP(frame->return_offset); + #if TIER_TWO + frame->instr_ptr += frame->return_offset; + #endif + res = temp; + LLTRACE_RESUME_FRAME(); + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INSTRUMENTED_YIELD_VALUE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INSTRUMENTED_YIELD_VALUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); + opcode = INSTRUMENTED_YIELD_VALUE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef val; + _PyStackRef retval; + _PyStackRef value; + // _YIELD_VALUE_EVENT + { + val = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_arg( + tstate, PY_MONITORING_EVENT_PY_YIELD, + frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + if (frame->instr_ptr != this_instr) { + next_instr = frame->instr_ptr; + TRACING_DISPATCH(); + } + } + // _YIELD_VALUE + { + retval = val; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + frame->instr_ptr++; + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + _PyStackRef temp = retval; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + #if TIER_ONE + assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || + frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #endif + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + #if TIER_TWO + frame->instr_ptr += (1 + INLINE_CACHE_ENTRIES_SEND); + #endif + value = PyStackRef_MakeHeapSafe(temp); + LLTRACE_RESUME_FRAME(); + } + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(INTERPRETER_EXIT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = INTERPRETER_EXIT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(INTERPRETER_EXIT); + opcode = INTERPRETER_EXIT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef retval; + retval = stack_pointer[-1]; + assert(frame->owner == FRAME_OWNED_BY_INTERPRETER); + assert(_PyFrame_IsIncomplete(frame)); + tstate->current_frame = frame->previous; + assert(!_PyErr_Occurred(tstate)); + PyObject *result = PyStackRef_AsPyObjectSteal(retval); + #if !_Py_TAIL_CALL_INTERP + assert(frame == &entry.frame); + #endif + #ifdef _Py_TIER2 + _PyStackRef executor = frame->localsplus[0]; + assert(tstate->current_executor == NULL); + if (!PyStackRef_IsNull(executor)) { + tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(executor); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += 1; + } + #endif + LLTRACE_RESUME_FRAME(); + return result; + } + + TRACING_TARGET(IS_OP) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = IS_OP; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(IS_OP); + opcode = IS_OP; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef left; + _PyStackRef right; + _PyStackRef b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = right; + right = PyStackRef_NULL; + stack_pointer[-1] = right; + PyStackRef_CLOSE(tmp); + tmp = left; + left = PyStackRef_NULL; + stack_pointer[-2] = left; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + b = res ? PyStackRef_True : PyStackRef_False; + stack_pointer[0] = b; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(JUMP_BACKWARD) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = JUMP_BACKWARD; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(JUMP_BACKWARD); + PREDICTED_TRACING_JUMP_BACKWARD:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + opcode = JUMP_BACKWARD; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + /* Skip 1 cache entry */ + // _SPECIALIZE_JUMP_BACKWARD + { + #if ENABLE_SPECIALIZATION + if (this_instr->op.code == JUMP_BACKWARD) { + this_instr->op.code = (tstate->interp->jit && !PyStackRef_IsNull(frame->f_funcobj)) ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; + next_instr = this_instr; + DISPATCH_SAME_OPARG(); + } + #endif + } + // _CHECK_PERIODIC + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + // _JUMP_BACKWARD_NO_INTERRUPT + { + #if TIER_ONE + assert(oparg <= INSTR_OFFSET()); + #endif + JUMPBY(-oparg); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(JUMP_BACKWARD_JIT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = JUMP_BACKWARD_JIT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(JUMP_BACKWARD_JIT); + opcode = JUMP_BACKWARD_JIT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(1 == 1, "incorrect cache size"); + /* Skip 1 cache entry */ + // _CHECK_PERIODIC + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + // _JUMP_BACKWARD_NO_INTERRUPT + { + #if TIER_ONE + assert(oparg <= INSTR_OFFSET()); + #endif + JUMPBY(-oparg); + } + // _JIT + { + #ifdef _Py_TIER2 + _Py_BackoffCounter counter = this_instr[1].counter; + if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && + this_instr->op.code == JUMP_BACKWARD_JIT && + next_instr->op.code != ENTER_EXECUTOR) { + if (tstate->interp->jit_tracer_code_buffer == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + tstate->interp->jit_tracer_code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (tstate->interp->jit_tracer_code_buffer == NULL) { + TRACING_DISPATCH(); + } + } + _PyJIT_InitializeTracing(tstate, frame, this_instr, STACK_LEVEL(), 0, NULL); + ENTER_TRACING(); + TRACING_DISPATCH(); + } + else { + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + } + #endif + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(JUMP_BACKWARD_NO_INTERRUPT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = JUMP_BACKWARD_NO_INTERRUPT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); + opcode = JUMP_BACKWARD_NO_INTERRUPT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + #if TIER_ONE + assert(oparg <= INSTR_OFFSET()); + #endif + JUMPBY(-oparg); + TRACING_DISPATCH(); + } + + TRACING_TARGET(JUMP_BACKWARD_NO_JIT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = JUMP_BACKWARD_NO_JIT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(JUMP_BACKWARD_NO_JIT); + opcode = JUMP_BACKWARD_NO_JIT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(1 == 1, "incorrect cache size"); + /* Skip 1 cache entry */ + // _CHECK_PERIODIC + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + // _JUMP_BACKWARD_NO_INTERRUPT + { + #if TIER_ONE + assert(oparg <= INSTR_OFFSET()); + #endif + JUMPBY(-oparg); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(JUMP_FORWARD) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = JUMP_FORWARD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(JUMP_FORWARD); + opcode = JUMP_FORWARD; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + JUMPBY(oparg); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LIST_APPEND) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LIST_APPEND; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LIST_APPEND); + opcode = LIST_APPEND; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef list; + _PyStackRef v; + v = stack_pointer[-1]; + list = stack_pointer[-2 - (oparg-1)]; + int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), + PyStackRef_AsPyObjectSteal(v)); + if (err < 0) { + TRACING_JUMP_TO_LABEL(pop_1_error); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LIST_EXTEND) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LIST_EXTEND; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LIST_EXTEND); + opcode = LIST_EXTEND; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef list_st; + _PyStackRef iterable_st; + iterable_st = stack_pointer[-1]; + list_st = stack_pointer[-2 - (oparg-1)]; + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (none_val == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches && + (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) + { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Clear(tstate); + _PyErr_Format(tstate, PyExc_TypeError, + "Value after * must be an iterable, not %.200s", + Py_TYPE(iterable)->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + assert(Py_IsNone(none_val)); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR); + PREDICTED_TRACING_LOAD_ATTR:; + _Py_CODEUNIT* const this_instr = next_instr - 10; + (void)this_instr; + opcode = LOAD_ATTR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef owner; + _PyStackRef *attr; + _PyStackRef *self_or_null; + // _SPECIALIZE_LOAD_ATTR + { + owner = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_LoadAttr(owner, next_instr, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(LOAD_ATTR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 8 cache entries */ + // _LOAD_ATTR + { + attr = &stack_pointer[-1]; + self_or_null = &stack_pointer[0]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + if (oparg & 1) { + *attr = PyStackRef_NULL; + _PyFrame_SetStackPointer(frame, stack_pointer); + int is_meth = _PyObject_GetMethodStackRef(tstate, PyStackRef_AsPyObjectBorrow(owner), name, attr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (is_meth) { + assert(!PyStackRef_IsNull(*attr)); + self_or_null[0] = owner; + } + else { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsNull(*attr)) { + TRACING_JUMP_TO_LABEL(error); + } + self_or_null[0] = PyStackRef_NULL; + stack_pointer += 1; + } + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + *attr = PyStackRef_FromPyObjectSteal(attr_o); + stack_pointer += 1; + } + } + stack_pointer += (oparg&1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_CLASS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_CLASS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_CLASS); + opcode = LOAD_ATTR_CLASS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _CHECK_ATTR_CLASS + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (!PyType_Check(owner_o)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_CLASS + { + PyObject *descr = read_obj(&this_instr[6].cache); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = PyStackRef_FromPyObjectNew(descr); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = owner; + owner = attr; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[0]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_CLASS_WITH_METACLASS_CHECK; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); + opcode = LOAD_ATTR_CLASS_WITH_METACLASS_CHECK; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _CHECK_ATTR_CLASS + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (!PyType_Check(owner_o)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + // _GUARD_TYPE_VERSION + { + uint32_t type_version = read_u32(&this_instr[4].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + // _LOAD_ATTR_CLASS + { + PyObject *descr = read_obj(&this_instr[6].cache); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + attr = PyStackRef_FromPyObjectNew(descr); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = owner; + owner = attr; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[0]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); + opcode = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + /* Skip 1 cache entry */ + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + uint32_t func_version = read_u32(&this_instr[4].cache); + PyObject *getattribute = read_obj(&this_instr[6].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert((oparg & 1) == 0); + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + PyTypeObject *cls = Py_TYPE(owner_o); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); + PyFunctionObject *f = (PyFunctionObject *)getattribute; + assert(func_version != 0); + if (f->func_version != func_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + PyCodeObject *code = (PyCodeObject *)f->func_code; + assert(code->co_argcount == 2); + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + STAT_INC(LOAD_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( + tstate, PyStackRef_FromPyObjectNew(f), 2, frame); + new_frame->localsplus[0] = owner; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + new_frame->localsplus[1] = PyStackRef_FromPyObjectNew(name); + frame->return_offset = 10u ; + TRACING_DISPATCH_INLINED(new_frame); + } + + TRACING_TARGET(LOAD_ATTR_INSTANCE_VALUE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_INSTANCE_VALUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); + opcode = LOAD_ATTR_INSTANCE_VALUE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + // _CHECK_MANAGED_OBJECT_HAS_VALUES + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + // _LOAD_ATTR_INSTANCE_VALUE + { + uint16_t offset = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); + if (attr_o == NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr); + if (!increfed) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif + STAT_INC(LOAD_ATTR, hit); + stack_pointer[-1] = attr; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + /* Skip 5 cache entries */ + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[0]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_METHOD_LAZY_DICT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); + opcode = LOAD_ATTR_METHOD_LAZY_DICT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + // _CHECK_ATTR_METHOD_LAZY_DICT + { + uint16_t dictoffset = read_u16(&this_instr[4].cache); + char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; + PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); + if (dict != NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + /* Skip 1 cache entry */ + // _LOAD_ATTR_METHOD_LAZY_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_METHOD_NO_DICT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_METHOD_NO_DICT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); + opcode = LOAD_ATTR_METHOD_NO_DICT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_METHOD_NO_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_METHOD_WITH_VALUES; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); + opcode = LOAD_ATTR_METHOD_WITH_VALUES; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef self; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + PyDictValues *ivs = _PyObject_InlineValues(owner_o); + if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + // _GUARD_KEYS_VERSION + { + uint32_t keys_version = read_u32(&this_instr[4].cache); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + // _LOAD_ATTR_METHOD_WITH_VALUES + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert(oparg & 1); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); + attr = PyStackRef_FromPyObjectNew(descr); + self = owner; + } + stack_pointer[-1] = attr; + stack_pointer[0] = self; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_MODULE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_MODULE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_MODULE); + opcode = LOAD_ATTR_MODULE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _LOAD_ATTR_MODULE + { + owner = stack_pointer[-1]; + uint32_t dict_version = read_u32(&this_instr[2].cache); + uint16_t index = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; + assert(dict != NULL); + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + assert(keys->dk_kind == DICT_KEYS_UNICODE); + assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(keys->dk_nentries)); + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(keys) + index; + PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); + if (attr_o == NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); + if (!increfed) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif + STAT_INC(LOAD_ATTR, hit); + stack_pointer[-1] = attr; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + /* Skip 5 cache entries */ + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[0]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_NONDESCRIPTOR_NO_DICT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); + opcode = LOAD_ATTR_NONDESCRIPTOR_NO_DICT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + attr = PyStackRef_FromPyObjectNew(descr); + } + stack_pointer[0] = attr; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); + opcode = LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + PyDictValues *ivs = _PyObject_InlineValues(owner_o); + if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + // _GUARD_KEYS_VERSION + { + uint32_t keys_version = read_u32(&this_instr[4].cache); + PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; + PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES + { + PyObject *descr = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + STAT_INC(LOAD_ATTR, hit); + assert(descr != NULL); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + attr = PyStackRef_FromPyObjectNew(descr); + } + stack_pointer[0] = attr; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_PROPERTY) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_PROPERTY; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); + opcode = LOAD_ATTR_PROPERTY; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + /* Skip 2 cache entries */ + // _LOAD_ATTR_PROPERTY_FRAME + { + PyObject *fget = read_obj(&this_instr[6].cache); + assert((oparg & 1) == 0); + assert(Py_IS_TYPE(fget, &PyFunction_Type)); + PyFunctionObject *f = (PyFunctionObject *)fget; + PyCodeObject *code = (PyCodeObject *)f->func_code; + if ((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + if (code->co_kwonlyargcount) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + if (code->co_argcount != 1) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + STAT_INC(LOAD_ATTR, hit); + _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); + pushed_frame->localsplus[0] = owner; + new_frame = PyStackRef_Wrap(pushed_frame); + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_SLOT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_SLOT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_SLOT); + opcode = LOAD_ATTR_SLOT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + // _LOAD_ATTR_SLOT + { + uint16_t index = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + PyObject **addr = (PyObject **)((char *)owner_o + index); + PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); + if (attr_o == NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); + if (!increfed) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif + STAT_INC(LOAD_ATTR, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = owner; + owner = attr; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + /* Skip 5 cache entries */ + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[0]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_ATTR_WITH_HINT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_ATTR_WITH_HINT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 10; + INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); + opcode = LOAD_ATTR_WITH_HINT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef attr; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + // _LOAD_ATTR_WITH_HINT + { + uint16_t hint = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); + if (dict == NULL) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + PyDictKeysObject *dk = FT_ATOMIC_LOAD_PTR(dict->ma_keys); + assert(PyDict_CheckExact((PyObject *)dict)); + #ifdef Py_GIL_DISABLED + if (!_Py_IsOwnedByCurrentThread((PyObject *)dict) && !_PyObject_GC_IS_SHARED(dict)) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + #endif + PyObject *attr_o; + if (hint >= (size_t)FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_nentries)) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + if (dk->dk_kind != DICT_KEYS_UNICODE) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dk) + hint; + if (FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_key) != name) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + attr_o = FT_ATOMIC_LOAD_PTR(ep->me_value); + if (attr_o == NULL) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + STAT_INC(LOAD_ATTR, hit); + #ifdef Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); + if (!increfed) { + if (true) { + UPDATE_MISS_STATS(LOAD_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); + } + } + #else + attr = PyStackRef_FromPyObjectNew(attr_o); + #endif + stack_pointer[-1] = attr; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + /* Skip 5 cache entries */ + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[0]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer += (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_BUILD_CLASS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_BUILD_CLASS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_BUILD_CLASS); + opcode = LOAD_BUILD_CLASS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef bc; + PyObject *bc_o; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + TRACING_JUMP_TO_LABEL(error); + } + if (bc_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetString(tstate, PyExc_NameError, + "__build_class__ not found"); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + bc = PyStackRef_FromPyObjectSteal(bc_o); + stack_pointer[0] = bc; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_COMMON_CONSTANT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_COMMON_CONSTANT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); + opcode = LOAD_COMMON_CONSTANT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + assert(oparg < NUM_COMMON_CONSTANTS); + value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_CONST) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_CONST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_CONST); + opcode = LOAD_CONST; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); + value = PyStackRef_FromPyObjectBorrow(obj); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_DEREF) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_DEREF; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_DEREF); + opcode = LOAD_DEREF; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + _PyFrame_SetStackPointer(frame, stack_pointer); + value = _PyCell_GetStackRef(cell); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsNull(value)) { + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_FAST) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FAST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST); + opcode = LOAD_FAST; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_DUP(GETLOCAL(oparg)); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_FAST_AND_CLEAR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FAST_AND_CLEAR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); + opcode = LOAD_FAST_AND_CLEAR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + value = GETLOCAL(oparg); + GETLOCAL(oparg) = PyStackRef_NULL; + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_FAST_BORROW) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FAST_BORROW; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_BORROW); + opcode = LOAD_FAST_BORROW; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + assert(!PyStackRef_IsNull(GETLOCAL(oparg))); + value = PyStackRef_Borrow(GETLOCAL(oparg)); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_FAST_BORROW_LOAD_FAST_BORROW) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FAST_BORROW_LOAD_FAST_BORROW; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_BORROW_LOAD_FAST_BORROW); + opcode = LOAD_FAST_BORROW_LOAD_FAST_BORROW; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value1; + _PyStackRef value2; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + value1 = PyStackRef_Borrow(GETLOCAL(oparg1)); + value2 = PyStackRef_Borrow(GETLOCAL(oparg2)); + stack_pointer[0] = value1; + stack_pointer[1] = value2; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_FAST_CHECK) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FAST_CHECK; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_CHECK); + opcode = LOAD_FAST_CHECK; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + _PyStackRef value_s = GETLOCAL(oparg); + if (PyStackRef_IsNull(value_s)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, + UNBOUNDLOCAL_ERROR_MSG, + PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + value = PyStackRef_DUP(value_s); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_FAST_LOAD_FAST) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FAST_LOAD_FAST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); + opcode = LOAD_FAST_LOAD_FAST; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value1; + _PyStackRef value2; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + value1 = PyStackRef_DUP(GETLOCAL(oparg1)); + value2 = PyStackRef_DUP(GETLOCAL(oparg2)); + stack_pointer[0] = value1; + stack_pointer[1] = value2; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_FROM_DICT_OR_DEREF) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FROM_DICT_OR_DEREF; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); + opcode = LOAD_FROM_DICT_OR_DEREF; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef class_dict_st; + _PyStackRef value; + class_dict_st = stack_pointer[-1]; + PyObject *value_o; + PyObject *name; + PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); + assert(class_dict); + assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); + name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(class_dict, name, &value_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + TRACING_JUMP_TO_LABEL(error); + } + if (!value_o) { + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + value_o = PyCell_GetRef(cell); + if (value_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(class_dict_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + value = PyStackRef_FromPyObjectSteal(value_o); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_FROM_DICT_OR_GLOBALS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_FROM_DICT_OR_GLOBALS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); + opcode = LOAD_FROM_DICT_OR_GLOBALS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef mod_or_class_dict; + _PyStackRef v; + mod_or_class_dict = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *v_o; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(mod_or_class_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + TRACING_JUMP_TO_LABEL(error); + } + if (v_o == NULL) { + if (PyDict_CheckExact(GLOBALS()) + && PyDict_CheckExact(BUILTINS())) + { + _PyFrame_SetStackPointer(frame, stack_pointer); + v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + (PyDictObject *)BUILTINS(), + name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (v_o == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_JUMP_TO_LABEL(error); + } + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(GLOBALS(), name, &v_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + TRACING_JUMP_TO_LABEL(error); + } + if (v_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(BUILTINS(), name, &v_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + TRACING_JUMP_TO_LABEL(error); + } + if (v_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + } + } + } + v = PyStackRef_FromPyObjectSteal(v_o); + stack_pointer[0] = v; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_GLOBAL) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_GLOBAL; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL); + PREDICTED_TRACING_LOAD_GLOBAL:; + _Py_CODEUNIT* const this_instr = next_instr - 5; + (void)this_instr; + opcode = LOAD_GLOBAL; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef *res; + _PyStackRef *null; + // _SPECIALIZE_LOAD_GLOBAL + { + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(LOAD_GLOBAL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 1 cache entry */ + /* Skip 1 cache entry */ + /* Skip 1 cache entry */ + // _LOAD_GLOBAL + { + res = &stack_pointer[0]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (PyStackRef_IsNull(*res)) { + TRACING_JUMP_TO_LABEL(error); + } + } + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[1]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_GLOBAL_BUILTIN) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_GLOBAL_BUILTIN; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); + opcode = LOAD_GLOBAL_BUILTIN; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); + _PyStackRef res; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _GUARD_GLOBALS_VERSION + { + uint16_t version = read_u16(&this_instr[2].cache); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + if (!PyDict_CheckExact(dict)) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); + } + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); + } + assert(DK_IS_UNICODE(keys)); + } + // _LOAD_GLOBAL_BUILTINS + { + uint16_t version = read_u16(&this_instr[3].cache); + uint16_t index = read_u16(&this_instr[4].cache); + PyDictObject *dict = (PyDictObject *)BUILTINS(); + if (!PyDict_CheckExact(dict)) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); + } + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); + } + assert(DK_IS_UNICODE(keys)); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); + PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); + if (res_o == NULL) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); + } + #if Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); + if (!increfed) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); + } + #else + res = PyStackRef_FromPyObjectNew(res_o); + #endif + STAT_INC(LOAD_GLOBAL, hit); + } + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[1]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer[0] = res; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_GLOBAL_MODULE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_GLOBAL_MODULE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); + opcode = LOAD_GLOBAL_MODULE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); + _PyStackRef res; + _PyStackRef *null; + /* Skip 1 cache entry */ + // _NOP + { + } + // _LOAD_GLOBAL_MODULE + { + uint16_t version = read_u16(&this_instr[2].cache); + uint16_t index = read_u16(&this_instr[4].cache); + PyDictObject *dict = (PyDictObject *)GLOBALS(); + if (!PyDict_CheckExact(dict)) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); + } + PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); + if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); + } + assert(DK_IS_UNICODE(keys)); + PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); + assert(index < DK_SIZE(keys)); + PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); + if (res_o == NULL) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); + } + #if Py_GIL_DISABLED + int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); + if (!increfed) { + UPDATE_MISS_STATS(LOAD_GLOBAL); + assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); + } + #else + res = PyStackRef_FromPyObjectNew(res_o); + #endif + STAT_INC(LOAD_GLOBAL, hit); + } + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[1]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer[0] = res; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_LOCALS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_LOCALS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_LOCALS); + opcode = LOAD_LOCALS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef locals; + PyObject *l = LOCALS(); + if (l == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetString(tstate, PyExc_SystemError, + "no locals found"); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + locals = PyStackRef_FromPyObjectNew(l); + stack_pointer[0] = locals; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_NAME) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_NAME; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_NAME); + opcode = LOAD_NAME; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef v; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *v_o = _PyEval_LoadName(tstate, frame, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (v_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + v = PyStackRef_FromPyObjectSteal(v_o); + stack_pointer[0] = v; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_SMALL_INT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_SMALL_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_SMALL_INT); + opcode = LOAD_SMALL_INT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + assert(oparg < _PY_NSMALLPOSINTS); + PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; + value = PyStackRef_FromPyObjectBorrow(obj); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_SPECIAL) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_SPECIAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(LOAD_SPECIAL); + opcode = LOAD_SPECIAL; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef self; + _PyStackRef *method_and_self; + // _INSERT_NULL + { + self = stack_pointer[-1]; + method_and_self = &stack_pointer[-1]; + method_and_self[1] = self; + method_and_self[0] = PyStackRef_NULL; + } + // _LOAD_SPECIAL + { + method_and_self = &stack_pointer[-1]; + PyObject *name = _Py_SpecialMethods[oparg].name; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyObject_LookupSpecialMethod(name, method_and_self); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err <= 0) { + if (err == 0) { + PyObject *owner = PyStackRef_AsPyObjectBorrow(method_and_self[1]); + _PyFrame_SetStackPointer(frame, stack_pointer); + const char *errfmt = _PyEval_SpecialMethodCanSuggest(owner, oparg) + ? _Py_SpecialMethods[oparg].error_suggestion + : _Py_SpecialMethods[oparg].error; + stack_pointer = _PyFrame_GetStackPointer(frame); + assert(!_PyErr_Occurred(tstate)); + assert(errfmt != NULL); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, errfmt, owner); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_SUPER_ATTR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_SUPER_ATTR; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR); + PREDICTED_TRACING_LOAD_SUPER_ATTR:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + opcode = LOAD_SUPER_ATTR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + opcode = LOAD_SUPER_ATTR; + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr; + _PyStackRef *null; + // _SPECIALIZE_LOAD_SUPER_ATTR + { + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + int load_method = oparg & 1; + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _LOAD_SUPER_ATTR + { + self_st = stack_pointer[-1]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + TRACING_JUMP_TO_LABEL(error); + } + } + PyObject *stack[] = {class, self}; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + if (super == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, global_super, arg); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_CLEAR(super); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + if (super == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = PyObject_GetAttr(super, name); + Py_DECREF(super); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + attr = PyStackRef_FromPyObjectSteal(attr_o); + } + // _PUSH_NULL_CONDITIONAL + { + null = &stack_pointer[1]; + if (oparg & 1) { + null[0] = PyStackRef_NULL; + } + } + stack_pointer[0] = attr; + stack_pointer += 1 + (oparg & 1); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_SUPER_ATTR_ATTR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_SUPER_ATTR_ATTR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); + opcode = LOAD_SUPER_ATTR_ATTR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr_st; + /* Skip 1 cache entry */ + self_st = stack_pointer[-1]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + assert(!(oparg & 1)); + if (global_super != (PyObject *)&PySuper_Type) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_SUPER_ATTR); + } + if (!PyType_Check(class)) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_SUPER_ATTR); + } + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + _PyStackRef tmp = self_st; + self_st = PyStackRef_NULL; + stack_pointer[-1] = self_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-2] = class_st; + PyStackRef_CLOSE(tmp); + tmp = global_super_st; + global_super_st = PyStackRef_NULL; + stack_pointer[-3] = global_super_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + if (attr == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + attr_st = PyStackRef_FromPyObjectSteal(attr); + stack_pointer[0] = attr_st; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(LOAD_SUPER_ATTR_METHOD) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = LOAD_SUPER_ATTR_METHOD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); + opcode = LOAD_SUPER_ATTR_METHOD; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); + _PyStackRef global_super_st; + _PyStackRef class_st; + _PyStackRef self_st; + _PyStackRef attr; + _PyStackRef self_or_null; + /* Skip 1 cache entry */ + self_st = stack_pointer[-1]; + class_st = stack_pointer[-2]; + global_super_st = stack_pointer[-3]; + PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); + PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); + PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); + assert(oparg & 1); + if (global_super != (PyObject *)&PySuper_Type) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_SUPER_ATTR); + } + if (!PyType_Check(class)) { + UPDATE_MISS_STATS(LOAD_SUPER_ATTR); + assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + JUMP_TO_PREDICTED(TRACING_LOAD_SUPER_ATTR); + } + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + PyTypeObject *cls = (PyTypeObject *)class; + int method_found = 0; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attr_o = _PySuper_Lookup(cls, self, name, + Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (attr_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + if (method_found) { + self_or_null = self_st; + } else { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(self_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + self_or_null = PyStackRef_NULL; + stack_pointer += 1; + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = global_super_st; + global_super_st = self_or_null; + stack_pointer[-2] = global_super_st; + PyStackRef_CLOSE(tmp); + tmp = class_st; + class_st = PyStackRef_NULL; + stack_pointer[-1] = class_st; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + attr = PyStackRef_FromPyObjectSteal(attr_o); + stack_pointer[0] = attr; + stack_pointer[1] = self_or_null; + stack_pointer += 2; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(MAKE_CELL) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = MAKE_CELL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAKE_CELL); + opcode = MAKE_CELL; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + PyObject *cell = PyCell_New(initial); + if (cell == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = PyStackRef_FromPyObjectSteal(cell); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(MAKE_FUNCTION) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = MAKE_FUNCTION; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAKE_FUNCTION); + opcode = MAKE_FUNCTION; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef codeobj_st; + _PyStackRef func; + codeobj_st = stack_pointer[-1]; + PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyFunctionObject *func_obj = (PyFunctionObject *) + PyFunction_New(codeobj, GLOBALS()); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(codeobj_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (func_obj == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + _PyFunction_SetVersion( + func_obj, ((PyCodeObject *)codeobj)->co_version); + func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); + stack_pointer[0] = func; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(MAP_ADD) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = MAP_ADD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MAP_ADD); + opcode = MAP_ADD; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef dict_st; + _PyStackRef key; + _PyStackRef value; + value = stack_pointer[-1]; + key = stack_pointer[-2]; + dict_st = stack_pointer[-3 - (oparg - 1)]; + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + assert(PyDict_CheckExact(dict)); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyDict_SetItem_Take2( + (PyDictObject *)dict, + PyStackRef_AsPyObjectSteal(key), + PyStackRef_AsPyObjectSteal(value) + ); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(pop_2_error); + } + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(MATCH_CLASS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = MATCH_CLASS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_CLASS); + opcode = MATCH_CLASS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef subject; + _PyStackRef type; + _PyStackRef names; + _PyStackRef attrs; + names = stack_pointer[-1]; + type = stack_pointer[-2]; + subject = stack_pointer[-3]; + assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names))); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *attrs_o = _PyEval_MatchClass(tstate, + PyStackRef_AsPyObjectBorrow(subject), + PyStackRef_AsPyObjectBorrow(type), oparg, + PyStackRef_AsPyObjectBorrow(names)); + _PyStackRef tmp = names; + names = PyStackRef_NULL; + stack_pointer[-1] = names; + PyStackRef_CLOSE(tmp); + tmp = type; + type = PyStackRef_NULL; + stack_pointer[-2] = type; + PyStackRef_CLOSE(tmp); + tmp = subject; + subject = PyStackRef_NULL; + stack_pointer[-3] = subject; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + if (attrs_o) { + assert(PyTuple_CheckExact(attrs_o)); + attrs = PyStackRef_FromPyObjectSteal(attrs_o); + } + else { + if (_PyErr_Occurred(tstate)) { + TRACING_JUMP_TO_LABEL(error); + } + attrs = PyStackRef_None; + } + stack_pointer[0] = attrs; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(MATCH_KEYS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = MATCH_KEYS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_KEYS); + opcode = MATCH_KEYS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef subject; + _PyStackRef keys; + _PyStackRef values_or_none; + keys = stack_pointer[-1]; + subject = stack_pointer[-2]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, + PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (values_or_none_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); + stack_pointer[0] = values_or_none; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(MATCH_MAPPING) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = MATCH_MAPPING; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_MAPPING); + opcode = MATCH_MAPPING; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef subject; + _PyStackRef res; + subject = stack_pointer[-1]; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; + res = match ? PyStackRef_True : PyStackRef_False; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(MATCH_SEQUENCE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = MATCH_SEQUENCE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(MATCH_SEQUENCE); + opcode = MATCH_SEQUENCE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef subject; + _PyStackRef res; + subject = stack_pointer[-1]; + int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; + res = match ? PyStackRef_True : PyStackRef_False; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(NOP) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = NOP; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(NOP); + opcode = NOP; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + TRACING_DISPATCH(); + } + + TRACING_TARGET(NOT_TAKEN) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = NOT_TAKEN; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(NOT_TAKEN); + opcode = NOT_TAKEN; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + TRACING_DISPATCH(); + } + + TRACING_TARGET(POP_EXCEPT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = POP_EXCEPT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_EXCEPT); + opcode = POP_EXCEPT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef exc_value; + exc_value = stack_pointer[-1]; + _PyErr_StackItem *exc_info = tstate->exc_info; + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_XSETREF(exc_info->exc_value, + PyStackRef_IsNone(exc_value) + ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(POP_ITER) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = POP_ITER; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_ITER); + opcode = POP_ITER; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef iter; + _PyStackRef index_or_null; + index_or_null = stack_pointer[-1]; + iter = stack_pointer[-2]; + (void)index_or_null; + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iter); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(POP_JUMP_IF_FALSE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = POP_JUMP_IF_FALSE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_FALSE); + opcode = POP_JUMP_IF_FALSE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef cond; + /* Skip 1 cache entry */ + cond = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsFalse(cond); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(POP_JUMP_IF_NONE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = POP_JUMP_IF_NONE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_NONE); + opcode = POP_JUMP_IF_NONE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + _PyStackRef b; + _PyStackRef cond; + /* Skip 1 cache entry */ + // _IS_NONE + { + value = stack_pointer[-1]; + if (PyStackRef_IsNone(value)) { + b = PyStackRef_True; + } + else { + b = PyStackRef_False; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = value; + value = b; + stack_pointer[-1] = value; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + // _POP_JUMP_IF_TRUE + { + cond = b; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsTrue(cond); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(POP_JUMP_IF_NOT_NONE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = POP_JUMP_IF_NOT_NONE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); + opcode = POP_JUMP_IF_NOT_NONE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + _PyStackRef b; + _PyStackRef cond; + /* Skip 1 cache entry */ + // _IS_NONE + { + value = stack_pointer[-1]; + if (PyStackRef_IsNone(value)) { + b = PyStackRef_True; + } + else { + b = PyStackRef_False; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = value; + value = b; + stack_pointer[-1] = value; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + } + // _POP_JUMP_IF_FALSE + { + cond = b; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsFalse(cond); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(POP_JUMP_IF_TRUE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = POP_JUMP_IF_TRUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(POP_JUMP_IF_TRUE); + opcode = POP_JUMP_IF_TRUE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef cond; + /* Skip 1 cache entry */ + cond = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(cond)); + int flag = PyStackRef_IsTrue(cond); + JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(POP_TOP) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = POP_TOP; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(POP_TOP); + opcode = POP_TOP; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + value = stack_pointer[-1]; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(PUSH_EXC_INFO) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = PUSH_EXC_INFO; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(PUSH_EXC_INFO); + opcode = PUSH_EXC_INFO; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef exc; + _PyStackRef prev_exc; + _PyStackRef new_exc; + exc = stack_pointer[-1]; + _PyErr_StackItem *exc_info = tstate->exc_info; + if (exc_info->exc_value != NULL) { + prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value); + } + else { + prev_exc = PyStackRef_None; + } + assert(PyStackRef_ExceptionInstanceCheck(exc)); + exc_info->exc_value = PyStackRef_AsPyObjectNew(exc); + new_exc = exc; + stack_pointer[-1] = prev_exc; + stack_pointer[0] = new_exc; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(PUSH_NULL) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = PUSH_NULL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(PUSH_NULL); + opcode = PUSH_NULL; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef res; + res = PyStackRef_NULL; + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(RAISE_VARARGS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = RAISE_VARARGS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RAISE_VARARGS); + opcode = RAISE_VARARGS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef *args; + args = &stack_pointer[-oparg]; + assert(oparg < 3); + PyObject *cause = oparg == 2 ? PyStackRef_AsPyObjectSteal(args[1]) : NULL; + PyObject *exc = oparg > 0 ? PyStackRef_AsPyObjectSteal(args[0]) : NULL; + stack_pointer += -oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = do_raise(tstate, exc, cause); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + assert(oparg == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + monitor_reraise(tstate, frame, this_instr); + TRACING_JUMP_TO_LABEL(exception_unwind); + } + TRACING_JUMP_TO_LABEL(error); + } + + TRACING_TARGET(RERAISE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = RERAISE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RERAISE); + opcode = RERAISE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef *values; + _PyStackRef exc_st; + exc_st = stack_pointer[-1]; + values = &stack_pointer[-1 - oparg]; + PyObject *exc = PyStackRef_AsPyObjectSteal(exc_st); + assert(oparg >= 0 && oparg <= 2); + if (oparg) { + frame->instr_ptr = _PyFrame_GetBytecode(frame) + PyStackRef_UntagInt(values[0]); + } + assert(exc && PyExceptionInstance_Check(exc)); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_SetRaisedException(tstate, exc); + monitor_reraise(tstate, frame, this_instr); + TRACING_JUMP_TO_LABEL(exception_unwind); + } + + TRACING_TARGET(RESERVED) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = RESERVED; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESERVED); + opcode = RESERVED; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + assert(0 && "Executing RESERVED instruction."); + Py_FatalError("Executing RESERVED instruction."); + TRACING_DISPATCH(); + } + + TRACING_TARGET(RESUME) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = RESUME; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESUME); + PREDICTED_TRACING_RESUME:; + _Py_CODEUNIT* const this_instr = next_instr - 1; + (void)this_instr; + opcode = RESUME; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + // _LOAD_BYTECODE + { + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_CODEUNIT *bytecode = + _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (bytecode == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); + frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; + frame->instr_ptr = bytecode + off; + next_instr = frame->instr_ptr; + TRACING_DISPATCH(); + } + #endif + } + // _MAYBE_INSTRUMENT + { + #ifdef Py_GIL_DISABLED + + int check_instrumentation = 1; + #else + int check_instrumentation = (tstate->tracing == 0); + #endif + if (check_instrumentation) { + uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + if (code_version != global_version) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + next_instr = this_instr; + TRACING_DISPATCH(); + } + } + } + // _QUICKEN_RESUME + { + #if ENABLE_SPECIALIZATION_FT + if (tstate->tracing == 0 && this_instr->op.code == RESUME) { + FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); + } + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _CHECK_PERIODIC_IF_NOT_YIELD_FROM + { + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = check_periodics(tstate); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err != 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(RESUME_CHECK) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = RESUME_CHECK; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RESUME_CHECK); + opcode = RESUME_CHECK; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(0 == 0, "incorrect cache size"); + #if defined(__EMSCRIPTEN__) + if (_Py_emscripten_signal_clock == 0) { + UPDATE_MISS_STATS(RESUME); + assert(_PyOpcode_Deopt[opcode] == (RESUME)); + JUMP_TO_PREDICTED(TRACING_RESUME); + } + _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; + #endif + uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); + uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); + assert((version & _PY_EVAL_EVENTS_MASK) == 0); + if (eval_breaker != version) { + UPDATE_MISS_STATS(RESUME); + assert(_PyOpcode_Deopt[opcode] == (RESUME)); + JUMP_TO_PREDICTED(TRACING_RESUME); + } + #ifdef Py_GIL_DISABLED + if (frame->tlbc_index != + ((_PyThreadStateImpl *)tstate)->tlbc_index) { + UPDATE_MISS_STATS(RESUME); + assert(_PyOpcode_Deopt[opcode] == (RESUME)); + JUMP_TO_PREDICTED(TRACING_RESUME); + } + #endif + TRACING_DISPATCH(); + } + + TRACING_TARGET(RETURN_GENERATOR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = RETURN_GENERATOR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_GENERATOR); + opcode = RETURN_GENERATOR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef res; + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (gen == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + frame->instr_ptr++; + _PyFrame_Copy(frame, gen_frame); + assert(frame->frame_obj == NULL); + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *prev = frame->previous; + _PyThreadState_PopFrame(tstate, frame); + frame = tstate->current_frame = prev; + LOAD_IP(frame->return_offset); + #if TIER_TWO + frame->instr_ptr += (frame->return_offset); + #endif + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(RETURN_VALUE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = RETURN_VALUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(RETURN_VALUE); + opcode = RETURN_VALUE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef retval; + _PyStackRef res; + retval = stack_pointer[-1]; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + _PyStackRef temp = PyStackRef_MakeHeapSafe(retval); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(STACK_LEVEL() == 0); + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *dying = frame; + frame = tstate->current_frame = dying->previous; + _PyEval_FrameClearAndPop(tstate, dying); + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_IP(frame->return_offset); + #if TIER_TWO + frame->instr_ptr += frame->return_offset; + #endif + res = temp; + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(SEND) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = SEND; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND); + PREDICTED_TRACING_SEND:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + opcode = SEND; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef receiver; + _PyStackRef v; + _PyStackRef retval; + // _SPECIALIZE_SEND + { + receiver = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_Send(receiver, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(SEND); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _SEND + { + v = stack_pointer[-1]; + PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); + PyObject *retval_o; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + if ((tstate->interp->eval_frame == NULL) && + (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && + ((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING) + { + PyGenObject *gen = (PyGenObject *)receiver_o; + _PyInterpreterFrame *gen_frame = &gen->gi_iframe; + _PyFrame_StackPush(gen_frame, PyStackRef_MakeHeapSafe(v)); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert( 2u + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)( 2u + oparg); + assert(gen_frame->previous == NULL); + gen_frame->previous = frame; + TRACING_DISPATCH_INLINED(gen_frame); + } + if (PyStackRef_IsNone(v) && PyIter_Check(receiver_o)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + retval_o = PyObject_CallMethodOneArg(receiver_o, + &_Py_ID(send), + PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + if (retval_o == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (matches) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyEval_MonitorRaise(tstate, frame, this_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyGen_FetchStopIterationValue(&retval_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err == 0) { + assert(retval_o != NULL); + JUMPBY(oparg); + } + else { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); + retval = PyStackRef_FromPyObjectSteal(retval_o); + } + stack_pointer[0] = retval; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(SEND_GEN) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = SEND_GEN; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(SEND_GEN); + opcode = SEND_GEN; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); + _PyStackRef receiver; + _PyStackRef v; + _PyStackRef gen_frame; + _PyStackRef new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + if (tstate->interp->eval_frame) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + JUMP_TO_PREDICTED(TRACING_SEND); + } + } + // _SEND_GEN_FRAME + { + v = stack_pointer[-1]; + receiver = stack_pointer[-2]; + PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); + if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + JUMP_TO_PREDICTED(TRACING_SEND); + } + if (gen->gi_frame_state >= FRAME_EXECUTING) { + UPDATE_MISS_STATS(SEND); + assert(_PyOpcode_Deopt[opcode] == (SEND)); + JUMP_TO_PREDICTED(TRACING_SEND); + } + STAT_INC(SEND, hit); + _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; + _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert( 2u + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)( 2u + oparg); + pushed_frame->previous = frame; + gen_frame = PyStackRef_Wrap(pushed_frame); + } + // _PUSH_FRAME + { + new_frame = gen_frame; + assert(tstate->interp->eval_frame == NULL); + _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + assert(temp->previous == frame || temp->previous->previous == frame); + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = temp; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(SETUP_ANNOTATIONS) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = SETUP_ANNOTATIONS; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SETUP_ANNOTATIONS); + opcode = SETUP_ANNOTATIONS; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + PyObject *ann_dict; + if (LOCALS() == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_SystemError, + "no locals found when setting up annotations"); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + TRACING_JUMP_TO_LABEL(error); + } + if (ann_dict == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + ann_dict = PyDict_New(); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (ann_dict == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), + ann_dict); + Py_DECREF(ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(ann_dict); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(SET_ADD) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = SET_ADD; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_ADD); + opcode = SET_ADD; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef set; + _PyStackRef v; + v = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PySet_AddTakeRef((PySetObject *)PyStackRef_AsPyObjectBorrow(set), + PyStackRef_AsPyObjectSteal(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(pop_1_error); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(SET_FUNCTION_ATTRIBUTE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = SET_FUNCTION_ATTRIBUTE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); + opcode = SET_FUNCTION_ATTRIBUTE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef attr_st; + _PyStackRef func_in; + _PyStackRef func_out; + func_in = stack_pointer[-1]; + attr_st = stack_pointer[-2]; + PyObject *func = PyStackRef_AsPyObjectBorrow(func_in); + PyObject *attr = PyStackRef_AsPyObjectSteal(attr_st); + func_out = func_in; + assert(PyFunction_Check(func)); + size_t offset = _Py_FunctionAttributeOffsets[oparg]; + assert(offset != 0); + PyObject **ptr = (PyObject **)(((char *)func) + offset); + assert(*ptr == NULL); + *ptr = attr; + stack_pointer[-2] = func_out; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(SET_UPDATE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = SET_UPDATE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SET_UPDATE); + opcode = SET_UPDATE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef set; + _PyStackRef iterable; + iterable = stack_pointer[-1]; + set = stack_pointer[-2 - (oparg-1)]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), + PyStackRef_AsPyObjectBorrow(iterable)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(iterable); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + TRACING_JUMP_TO_LABEL(error); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_ATTR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_ATTR; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR); + PREDICTED_TRACING_STORE_ATTR:; + _Py_CODEUNIT* const this_instr = next_instr - 5; + (void)this_instr; + opcode = STORE_ATTR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef owner; + _PyStackRef v; + // _SPECIALIZE_STORE_ATTR + { + owner = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_StoreAttr(owner, next_instr, name); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(STORE_ATTR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 3 cache entries */ + // _STORE_ATTR + { + v = stack_pointer[-2]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), + name, PyStackRef_AsPyObjectBorrow(v)); + _PyStackRef tmp = owner; + owner = PyStackRef_NULL; + stack_pointer[-1] = owner; + PyStackRef_CLOSE(tmp); + tmp = v; + v = PyStackRef_NULL; + stack_pointer[-2] = v; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_ATTR_INSTANCE_VALUE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_ATTR_INSTANCE_VALUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); + opcode = STORE_ATTR_INSTANCE_VALUE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION_AND_LOCK + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(type_version != 0); + if (!LOCK_OBJECT(owner_o)) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); + } + PyTypeObject *tp = Py_TYPE(owner_o); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UNLOCK_OBJECT(owner_o); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); + } + } + } + // _GUARD_DORV_NO_DICT + { + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_dictoffset < 0); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); + if (_PyObject_GetManagedDict(owner_o) || + !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { + UNLOCK_OBJECT(owner_o); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); + } + } + } + // _STORE_ATTR_INSTANCE_VALUE + { + value = stack_pointer[-2]; + uint16_t offset = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + STAT_INC(STORE_ATTR, hit); + assert(_PyObject_GetManagedDict(owner_o) == NULL); + PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); + PyObject *old_value = *value_ptr; + FT_ATOMIC_STORE_PTR_RELEASE(*value_ptr, PyStackRef_AsPyObjectSteal(value)); + if (old_value == NULL) { + PyDictValues *values = _PyObject_InlineValues(owner_o); + Py_ssize_t index = value_ptr - values->values; + _PyDictValues_AddToInsertionOrder(values, index); + } + UNLOCK_OBJECT(owner_o); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + Py_XDECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_ATTR_SLOT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_ATTR_SLOT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_SLOT); + opcode = STORE_ATTR_SLOT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); + } + } + // _STORE_ATTR_SLOT + { + value = stack_pointer[-2]; + uint16_t index = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + if (!LOCK_OBJECT(owner_o)) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); + } + char *addr = (char *)owner_o + index; + STAT_INC(STORE_ATTR, hit); + PyObject *old_value = *(PyObject **)addr; + FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(owner_o); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + Py_XDECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_ATTR_WITH_HINT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_ATTR_WITH_HINT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 5; + INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); + opcode = STORE_ATTR_WITH_HINT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); + } + } + // _STORE_ATTR_WITH_HINT + { + value = stack_pointer[-2]; + uint16_t hint = read_u16(&this_instr[4].cache); + PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); + assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); + PyDictObject *dict = _PyObject_GetManagedDict(owner_o); + if (dict == NULL) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); + } + if (!LOCK_OBJECT(dict)) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); + } + assert(PyDict_CheckExact((PyObject *)dict)); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (hint >= (size_t)dict->ma_keys->dk_nentries || + !DK_IS_UNICODE(dict->ma_keys)) { + UNLOCK_OBJECT(dict); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); + } + } + PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; + if (ep->me_key != name) { + UNLOCK_OBJECT(dict); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); + } + } + PyObject *old_value = ep->me_value; + if (old_value == NULL) { + UNLOCK_OBJECT(dict); + if (true) { + UPDATE_MISS_STATS(STORE_ATTR); + assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + JUMP_TO_PREDICTED(TRACING_STORE_ATTR); + } + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value)); + UNLOCK_OBJECT(dict); + STAT_INC(STORE_ATTR, hit); + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(owner); + Py_XDECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_DEREF) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_DEREF; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_DEREF); + opcode = STORE_DEREF; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef v; + v = stack_pointer[-1]; + PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_FAST) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_FAST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST); + opcode = STORE_FAST; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + value = stack_pointer[-1]; + _PyStackRef tmp = GETLOCAL(oparg); + GETLOCAL(oparg) = value; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_FAST_LOAD_FAST) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_FAST_LOAD_FAST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); + opcode = STORE_FAST_LOAD_FAST; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value1; + _PyStackRef value2; + value1 = stack_pointer[-1]; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + _PyStackRef tmp = GETLOCAL(oparg1); + GETLOCAL(oparg1) = value1; + value2 = PyStackRef_DUP(GETLOCAL(oparg2)); + stack_pointer[-1] = value2; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_FAST_STORE_FAST) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_FAST_STORE_FAST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_FAST_STORE_FAST); + opcode = STORE_FAST_STORE_FAST; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value2; + _PyStackRef value1; + value1 = stack_pointer[-1]; + value2 = stack_pointer[-2]; + uint32_t oparg1 = oparg >> 4; + uint32_t oparg2 = oparg & 15; + _PyStackRef tmp = GETLOCAL(oparg1); + GETLOCAL(oparg1) = value1; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + tmp = GETLOCAL(oparg2); + GETLOCAL(oparg2) = value2; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_XCLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_GLOBAL) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_GLOBAL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_GLOBAL); + opcode = STORE_GLOBAL; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef v; + v = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_NAME) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_NAME; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_NAME); + opcode = STORE_NAME; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef v; + v = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *ns = LOCALS(); + int err; + if (ns == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_SystemError, + "no locals found when storing %R", name); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); + TRACING_JUMP_TO_LABEL(error); + } + if (PyDict_CheckExact(ns)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(v); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_SLICE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_SLICE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(STORE_SLICE); + opcode = STORE_SLICE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef v; + _PyStackRef container; + _PyStackRef start; + _PyStackRef stop; + // _SPECIALIZE_STORE_SLICE + { + #if ENABLE_SPECIALIZATION + OPCODE_DEFERRED_INC(STORE_SLICE); + #endif /* ENABLE_SPECIALIZATION */ + } + // _STORE_SLICE + { + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + v = stack_pointer[-4]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), + PyStackRef_AsPyObjectSteal(stop)); + stack_pointer = _PyFrame_GetStackPointer(frame); + int err; + if (slice == NULL) { + err = 1; + } + else { + stack_pointer += -2; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); + Py_DECREF(slice); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += 2; + } + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = container; + container = PyStackRef_NULL; + stack_pointer[-3] = container; + PyStackRef_CLOSE(tmp); + tmp = v; + v = PyStackRef_NULL; + stack_pointer[-4] = v; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -4; + assert(WITHIN_STACK_BOUNDS()); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_SUBSCR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_SUBSCR; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR); + PREDICTED_TRACING_STORE_SUBSCR:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + opcode = STORE_SUBSCR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef container; + _PyStackRef sub; + _PyStackRef v; + // _SPECIALIZE_STORE_SUBSCR + { + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_StoreSubscr(container, sub, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(STORE_SUBSCR); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + // _STORE_SUBSCR + { + v = stack_pointer[-3]; + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); + _PyStackRef tmp = sub; + sub = PyStackRef_NULL; + stack_pointer[-1] = sub; + PyStackRef_CLOSE(tmp); + tmp = container; + container = PyStackRef_NULL; + stack_pointer[-2] = container; + PyStackRef_CLOSE(tmp); + tmp = v; + v = PyStackRef_NULL; + stack_pointer[-3] = v; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_SUBSCR_DICT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_SUBSCR_DICT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_DICT); + opcode = STORE_SUBSCR_DICT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); + _PyStackRef nos; + _PyStackRef value; + _PyStackRef dict_st; + _PyStackRef sub; + // _GUARD_NOS_DICT + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyDict_CheckExact(o)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); + } + } + /* Skip 1 cache entry */ + // _STORE_SUBSCR_DICT + { + sub = stack_pointer[-1]; + dict_st = nos; + value = stack_pointer[-3]; + PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); + assert(PyDict_CheckExact(dict)); + STAT_INC(STORE_SUBSCR, hit); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = _PyDict_SetItem_Take2((PyDictObject *)dict, + PyStackRef_AsPyObjectSteal(sub), + PyStackRef_AsPyObjectSteal(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(dict_st); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err) { + TRACING_JUMP_TO_LABEL(error); + } + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(STORE_SUBSCR_LIST_INT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = STORE_SUBSCR_LIST_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); + opcode = STORE_SUBSCR_LIST_INT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); + _PyStackRef value; + _PyStackRef nos; + _PyStackRef list_st; + _PyStackRef sub_st; + // _GUARD_TOS_INT + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!_PyLong_CheckExactAndCompact(value_o)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); + } + } + // _GUARD_NOS_LIST + { + nos = stack_pointer[-2]; + PyObject *o = PyStackRef_AsPyObjectBorrow(nos); + if (!PyList_CheckExact(o)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); + } + } + /* Skip 1 cache entry */ + // _STORE_SUBSCR_LIST_INT + { + sub_st = value; + list_st = nos; + value = stack_pointer[-3]; + PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); + PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); + assert(PyLong_CheckExact(sub)); + assert(PyList_CheckExact(list)); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); + } + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (!LOCK_OBJECT(list)) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); + } + if (index >= PyList_GET_SIZE(list)) { + UNLOCK_OBJECT(list); + if (true) { + UPDATE_MISS_STATS(STORE_SUBSCR); + assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); + } + } + STAT_INC(STORE_SUBSCR, hit); + PyObject *old_value = PyList_GET_ITEM(list, index); + FT_ATOMIC_STORE_PTR_RELEASE(_PyList_ITEMS(list)[index], + PyStackRef_AsPyObjectSteal(value)); + assert(old_value != NULL); + UNLOCK_OBJECT(list); + PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); + stack_pointer += -3; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(list_st); + Py_DECREF(old_value); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(SWAP) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = SWAP; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(SWAP); + opcode = SWAP; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef bottom; + _PyStackRef top; + top = stack_pointer[-1]; + bottom = stack_pointer[-2 - (oparg-2)]; + _PyStackRef temp = bottom; + bottom = top; + top = temp; + stack_pointer[-2 - (oparg-2)] = bottom; + stack_pointer[-1] = top; + TRACING_DISPATCH(); + } + + TRACING_TARGET(TO_BOOL) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = TO_BOOL; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL); + PREDICTED_TRACING_TO_BOOL:; + _Py_CODEUNIT* const this_instr = next_instr - 4; + (void)this_instr; + opcode = TO_BOOL; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + _PyStackRef res; + // _SPECIALIZE_TO_BOOL + { + value = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_ToBool(value, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(TO_BOOL); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + } + /* Skip 2 cache entries */ + // _TO_BOOL + { + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + TRACING_JUMP_TO_LABEL(error); + } + res = err ? PyStackRef_True : PyStackRef_False; + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(TO_BOOL_ALWAYS_TRUE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = TO_BOOL_ALWAYS_TRUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); + opcode = TO_BOOL_ALWAYS_TRUE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef owner; + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + // _GUARD_TYPE_VERSION + { + owner = stack_pointer[-1]; + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); + assert(type_version != 0); + if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TRACING_TO_BOOL); + } + } + // _REPLACE_WITH_TRUE + { + value = owner; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_True; + } + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(TO_BOOL_BOOL) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = TO_BOOL_BOOL; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_BOOL); + opcode = TO_BOOL_BOOL; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + if (!PyStackRef_BoolCheck(value)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TRACING_TO_BOOL); + } + STAT_INC(TO_BOOL, hit); + TRACING_DISPATCH(); + } + + TRACING_TARGET(TO_BOOL_INT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = TO_BOOL_INT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_INT); + opcode = TO_BOOL_INT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyLong_CheckExact(value_o)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TRACING_TO_BOOL); + } + STAT_INC(TO_BOOL, hit); + if (_PyLong_IsZero((PyLongObject *)value_o)) { + assert(_Py_IsImmortal(value_o)); + res = PyStackRef_False; + } + else { + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_True; + stack_pointer += 1; + } + stack_pointer[-1] = res; + TRACING_DISPATCH(); + } + + TRACING_TARGET(TO_BOOL_LIST) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = TO_BOOL_LIST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_LIST); + opcode = TO_BOOL_LIST; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef value; + _PyStackRef res; + // _GUARD_TOS_LIST + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyList_CheckExact(o)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TRACING_TO_BOOL); + } + } + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _TO_BOOL_LIST + { + value = tos; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + assert(PyList_CheckExact(value_o)); + STAT_INC(TO_BOOL, hit); + res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyStackRef tmp = value; + value = res; + stack_pointer[-1] = value; + PyStackRef_CLOSE(tmp); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(TO_BOOL_NONE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = TO_BOOL_NONE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_NONE); + opcode = TO_BOOL_NONE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + value = stack_pointer[-1]; + if (!PyStackRef_IsNone(value)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TRACING_TO_BOOL); + } + STAT_INC(TO_BOOL, hit); + res = PyStackRef_False; + stack_pointer[-1] = res; + TRACING_DISPATCH(); + } + + TRACING_TARGET(TO_BOOL_STR) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = TO_BOOL_STR; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(TO_BOOL_STR); + opcode = TO_BOOL_STR; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + _PyStackRef value; + _PyStackRef res; + // _GUARD_TOS_UNICODE + { + value = stack_pointer[-1]; + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (!PyUnicode_CheckExact(value_o)) { + UPDATE_MISS_STATS(TO_BOOL); + assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + JUMP_TO_PREDICTED(TRACING_TO_BOOL); + } + } + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _TO_BOOL_STR + { + STAT_INC(TO_BOOL, hit); + PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); + if (value_o == &_Py_STR(empty)) { + assert(_Py_IsImmortal(value_o)); + res = PyStackRef_False; + } + else { + assert(Py_SIZE(value_o)); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + res = PyStackRef_True; + stack_pointer += 1; + } + } + stack_pointer[-1] = res; + TRACING_DISPATCH(); + } + + TRACING_TARGET(UNARY_INVERT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = UNARY_INVERT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_INVERT); + opcode = UNARY_INVERT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(UNARY_NEGATIVE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = UNARY_NEGATIVE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_NEGATIVE); + opcode = UNARY_NEGATIVE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(value); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(UNARY_NOT) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = UNARY_NOT; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNARY_NOT); + opcode = UNARY_NOT; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef value; + _PyStackRef res; + value = stack_pointer[-1]; + assert(PyStackRef_BoolCheck(value)); + res = PyStackRef_IsFalse(value) + ? PyStackRef_True : PyStackRef_False; + stack_pointer[-1] = res; + TRACING_DISPATCH(); + } + + TRACING_TARGET(UNPACK_EX) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = UNPACK_EX; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(UNPACK_EX); + opcode = UNPACK_EX; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef seq; + _PyStackRef *top; + seq = stack_pointer[-1]; + top = &stack_pointer[(oparg & 0xFF) + (oparg >> 8)]; + PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg & 0xFF, oparg >> 8, top); + Py_DECREF(seq_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res == 0) { + TRACING_JUMP_TO_LABEL(error); + } + stack_pointer += 1 + (oparg & 0xFF) + (oparg >> 8); + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(UNPACK_SEQUENCE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = UNPACK_SEQUENCE; + (void)(opcode); + #endif + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE); + PREDICTED_TRACING_UNPACK_SEQUENCE:; + _Py_CODEUNIT* const this_instr = next_instr - 2; + (void)this_instr; + opcode = UNPACK_SEQUENCE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef seq; + _PyStackRef *top; + // _SPECIALIZE_UNPACK_SEQUENCE + { + seq = stack_pointer[-1]; + uint16_t counter = read_u16(&this_instr[1].cache); + (void)counter; + #if ENABLE_SPECIALIZATION_FT + if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { + next_instr = this_instr; + _PyFrame_SetStackPointer(frame, stack_pointer); + _Py_Specialize_UnpackSequence(seq, next_instr, oparg); + stack_pointer = _PyFrame_GetStackPointer(frame); + DISPATCH_SAME_OPARG(); + } + OPCODE_DEFERRED_INC(UNPACK_SEQUENCE); + ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); + #endif /* ENABLE_SPECIALIZATION_FT */ + (void)seq; + (void)counter; + } + // _UNPACK_SEQUENCE + { + top = &stack_pointer[-1 + oparg]; + PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg, -1, top); + Py_DECREF(seq_o); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res == 0) { + TRACING_JUMP_TO_LABEL(error); + } + } + stack_pointer += oparg; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(UNPACK_SEQUENCE_LIST) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = UNPACK_SEQUENCE_LIST; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); + opcode = UNPACK_SEQUENCE_LIST; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef seq; + _PyStackRef *values; + // _GUARD_TOS_LIST + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyList_CheckExact(o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); + } + } + /* Skip 1 cache entry */ + // _UNPACK_SEQUENCE_LIST + { + seq = tos; + values = &stack_pointer[-1]; + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + assert(PyList_CheckExact(seq_o)); + if (!LOCK_OBJECT(seq_o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); + } + if (PyList_GET_SIZE(seq_o) != oparg) { + UNLOCK_OBJECT(seq_o); + if (true) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); + } + } + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyList_ITEMS(seq_o); + for (int i = oparg; --i >= 0; ) { + *values++ = PyStackRef_FromPyObjectNew(items[i]); + } + UNLOCK_OBJECT(seq_o); + stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(seq); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(UNPACK_SEQUENCE_TUPLE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = UNPACK_SEQUENCE_TUPLE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); + opcode = UNPACK_SEQUENCE_TUPLE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef seq; + _PyStackRef *values; + // _GUARD_TOS_TUPLE + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyTuple_CheckExact(o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); + } + } + /* Skip 1 cache entry */ + // _UNPACK_SEQUENCE_TUPLE + { + seq = tos; + values = &stack_pointer[-1]; + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + assert(PyTuple_CheckExact(seq_o)); + if (PyTuple_GET_SIZE(seq_o) != oparg) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); + } + STAT_INC(UNPACK_SEQUENCE, hit); + PyObject **items = _PyTuple_ITEMS(seq_o); + for (int i = oparg; --i >= 0; ) { + *values++ = PyStackRef_FromPyObjectNew(items[i]); + } + stack_pointer += -1 + oparg; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(seq); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = UNPACK_SEQUENCE_TWO_TUPLE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 2; + INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); + opcode = UNPACK_SEQUENCE_TWO_TUPLE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); + _PyStackRef tos; + _PyStackRef seq; + _PyStackRef val1; + _PyStackRef val0; + // _GUARD_TOS_TUPLE + { + tos = stack_pointer[-1]; + PyObject *o = PyStackRef_AsPyObjectBorrow(tos); + if (!PyTuple_CheckExact(o)) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); + } + } + /* Skip 1 cache entry */ + // _UNPACK_SEQUENCE_TWO_TUPLE + { + seq = tos; + assert(oparg == 2); + PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); + assert(PyTuple_CheckExact(seq_o)); + if (PyTuple_GET_SIZE(seq_o) != 2) { + UPDATE_MISS_STATS(UNPACK_SEQUENCE); + assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); + } + STAT_INC(UNPACK_SEQUENCE, hit); + val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); + val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); + stack_pointer[-1] = val1; + stack_pointer[0] = val0; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(seq); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + TRACING_DISPATCH(); + } + + TRACING_TARGET(WITH_EXCEPT_START) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = WITH_EXCEPT_START; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(WITH_EXCEPT_START); + opcode = WITH_EXCEPT_START; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef exit_func; + _PyStackRef exit_self; + _PyStackRef lasti; + _PyStackRef val; + _PyStackRef res; + val = stack_pointer[-1]; + lasti = stack_pointer[-3]; + exit_self = stack_pointer[-4]; + exit_func = stack_pointer[-5]; + PyObject *exc, *tb; + PyObject *val_o = PyStackRef_AsPyObjectBorrow(val); + PyObject *exit_func_o = PyStackRef_AsPyObjectBorrow(exit_func); + assert(val_o && PyExceptionInstance_Check(val_o)); + exc = PyExceptionInstance_Class(val_o); + PyObject *original_tb = tb = PyException_GetTraceback(val_o); + if (tb == NULL) { + tb = Py_None; + } + assert(PyStackRef_IsTaggedInt(lasti)); + (void)lasti; + PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; + int has_self = !PyStackRef_IsNull(exit_self); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, + (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + Py_XDECREF(original_tb); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + TRACING_JUMP_TO_LABEL(error); + } + res = PyStackRef_FromPyObjectSteal(res_o); + stack_pointer[0] = res; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + + TRACING_TARGET(YIELD_VALUE) { + assert(IS_JIT_TRACING()); + #if _Py_TAIL_CALL_INTERP + int opcode = YIELD_VALUE; + (void)(opcode); + #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(YIELD_VALUE); + opcode = YIELD_VALUE; + PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); + (void)old_code; + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + (void)old_func; + int _jump_taken = false; + (void)_jump_taken; + _PyStackRef retval; + _PyStackRef value; + retval = stack_pointer[-1]; + assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); + frame->instr_ptr++; + PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + _PyStackRef temp = retval; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + #if TIER_ONE + assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || + frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #endif + stack_pointer = _PyFrame_GetStackPointer(frame); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + #if TIER_TWO + frame->instr_ptr += (1 + INLINE_CACHE_ENTRIES_SEND); + #endif + value = PyStackRef_MakeHeapSafe(temp); + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); + TRACING_DISPATCH(); + } + #endif /* END TRACING INSTRUCTIONS */ + #undef TRACING_JIT + #undef TIER_ONE diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 05243195807ccf..939035332cba72 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -5,7 +5,6 @@ import argparse -from tracer_generator import generate_tracer_cases from analyzer import ( Analysis, Instruction, @@ -171,7 +170,6 @@ def generate_tier1( out = CWriter(outfile, 2, lines) emitter = Emitter(out, analysis.labels) generate_tier1_cases(analysis, out, emitter) - generate_tracer_cases(analysis, out) outfile.write(f""" {INSTRUCTION_END_MARKER} #if !_Py_TAIL_CALL_INTERP @@ -256,12 +254,12 @@ def generate_tier1_cases( if is_tracing: # This is required so that the predicted ops reflect the correct opcode. out.emit(f"opcode = {name};\n") - out.emit(f"PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable);\n") - out.emit(f"(void)old_code;\n") - out.emit(f"PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj);\n") - out.emit(f"(void)old_func;\n") - out.emit(f"int _jump_taken = false;\n") - out.emit(f"(void)_jump_taken;\n") + out.emit(f"PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable);\n") + out.emit(f"(void)old_code;\n") + out.emit(f"PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj);\n") + out.emit(f"(void)old_func;\n") + out.emit(f"int _jump_taken = false;\n") + out.emit(f"(void)_jump_taken;\n") if inst.properties.uses_opcode: out.emit(f"opcode = {name};\n") if inst.family is not None: diff --git a/Tools/cases_generator/tracer_generator.py b/Tools/cases_generator/tracer_generator.py index 61088ff185e2d5..ebc9a63664e8f2 100644 --- a/Tools/cases_generator/tracer_generator.py +++ b/Tools/cases_generator/tracer_generator.py @@ -28,6 +28,9 @@ from typing import TextIO from lexer import Token from stack import Local, Stack, StackError, get_stack_effect, Storage +from tier1_generator import generate_tier1_cases + +DEFAULT_OUTPUT = ROOT / "Python/generated_tracer_cases.c.h" class TracerEmitter(Emitter): out: CWriter @@ -115,8 +118,51 @@ def deopt_if( def generate_tracer_cases( analysis: Analysis, out: CWriter ): - # Import inside to avoid circular imports. - from tier1_generator import generate_tier1_cases out.emit(f"#ifdef _Py_TIER2 /* BEGIN TRACING INSTRUCTIONS */\n") generate_tier1_cases(analysis, out, TracerEmitter(out, analysis.labels), is_tracing=True) - out.emit(f"#endif /* END TRACING INSTRUCTIONS */\n") \ No newline at end of file + out.emit(f"#endif /* END TRACING INSTRUCTIONS */\n") + +def generate_tracer( + filenames: list[str], analysis: Analysis, outfile: TextIO, lines: bool +) -> None: + write_header(__file__, filenames, outfile) + out = CWriter(outfile, 2, lines) + out.emit("#define TIER_ONE 1\n") + out.emit("#define TRACING_JIT 1\n") + generate_tracer_cases(analysis, out) + out.emit("#undef TRACING_JIT\n") + out.emit("#undef TIER_ONE\n") + +# For use in unittest +def generate_tracer_from_files( + filenames: list[str], outfilename: str, lines: bool +) -> None: + data = analyze_files(filenames) + with open(outfilename, "w") as outfile: + generate_tracer(filenames, data, outfile, lines) + + +arg_parser = argparse.ArgumentParser( + description="Generate the code for the interpreter switch.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, +) + +arg_parser.add_argument( + "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT +) + +arg_parser.add_argument( + "-l", "--emit-line-directives", help="Emit #line directives", action="store_true" +) + +arg_parser.add_argument( + "input", nargs=argparse.REMAINDER, help="Instruction definition file(s)" +) + +if __name__ == "__main__": + args = arg_parser.parse_args() + if len(args.input) == 0: + args.input.append(DEFAULT_INPUT) + data = analyze_files(args.input) + with open(args.output, "w") as outfile: + generate_tracer(args.input, data, outfile, args.emit_line_directives) \ No newline at end of file From d11494423e7a96c941f291270b24d48a0c7fb352 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 18 Oct 2025 21:56:18 +0100 Subject: [PATCH 051/190] Cleanup, bugfixes to sys trace --- .gitattributes | 1 + Objects/frameobject.c | 5 +++++ Python/bytecodes.c | 7 +++++-- Python/ceval_macros.h | 7 +++++-- Python/generated_cases.c.h | 7 +++++-- Python/generated_tracer_cases.c.h | 7 +++++-- Python/instrumentation.c | 2 ++ Python/optimizer.c | 7 +++---- Tools/c-analyzer/cpython/_parser.py | 1 + 9 files changed, 32 insertions(+), 12 deletions(-) diff --git a/.gitattributes b/.gitattributes index 823e3e975a23a8..58cd96c0d3290d 100644 --- a/.gitattributes +++ b/.gitattributes @@ -97,6 +97,7 @@ Programs/test_frozenmain.h generated Python/Python-ast.c generated Python/executor_cases.c.h generated Python/generated_cases.c.h generated +Python/generated_tracer_cases.c.h generated Python/optimizer_cases.c.h generated Python/opcode_targets.h generated Python/stdlib_module_names.h generated diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 0cae3703d1d0c6..33e57fbfc245b2 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -17,6 +17,8 @@ #include "frameobject.h" // PyFrameLocalsProxyObject #include "opcode.h" // EXTENDED_ARG +#include "../Include/pytypedefs.h" +#include "../Include/internal/pycore_optimizer.h" #include "clinic/frameobject.c.h" @@ -260,7 +262,10 @@ framelocalsproxy_setitem(PyObject *self, PyObject *key, PyObject *value) return -1; } +#if _Py_TIER2 _Py_Executors_InvalidateDependency(PyInterpreterState_Get(), co, 1); + _Py_JITTracer_InvalidateDependency(PyThreadState_GET(), co); +#endif _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); _PyStackRef oldvalue = fast[i]; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 5b47782ea60087..08fc82a7dd34b6 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2971,8 +2971,11 @@ dummy_func( DISPATCH(); } } - _PyJIT_InitializeTracing(tstate, frame, this_instr, STACK_LEVEL(), 0, NULL); - ENTER_TRACING(); + int _is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL); + if (!_is_sys_tracing) { + _PyJIT_InitializeTracing(tstate, frame, this_instr, STACK_LEVEL(), 0, NULL); + ENTER_TRACING(); + } // Don't add the JUMP_BACKWARD_JIT instruction to the trace. DISPATCH(); } diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index f68e84e973896e..39a57a6f830c82 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -161,8 +161,11 @@ } \ } while (0); # define RECORD_TRACE_NO_DISPATCH() do { \ - int _is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc == NULL); \ - if (_is_sys_tracing || IS_JIT_TRACING() && add_to_code_trace(tstate, frame, old_code, old_func, this_instr, next_instr, opcode, oparg, _jump_taken)) { \ + int _is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL); \ + if (_is_sys_tracing) { \ + LEAVE_TRACING(); \ + } \ + else if ((IS_JIT_TRACING() && add_to_code_trace(tstate, frame, old_code, old_func, this_instr, next_instr, opcode, oparg, _jump_taken))) { \ BAIL_TRACING_NO_DISPATCH(); \ } \ } while (0); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index c47b6bec7b27ce..3595fb6d9376fc 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -7668,8 +7668,11 @@ DISPATCH(); } } - _PyJIT_InitializeTracing(tstate, frame, this_instr, STACK_LEVEL(), 0, NULL); - ENTER_TRACING(); + int _is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL); + if (!_is_sys_tracing) { + _PyJIT_InitializeTracing(tstate, frame, this_instr, STACK_LEVEL(), 0, NULL); + ENTER_TRACING(); + } DISPATCH(); } else { diff --git a/Python/generated_tracer_cases.c.h b/Python/generated_tracer_cases.c.h index 1871af274084fc..1f08126774180d 100644 --- a/Python/generated_tracer_cases.c.h +++ b/Python/generated_tracer_cases.c.h @@ -8711,8 +8711,11 @@ TRACING_DISPATCH(); } } - _PyJIT_InitializeTracing(tstate, frame, this_instr, STACK_LEVEL(), 0, NULL); - ENTER_TRACING(); + int _is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL); + if (!_is_sys_tracing) { + _PyJIT_InitializeTracing(tstate, frame, this_instr, STACK_LEVEL(), 0, NULL); + ENTER_TRACING(); + } TRACING_DISPATCH(); } else { diff --git a/Python/instrumentation.c b/Python/instrumentation.c index b4b2bc5dc69f9d..325acc34a1b5ee 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -18,6 +18,7 @@ #include "pycore_tuple.h" // _PyTuple_FromArraySteal() #include "opcode_ids.h" +#include "../Include/internal/pycore_optimizer.h" /* Uncomment this to dump debugging output when assertions fail */ @@ -1785,6 +1786,7 @@ force_instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp) _PyCode_Clear_Executors(code); } _Py_Executors_InvalidateDependency(interp, code, 1); + _Py_JITTracer_InvalidateDependency(PyThreadState_GET(), code); #endif int code_len = (int)Py_SIZE(code); /* Exit early to avoid creating instrumentation diff --git a/Python/optimizer.c b/Python/optimizer.c index f3166afd7d59a6..8b16edab7cd218 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -649,8 +649,6 @@ _PyJIT_translate_single_bytecode_to_trace( const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; - RESERVE_RAW(expansion->nuops + needs_guard_ip + 3, "uop and various checks"); - ADD_TO_TRACE(_CHECK_VALIDITY, 0, 0, target); assert(opcode != ENTER_EXECUTOR && opcode != EXTENDED_ARG); @@ -686,6 +684,9 @@ _PyJIT_translate_single_bytecode_to_trace( max_length--; } + RESERVE_RAW(expansion->nuops + needs_guard_ip + 3, "uop and various checks"); + + switch (opcode) { case POP_JUMP_IF_NONE: case POP_JUMP_IF_NOT_NONE: @@ -1543,7 +1544,6 @@ _Py_JITTracer_InvalidateDependency(PyThreadState *old_tstate, void *obj) _PyBloomFilter obj_filter; _Py_BloomFilter_Init(&obj_filter); _Py_BloomFilter_Add(&obj_filter, obj); - HEAD_LOCK(&_PyRuntime); PyInterpreterState *interp = old_tstate->interp; @@ -1554,7 +1554,6 @@ _Py_JITTracer_InvalidateDependency(PyThreadState *old_tstate, void *obj) } } - HEAD_UNLOCK(&_PyRuntime); } /* Invalidate all executors */ void diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index 17c56c9e8762c6..bee987cb53adab 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -78,6 +78,7 @@ def format_tsv_lines(lines): 'Python/deepfreeze/*.c', 'Python/frozen_modules/*.h', 'Python/generated_cases.c.h', + 'Python/generated_tracer_cases.c.h', 'Python/executor_cases.c.h', 'Python/optimizer_cases.c.h', 'Python/opcode_targets.h', From e50ff65a4bc0a71a026817eddef86e0c64b81283 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 18 Oct 2025 21:59:06 +0100 Subject: [PATCH 052/190] Whole test suite passing --- Python/optimizer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/optimizer.c b/Python/optimizer.c index 8b16edab7cd218..39c1cfd6d697b7 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -684,7 +684,7 @@ _PyJIT_translate_single_bytecode_to_trace( max_length--; } - RESERVE_RAW(expansion->nuops + needs_guard_ip + 3, "uop and various checks"); + RESERVE_RAW(expansion->nuops + needs_guard_ip + 4, "uop and various checks"); switch (opcode) { From 54f6cd6bb72f30fda9d54e273f7118c39d470e69 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 18 Oct 2025 22:00:28 +0100 Subject: [PATCH 053/190] Remove unused buffer --- Include/internal/pycore_interp_structs.h | 1 - Python/optimizer.c | 6 ------ Python/pystate.c | 6 ------ 3 files changed, 13 deletions(-) diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 42295de90478f9..222db736a9f4e6 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -959,7 +959,6 @@ struct _is { // End Jit tracing state bool jit; bool compiling; - struct _PyUOpInstruction *jit_uop_buffer; struct _PyExecutorObject *executor_list_head; struct _PyExecutorObject *executor_deletion_list_head; struct _PyExecutorObject *cold_executor; diff --git a/Python/optimizer.c b/Python/optimizer.c index 39c1cfd6d697b7..5fdd73d1e6220a 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1195,12 +1195,6 @@ uop_optimize( { _PyBloomFilter *dependencies = &tstate->interp->jit_tracer_dependencies; PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->jit_uop_buffer == NULL) { - interp->jit_uop_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); - if (interp->jit_uop_buffer == NULL) { - return 0; - } - } _PyUOpInstruction *buffer = interp->jit_tracer_code_buffer; OPT_STAT_INC(attempts); char *env_var = Py_GETENV("PYTHON_UOPS_OPTIMIZE"); diff --git a/Python/pystate.c b/Python/pystate.c index 671daabd1edb23..aebe7c107d1b6c 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -553,8 +553,6 @@ init_interpreter(PyInterpreterState *interp, #endif #ifdef _Py_TIER2 - // Ensure the buffer is to be set as NULL. - interp->jit_uop_buffer = NULL; interp->jit_tracer_code_buffer = NULL; interp->jit_tracer_initial_instr = NULL; interp->jit_tracer_initial_stack_depth = -1; @@ -810,10 +808,6 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) #ifdef _Py_TIER2 _Py_ClearExecutorDeletionList(interp); - if (interp->jit_uop_buffer != NULL) { - _PyObject_VirtualFree(interp->jit_uop_buffer, UOP_BUFFER_SIZE); - interp->jit_uop_buffer = NULL; - } if (interp->jit_tracer_code_buffer != NULL) { _PyObject_VirtualFree(interp->jit_tracer_code_buffer, UOP_BUFFER_SIZE); interp->jit_tracer_code_buffer = NULL; From e3f18e645f05463ec012b242d18ea936e76a7e21 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 18 Oct 2025 22:01:15 +0100 Subject: [PATCH 054/190] Cleanup warnings --- Python/optimizer.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Python/optimizer.c b/Python/optimizer.c index 5fdd73d1e6220a..2d6f7f824c7370 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -117,7 +117,6 @@ Py_NO_INLINE int _PyOptimizer_Optimize( _PyInterpreterFrame *frame, PyThreadState *tstate) { - _PyStackRef *stack_pointer = frame->stackpointer; PyInterpreterState *interp = _PyInterpreterState_GET(); int chain_depth = tstate->interp->jit_tracer_initial_chain_depth; assert(interp->jit); @@ -143,7 +142,7 @@ _PyOptimizer_Optimize( } // We are the only one still holding a reference to this code object that // is practically dead. - if (_PyObject_IsUniquelyReferenced(code) || _PyObject_IsUniquelyReferenced(tstate->interp->jit_tracer_initial_func)) { + if (_PyObject_IsUniquelyReferenced((PyObject *)code) || _PyObject_IsUniquelyReferenced((PyObject *)tstate->interp->jit_tracer_initial_func)) { interp->compiling = false; return 0; } From 608772fd36c261b2cacebc19cc968871f0a0a77b Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 18 Oct 2025 22:08:20 +0100 Subject: [PATCH 055/190] refactor a little --- Include/internal/pycore_interp_structs.h | 30 +++---- Python/bytecodes.c | 6 +- Python/ceval.c | 2 +- Python/ceval_macros.h | 2 +- Python/generated_cases.c.h | 6 +- Python/generated_tracer_cases.c.h | 6 +- Python/optimizer.c | 100 +++++++++++------------ Python/pystate.c | 18 ++-- 8 files changed, 86 insertions(+), 84 deletions(-) diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 222db736a9f4e6..db42904985ecde 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -762,6 +762,21 @@ struct _Py_unique_id_pool { typedef _Py_CODEUNIT *(*_PyJitEntryFuncPtr)(struct _PyExecutorObject *exec, _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate); +typedef struct _PyJitState { + int jit_tracer_code_max_size; + int jit_tracer_code_curr_size; + _PyBloomFilter jit_tracer_dependencies; + bool jit_tracer_dependencies_still_valid; + _PyUOpInstruction *jit_tracer_code_buffer; + _Py_CODEUNIT *jit_tracer_initial_instr; + int jit_tracer_initial_stack_depth; + int jit_tracer_initial_chain_depth; + PyCodeObject *jit_tracer_initial_code; // Strong + PyFunctionObject *jit_tracer_initial_func; // Strong + struct _PyExitData *jit_tracer_previous_exit; + _PyInterpreterFrame *jit_tracer_current_frame; +} _PyJitState; + /* PyInterpreterState holds the global state for one of the runtime's interpreters. Typically the initial (main) interpreter is the only one. @@ -943,20 +958,7 @@ struct _is { struct types_state types; struct callable_cache callable_cache; PyObject *common_consts[NUM_COMMON_CONSTANTS]; - // JIT tracing state - int jit_tracer_code_max_size; - int jit_tracer_code_curr_size; - _PyBloomFilter jit_tracer_dependencies; - bool jit_tracer_dependencies_still_valid; - _PyUOpInstruction *jit_tracer_code_buffer; - _Py_CODEUNIT *jit_tracer_initial_instr; - int jit_tracer_initial_stack_depth; - int jit_tracer_initial_chain_depth; - PyCodeObject *jit_tracer_initial_code; // Strong - PyFunctionObject *jit_tracer_initial_func; // Strong - struct _PyExitData *jit_tracer_previous_exit; - _PyInterpreterFrame *jit_tracer_current_frame; - // End Jit tracing state + _PyJitState jit_state; bool jit; bool compiling; struct _PyExecutorObject *executor_list_head; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 08fc82a7dd34b6..e9ced8297c3375 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2964,9 +2964,9 @@ dummy_func( if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD_JIT && next_instr->op.code != ENTER_EXECUTOR) { - if (tstate->interp->jit_tracer_code_buffer == NULL) { - tstate->interp->jit_tracer_code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); - if (tstate->interp->jit_tracer_code_buffer == NULL) { + if (tstate->interp->jit_state.jit_tracer_code_buffer == NULL) { + tstate->interp->jit_state.jit_tracer_code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); + if (tstate->interp->jit_state.jit_tracer_code_buffer == NULL) { // Don't error, just go to next instruction. DISPATCH(); } diff --git a/Python/ceval.c b/Python/ceval.c index 560c0eb364c939..55e86f81b01a39 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -978,7 +978,7 @@ static int add_to_code_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, PyCodeObject *old_code, PyFunctionObject *old_func, _Py_CODEUNIT *this_instr, _Py_CODEUNIT *next_instr, int opcode, int oparg, int jump_taken) { assert(frame != NULL); - assert(tstate->interp->jit_tracer_code_curr_size < UOP_MAX_TRACE_LENGTH); + assert(tstate->interp->jit_state.jit_tracer_code_curr_size < UOP_MAX_TRACE_LENGTH); return !_PyJIT_translate_single_bytecode_to_trace(tstate, frame, this_instr, next_instr, old_code, old_func, opcode, oparg, jump_taken); } diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 39a57a6f830c82..086abfc0535b1a 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -434,7 +434,7 @@ do { \ } \ if (keep_tracing_bit) { \ assert(next_instr->op.code != ENTER_EXECUTOR); \ - assert(tstate->interp->jit_tracer_code_curr_size == 2); \ + assert(tstate->interp->jit_state.jit_tracer_code_curr_size == 2); \ ENTER_TRACING(); \ } \ DISPATCH(); \ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 3595fb6d9376fc..f8230170a25e8f 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -7660,11 +7660,11 @@ if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD_JIT && next_instr->op.code != ENTER_EXECUTOR) { - if (tstate->interp->jit_tracer_code_buffer == NULL) { + if (tstate->interp->jit_state.jit_tracer_code_buffer == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); - tstate->interp->jit_tracer_code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); + tstate->interp->jit_state.jit_tracer_code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); stack_pointer = _PyFrame_GetStackPointer(frame); - if (tstate->interp->jit_tracer_code_buffer == NULL) { + if (tstate->interp->jit_state.jit_tracer_code_buffer == NULL) { DISPATCH(); } } diff --git a/Python/generated_tracer_cases.c.h b/Python/generated_tracer_cases.c.h index 1f08126774180d..52364a8743b35c 100644 --- a/Python/generated_tracer_cases.c.h +++ b/Python/generated_tracer_cases.c.h @@ -8703,11 +8703,11 @@ if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD_JIT && next_instr->op.code != ENTER_EXECUTOR) { - if (tstate->interp->jit_tracer_code_buffer == NULL) { + if (tstate->interp->jit_state.jit_tracer_code_buffer == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); - tstate->interp->jit_tracer_code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); + tstate->interp->jit_state.jit_tracer_code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); stack_pointer = _PyFrame_GetStackPointer(frame); - if (tstate->interp->jit_tracer_code_buffer == NULL) { + if (tstate->interp->jit_state.jit_tracer_code_buffer == NULL) { TRACING_DISPATCH(); } } diff --git a/Python/optimizer.c b/Python/optimizer.c index 2d6f7f824c7370..bb3154dd0f2ca0 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -118,10 +118,10 @@ _PyOptimizer_Optimize( _PyInterpreterFrame *frame, PyThreadState *tstate) { PyInterpreterState *interp = _PyInterpreterState_GET(); - int chain_depth = tstate->interp->jit_tracer_initial_chain_depth; + int chain_depth = tstate->interp->jit_state.jit_tracer_initial_chain_depth; assert(interp->jit); assert(!interp->compiling); - assert(tstate->interp->jit_tracer_initial_stack_depth >= 0); + assert(tstate->interp->jit_state.jit_tracer_initial_stack_depth >= 0); interp->compiling = true; // The first executor in a chain and the MAX_CHAIN_DEPTH'th executor *must* // make progress in order to avoid infinite loops or excessively-long @@ -129,8 +129,8 @@ _PyOptimizer_Optimize( // this is true, since a deopt won't infinitely re-enter the executor: chain_depth %= MAX_CHAIN_DEPTH; bool progress_needed = chain_depth == 0; - PyCodeObject *code = (PyCodeObject *)tstate->interp->jit_tracer_initial_code; - _Py_CODEUNIT *start = tstate->interp->jit_tracer_initial_instr; + PyCodeObject *code = (PyCodeObject *)tstate->interp->jit_state.jit_tracer_initial_code; + _Py_CODEUNIT *start = tstate->interp->jit_state.jit_tracer_initial_instr; // A recursive trace might've cleared the values. In that case, bail. if (code == NULL) { interp->compiling = false; @@ -142,12 +142,12 @@ _PyOptimizer_Optimize( } // We are the only one still holding a reference to this code object that // is practically dead. - if (_PyObject_IsUniquelyReferenced((PyObject *)code) || _PyObject_IsUniquelyReferenced((PyObject *)tstate->interp->jit_tracer_initial_func)) { + if (_PyObject_IsUniquelyReferenced((PyObject *)code) || _PyObject_IsUniquelyReferenced((PyObject *)tstate->interp->jit_state.jit_tracer_initial_func)) { interp->compiling = false; return 0; } // One of our depencies while tracing was invalidated. Not worth compiling. - if (!tstate->interp->jit_tracer_dependencies_still_valid) { + if (!tstate->interp->jit_state.jit_tracer_dependencies_still_valid) { interp->compiling = false; return 0; } @@ -177,7 +177,7 @@ _PyOptimizer_Optimize( executor->vm_data.code = NULL; } if (chain_depth > 0) { - _PyExitData *prev_exit = tstate->interp->jit_tracer_previous_exit; + _PyExitData *prev_exit = tstate->interp->jit_state.jit_tracer_previous_exit; assert(prev_exit != NULL); prev_exit->executor = executor;; } @@ -565,14 +565,14 @@ _PyJIT_translate_single_bytecode_to_trace( int jump_taken) { - int is_first_instr = tstate->interp->jit_tracer_initial_instr == this_instr; - bool progress_needed = (tstate->interp->jit_tracer_initial_chain_depth % MAX_CHAIN_DEPTH) == 0 && is_first_instr;; - _PyBloomFilter *dependencies = &tstate->interp->jit_tracer_dependencies; + int is_first_instr = tstate->interp->jit_state.jit_tracer_initial_instr == this_instr; + bool progress_needed = (tstate->interp->jit_state.jit_tracer_initial_chain_depth % MAX_CHAIN_DEPTH) == 0 && is_first_instr;; + _PyBloomFilter *dependencies = &tstate->interp->jit_state.jit_tracer_dependencies; _Py_BloomFilter_Add(dependencies, old_code); _Py_CODEUNIT *target_instr = this_instr; - int trace_length = tstate->interp->jit_tracer_code_curr_size; - _PyUOpInstruction *trace = tstate->interp->jit_tracer_code_buffer; - int max_length = tstate->interp->jit_tracer_code_max_size; + int trace_length = tstate->interp->jit_state.jit_tracer_code_curr_size; + _PyUOpInstruction *trace = tstate->interp->jit_state.jit_tracer_code_buffer; + int max_length = tstate->interp->jit_state.jit_tracer_code_max_size; #ifdef Py_DEBUG char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); @@ -595,7 +595,7 @@ _PyJIT_translate_single_bytecode_to_trace( if (jump_taken || // This happens when a recursive call happens that we can't trace. Such as Python -> C -> Python calls // If we haven't guarded the IP, then it's untraceable. - (frame != tstate->interp->jit_tracer_current_frame && !needs_guard_ip) || + (frame != tstate->interp->jit_state.jit_tracer_current_frame && !needs_guard_ip) || // TODO handle extended args. oparg > 255 || opcode == EXTENDED_ARG || // TODO handle BINARY_OP_INPLACE_ADD_UNICODE @@ -627,7 +627,7 @@ _PyJIT_translate_single_bytecode_to_trace( } } - tstate->interp->jit_tracer_current_frame = frame; + tstate->interp->jit_state.jit_tracer_current_frame = frame; DPRINTF(2, "%p %d: %s(%d)\n", old_code, target, _PyOpcode_OpName[opcode], oparg); @@ -668,7 +668,7 @@ _PyJIT_translate_single_bytecode_to_trace( } // Loop back to the start - if (is_first_instr && tstate->interp->jit_tracer_code_curr_size > 2) { + if (is_first_instr && tstate->interp->jit_state.jit_tracer_code_curr_size > 2) { ADD_TO_TRACE(_CHECK_PERIODIC, 0, 0, target); ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0, 0); goto done; @@ -822,24 +822,24 @@ _PyJIT_translate_single_bytecode_to_trace( if (needs_guard_ip) { ADD_TO_TRACE(_GUARD_IP, 0, (uintptr_t)next_instr, 0); } - tstate->interp->jit_tracer_code_curr_size = trace_length; - tstate->interp->jit_tracer_code_max_size = max_length; + tstate->interp->jit_state.jit_tracer_code_curr_size = trace_length; + tstate->interp->jit_state.jit_tracer_code_max_size = max_length; return 1; done: - tstate->interp->jit_tracer_code_curr_size = trace_length; - tstate->interp->jit_tracer_code_max_size = max_length; + tstate->interp->jit_state.jit_tracer_code_curr_size = trace_length; + tstate->interp->jit_state.jit_tracer_code_max_size = max_length; return 0; full: - if (!is_terminator(&tstate->interp->jit_tracer_code_buffer[trace_length-1])) { + if (!is_terminator(&tstate->interp->jit_state.jit_tracer_code_buffer[trace_length-1])) { // Undo the last few instructions. - trace_length = tstate->interp->jit_tracer_code_curr_size; - max_length = tstate->interp->jit_tracer_code_max_size; + trace_length = tstate->interp->jit_state.jit_tracer_code_curr_size; + max_length = tstate->interp->jit_state.jit_tracer_code_max_size; // We previously reversed one. max_length += 1; ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); } - tstate->interp->jit_tracer_code_curr_size = trace_length; - tstate->interp->jit_tracer_code_max_size = max_length; + tstate->interp->jit_state.jit_tracer_code_curr_size = trace_length; + tstate->interp->jit_state.jit_tracer_code_max_size = max_length; return 0; } @@ -860,28 +860,28 @@ _PyJIT_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_ code->co_firstlineno, 2 * INSTR_IP(next_instr, code)); #endif - add_to_trace(tstate->interp->jit_tracer_code_buffer, 0, _START_EXECUTOR, 0, (uintptr_t)next_instr, INSTR_IP(next_instr, code)); - add_to_trace(tstate->interp->jit_tracer_code_buffer, 1, _MAKE_WARM, 0, 0, 0); - tstate->interp->jit_tracer_code_curr_size = 2; - tstate->interp->jit_tracer_code_max_size = UOP_MAX_TRACE_LENGTH; - tstate->interp->jit_tracer_initial_instr = next_instr; - tstate->interp->jit_tracer_initial_code = (PyCodeObject *)Py_NewRef(code); - tstate->interp->jit_tracer_initial_func = (PyFunctionObject *)Py_NewRef(_PyFrame_GetFunction(frame)); - tstate->interp->jit_tracer_previous_exit = exit; - _Py_BloomFilter_Init(&tstate->interp->jit_tracer_dependencies); - tstate->interp->jit_tracer_initial_stack_depth = curr_stackdepth; - tstate->interp->jit_tracer_initial_chain_depth = chain_depth; - tstate->interp->jit_tracer_current_frame = frame; - tstate->interp->jit_tracer_dependencies_still_valid = true; + add_to_trace(tstate->interp->jit_state.jit_tracer_code_buffer, 0, _START_EXECUTOR, 0, (uintptr_t)next_instr, INSTR_IP(next_instr, code)); + add_to_trace(tstate->interp->jit_state.jit_tracer_code_buffer, 1, _MAKE_WARM, 0, 0, 0); + tstate->interp->jit_state.jit_tracer_code_curr_size = 2; + tstate->interp->jit_state.jit_tracer_code_max_size = UOP_MAX_TRACE_LENGTH; + tstate->interp->jit_state.jit_tracer_initial_instr = next_instr; + tstate->interp->jit_state.jit_tracer_initial_code = (PyCodeObject *)Py_NewRef(code); + tstate->interp->jit_state.jit_tracer_initial_func = (PyFunctionObject *)Py_NewRef(_PyFrame_GetFunction(frame)); + tstate->interp->jit_state.jit_tracer_previous_exit = exit; + _Py_BloomFilter_Init(&tstate->interp->jit_state.jit_tracer_dependencies); + tstate->interp->jit_state.jit_tracer_initial_stack_depth = curr_stackdepth; + tstate->interp->jit_state.jit_tracer_initial_chain_depth = chain_depth; + tstate->interp->jit_state.jit_tracer_current_frame = frame; + tstate->interp->jit_state.jit_tracer_dependencies_still_valid = true; } void _PyJIT_FinalizeTracing(PyThreadState *tstate) { - Py_CLEAR(tstate->interp->jit_tracer_initial_code); - Py_CLEAR(tstate->interp->jit_tracer_initial_func); - tstate->interp->jit_tracer_code_curr_size = 2; - tstate->interp->jit_tracer_code_max_size = UOP_MAX_TRACE_LENGTH - 1; + Py_CLEAR(tstate->interp->jit_state.jit_tracer_initial_code); + Py_CLEAR(tstate->interp->jit_state.jit_tracer_initial_func); + tstate->interp->jit_state.jit_tracer_code_curr_size = 2; + tstate->interp->jit_state.jit_tracer_code_max_size = UOP_MAX_TRACE_LENGTH - 1; } @@ -1192,17 +1192,17 @@ uop_optimize( _PyExecutorObject **exec_ptr, bool progress_needed) { - _PyBloomFilter *dependencies = &tstate->interp->jit_tracer_dependencies; + _PyBloomFilter *dependencies = &tstate->interp->jit_state.jit_tracer_dependencies; PyInterpreterState *interp = _PyInterpreterState_GET(); - _PyUOpInstruction *buffer = interp->jit_tracer_code_buffer; + _PyUOpInstruction *buffer = interp->jit_state.jit_tracer_code_buffer; OPT_STAT_INC(attempts); char *env_var = Py_GETENV("PYTHON_UOPS_OPTIMIZE"); bool is_noopt = true; if (env_var == NULL || *env_var == '\0' || *env_var > '0') { is_noopt = false; } - int curr_stackentries = tstate->interp->jit_tracer_initial_stack_depth; - int length = interp->jit_tracer_code_curr_size; + int curr_stackentries = tstate->interp->jit_state.jit_tracer_initial_stack_depth; + int length = interp->jit_state.jit_tracer_code_curr_size; // Trace too short, don't bother. if (length <= 5) { return 0; @@ -1211,7 +1211,7 @@ uop_optimize( assert(length < UOP_MAX_TRACE_LENGTH); OPT_STAT_INC(traces_created); if (!is_noopt) { - length = _Py_uop_analyze_and_optimize(tstate->interp->jit_tracer_initial_func, buffer, + length = _Py_uop_analyze_and_optimize(tstate->interp->jit_state.jit_tracer_initial_func, buffer, length, curr_stackentries, dependencies); if (length <= 0) { @@ -1236,7 +1236,7 @@ uop_optimize( OPT_HIST(effective_trace_length(buffer, length), optimized_trace_length_hist); length = prepare_for_execution(buffer, length); assert(length <= UOP_MAX_TRACE_LENGTH); - _PyExecutorObject *executor = make_executor_from_uops(buffer, length, dependencies, tstate->interp->jit_tracer_initial_chain_depth); + _PyExecutorObject *executor = make_executor_from_uops(buffer, length, dependencies, tstate->interp->jit_state.jit_tracer_initial_chain_depth); if (executor == NULL) { return -1; } @@ -1541,9 +1541,9 @@ _Py_JITTracer_InvalidateDependency(PyThreadState *old_tstate, void *obj) PyInterpreterState *interp = old_tstate->interp; _Py_FOR_EACH_TSTATE_UNLOCKED(interp, tstate) { - if (bloom_filter_may_contain(&tstate->interp->jit_tracer_dependencies, &obj_filter)) + if (bloom_filter_may_contain(&tstate->interp->jit_state.jit_tracer_dependencies, &obj_filter)) { - tstate->interp->jit_tracer_dependencies_still_valid = false; + tstate->interp->jit_state.jit_tracer_dependencies_still_valid = false; } } diff --git a/Python/pystate.c b/Python/pystate.c index aebe7c107d1b6c..1bb116743c304c 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -553,12 +553,12 @@ init_interpreter(PyInterpreterState *interp, #endif #ifdef _Py_TIER2 - interp->jit_tracer_code_buffer = NULL; - interp->jit_tracer_initial_instr = NULL; - interp->jit_tracer_initial_stack_depth = -1; - interp->jit_tracer_initial_chain_depth = -1; - interp->jit_tracer_initial_code = NULL; - interp->jit_tracer_initial_func = NULL; + interp->jit_state.jit_tracer_code_buffer = NULL; + interp->jit_state.jit_tracer_initial_instr = NULL; + interp->jit_state.jit_tracer_initial_stack_depth = -1; + interp->jit_state.jit_tracer_initial_chain_depth = -1; + interp->jit_state.jit_tracer_initial_code = NULL; + interp->jit_state.jit_tracer_initial_func = NULL; #endif llist_init(&interp->mem_free_queue.head); llist_init(&interp->asyncio_tasks_head); @@ -808,9 +808,9 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) #ifdef _Py_TIER2 _Py_ClearExecutorDeletionList(interp); - if (interp->jit_tracer_code_buffer != NULL) { - _PyObject_VirtualFree(interp->jit_tracer_code_buffer, UOP_BUFFER_SIZE); - interp->jit_tracer_code_buffer = NULL; + if (interp->jit_state.jit_tracer_code_buffer != NULL) { + _PyObject_VirtualFree(interp->jit_state.jit_tracer_code_buffer, UOP_BUFFER_SIZE); + interp->jit_state.jit_tracer_code_buffer = NULL; } #endif _PyAST_Fini(interp); From 187271537966c029be6b881e92aa41d74d31780d Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 18 Oct 2025 22:28:24 +0100 Subject: [PATCH 056/190] Cleanup --- Include/internal/pycore_opcode_metadata.h | 4 +-- Include/internal/pycore_uop_metadata.h | 4 --- Objects/frameobject.c | 2 +- Python/bytecodes.c | 32 +++++++++++++---------- Python/ceval.c | 28 ++++++++++---------- Python/ceval_macros.h | 3 +-- Python/executor_cases.c.h | 23 ++++++++-------- Python/generated_cases.c.h | 26 +++++++++--------- Python/generated_tracer_cases.c.h | 26 +++++++++--------- Python/instrumentation.c | 2 +- Python/optimizer.c | 14 ++++++++++ Python/optimizer_cases.c.h | 4 +-- Tools/cases_generator/tier2_generator.py | 2 +- 13 files changed, 92 insertions(+), 78 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 477553e2563f8c..5d642604515e46 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1406,8 +1406,8 @@ _PyOpcode_macro_expansion[256] = { [IMPORT_FROM] = { .nuops = 1, .uops = { { _IMPORT_FROM, OPARG_SIMPLE, 0 } } }, [IMPORT_NAME] = { .nuops = 1, .uops = { { _IMPORT_NAME, OPARG_SIMPLE, 0 } } }, [IS_OP] = { .nuops = 1, .uops = { { _IS_OP, OPARG_SIMPLE, 0 } } }, - [JUMP_BACKWARD_NO_INTERRUPT] = { .nuops = 1, .uops = { { _JUMP_BACKWARD_NO_INTERRUPT, OPARG_SIMPLE, 0 } } }, - [JUMP_BACKWARD_NO_JIT] = { .nuops = 2, .uops = { { _CHECK_PERIODIC, OPARG_SIMPLE, 1 }, { _JUMP_BACKWARD_NO_INTERRUPT, OPARG_SIMPLE, 1 } } }, + [JUMP_BACKWARD_NO_INTERRUPT] = { .nuops = 1, .uops = { { _JUMP_BACKWARD_NO_INTERRUPT, OPARG_REPLACED, 0 } } }, + [JUMP_BACKWARD_NO_JIT] = { .nuops = 2, .uops = { { _CHECK_PERIODIC, OPARG_SIMPLE, 1 }, { _JUMP_BACKWARD_NO_INTERRUPT, OPARG_REPLACED, 1 } } }, [LIST_APPEND] = { .nuops = 1, .uops = { { _LIST_APPEND, OPARG_SIMPLE, 0 } } }, [LIST_EXTEND] = { .nuops = 1, .uops = { { _LIST_EXTEND, OPARG_SIMPLE, 0 } } }, [LOAD_ATTR] = { .nuops = 1, .uops = { { _LOAD_ATTR, OPARG_SIMPLE, 8 } } }, diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 79f0b7a68d1f44..82a0fa92652524 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -206,7 +206,6 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_IMPORT_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_IMPORT_FROM] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_IS_NONE] = HAS_ESCAPES_FLAG, - [_JUMP_BACKWARD_NO_INTERRUPT] = HAS_ARG_FLAG | HAS_JUMP_FLAG, [_GET_LEN] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_MATCH_CLASS] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_MATCH_MAPPING] = 0, @@ -524,7 +523,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_ITER_NEXT_LIST_TIER_TWO] = "_ITER_NEXT_LIST_TIER_TWO", [_ITER_NEXT_RANGE] = "_ITER_NEXT_RANGE", [_ITER_NEXT_TUPLE] = "_ITER_NEXT_TUPLE", - [_JUMP_BACKWARD_NO_INTERRUPT] = "_JUMP_BACKWARD_NO_INTERRUPT", [_JUMP_TO_TOP] = "_JUMP_TO_TOP", [_LIST_APPEND] = "_LIST_APPEND", [_LIST_EXTEND] = "_LIST_EXTEND", @@ -1049,8 +1047,6 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _IS_NONE: return 1; - case _JUMP_BACKWARD_NO_INTERRUPT: - return 0; case _GET_LEN: return 0; case _MATCH_CLASS: diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 33e57fbfc245b2..156daef7f9c6ad 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -18,7 +18,7 @@ #include "frameobject.h" // PyFrameLocalsProxyObject #include "opcode.h" // EXTENDED_ARG #include "../Include/pytypedefs.h" -#include "../Include/internal/pycore_optimizer.h" +#include "pycore_optimizer.h" #include "clinic/frameobject.c.h" diff --git a/Python/bytecodes.c b/Python/bytecodes.c index e9ced8297c3375..4c3a595a57d2d8 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1251,10 +1251,12 @@ dummy_func( frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); RELOAD_STACK(); + #if TIER_ONE LOAD_IP(frame->return_offset); -#if TIER_TWO - frame->instr_ptr += frame->return_offset; -#endif + #endif + #if TIER_TWO + TIER2_STORE_IP(frame->return_offset); + #endif res = temp; LLTRACE_RESUME_FRAME(); } @@ -1441,10 +1443,12 @@ dummy_func( _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); #endif RELOAD_STACK(); + #if TIER_ONE LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); -#if TIER_TWO - frame->instr_ptr += (1 + INLINE_CACHE_ENTRIES_SEND); -#endif + #endif + #if TIER_TWO + TIER2_STORE_IP(1 + INLINE_CACHE_ENTRIES_SEND); + #endif value = PyStackRef_MakeHeapSafe(temp); LLTRACE_RESUME_FRAME(); } @@ -3084,15 +3088,13 @@ dummy_func( macro(POP_JUMP_IF_NOT_NONE) = unused/1 + _IS_NONE + _POP_JUMP_IF_FALSE; - inst(JUMP_BACKWARD_NO_INTERRUPT, (--)) { + replaced inst(JUMP_BACKWARD_NO_INTERRUPT, (--)) { /* This bytecode is used in the `yield from` or `await` loop. * If there is an interrupt, we want it handled in the innermost * generator or coroutine, so we deliberately do not check it here. * (see bpo-30039). */ -#if TIER_ONE assert(oparg <= INSTR_OFFSET()); -#endif JUMPBY(-oparg); } @@ -3233,15 +3235,15 @@ dummy_func( } op(_FOR_ITER_TIER_TWO, (iter, null_or_index -- iter, null_or_index, next)) { - TIER2_JUMPBY(1 + INLINE_CACHE_ENTRIES_FOR_ITER); + TIER2_STORE_IP(1 + INLINE_CACHE_ENTRIES_FOR_ITER); _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); if (!PyStackRef_IsValid(item)) { if (PyStackRef_IsError(item)) { ERROR_NO_POP(); } /* iterator ended normally */ - // Jump forward by oparg and skip the following END_FOR - TIER2_JUMPBY(oparg + 1); + /* This just sets the IP to what it expects */ + TIER2_STORE_IP(oparg + 1); EXIT_IF(true); } next = item; @@ -4992,10 +4994,12 @@ dummy_func( _PyInterpreterFrame *prev = frame->previous; _PyThreadState_PopFrame(tstate, frame); frame = tstate->current_frame = prev; + #if TIER_ONE LOAD_IP(frame->return_offset); -#if TIER_TWO + #endif + #if TIER_TWO frame->instr_ptr += (frame->return_offset); -#endif + #endif RELOAD_STACK(); res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); LLTRACE_RESUME_FRAME(); diff --git a/Python/ceval.c b/Python/ceval.c index 55e86f81b01a39..c16d26b5411921 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1194,26 +1194,26 @@ _PyTier2Interpreter( for (;;) { uopcode = next_uop->opcode; #ifdef Py_DEBUG - if (frame->lltrace >= 2) { + if (frame->lltrace >= 4) { if (next_uop->opcode != _YIELD_VALUE && next_uop->opcode != _FOR_ITER_GEN_FRAME && next_uop->opcode != _PUSH_FRAME && next_uop->opcode != _PY_FRAME_KW && next_uop->opcode != _SAVE_RETURN_OFFSET && next_uop->opcode != _SAVE_RETURN_OFFSET) { - // if (next_uop->opcode != _START_EXECUTOR) { - // if (next_uop->format == UOP_FORMAT_TARGET) { - // _Py_CODEUNIT *aim = _PyFrame_GetBytecode(frame) + next_uop->target; - // printf(" aim=[%s]\n", _PyOpcode_OpName[aim->op.code]); - // } - // else if (next_uop->format == UOP_FORMAT_JUMP) { - // _PyUOpInstruction *aim_uop = current_executor->trace + next_uop->jump_target; - // if (aim_uop->format == UOP_FORMAT_TARGET) { - // _Py_CODEUNIT *aim = _PyFrame_GetBytecode(frame) + aim_uop->target; - // printf(" aim=[%s]\n", _PyOpcode_OpName[aim->op.code]); - // } - // } - // } + if (next_uop->opcode != _START_EXECUTOR) { + if (next_uop->format == UOP_FORMAT_TARGET) { + _Py_CODEUNIT *aim = _PyFrame_GetBytecode(frame) + next_uop->target; + printf(" aim=[%s]\n", _PyOpcode_OpName[aim->op.code]); + } + else if (next_uop->format == UOP_FORMAT_JUMP) { + _PyUOpInstruction *aim_uop = current_executor->trace + next_uop->jump_target; + if (aim_uop->format == UOP_FORMAT_TARGET) { + _Py_CODEUNIT *aim = _PyFrame_GetBytecode(frame) + aim_uop->target; + printf(" aim=[%s]\n", _PyOpcode_OpName[aim->op.code]); + } + } + } dump_stack(frame, stack_pointer); } if (next_uop->opcode == _START_EXECUTOR) { diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 086abfc0535b1a..dab0ff33c9ae03 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -78,7 +78,6 @@ # define TAIL_CALL_ARGS frame, stack_pointer, tstate, next_instr, instruction_funcptr_table, oparg #endif - #if _Py_TAIL_CALL_INTERP // Note: [[clang::musttail]] works for GCC 15, but not __attribute__((musttail)) at the moment. # define Py_MUSTTAIL [[clang::musttail]] @@ -267,7 +266,7 @@ GETITEM(PyObject *v, Py_ssize_t i) { * and skipped instructions. */ #define JUMPBY(x) (next_instr += (x)) -#define TIER2_JUMPBY(x) (frame->instr_ptr += (x)) +#define TIER2_STORE_IP(x) (frame->instr_ptr += (x)) #define SKIP_OVER(x) (next_instr += (x)) #define STACK_LEVEL() ((int)(stack_pointer - _PyFrame_Stackbase(frame))) diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 0a664209830f1f..ade852159afb42 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1930,9 +1930,11 @@ frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); stack_pointer = _PyFrame_GetStackPointer(frame); + #if TIER_ONE LOAD_IP(frame->return_offset); + #endif #if TIER_TWO - frame->instr_ptr += frame->return_offset; + TIER2_STORE_IP(frame->return_offset); #endif res = temp; LLTRACE_RESUME_FRAME(); @@ -2099,9 +2101,11 @@ _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); #endif stack_pointer = _PyFrame_GetStackPointer(frame); + #if TIER_ONE LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + #endif #if TIER_TWO - frame->instr_ptr += (1 + INLINE_CACHE_ENTRIES_SEND); + TIER2_STORE_IP(1 + INLINE_CACHE_ENTRIES_SEND); #endif value = PyStackRef_MakeHeapSafe(temp); LLTRACE_RESUME_FRAME(); @@ -4195,14 +4199,7 @@ break; } - case _JUMP_BACKWARD_NO_INTERRUPT: { - oparg = CURRENT_OPARG(); - #if TIER_ONE - assert(oparg <= INSTR_OFFSET()); - #endif - TIER2_JUMPBY(-oparg); - break; - } + /* _JUMP_BACKWARD_NO_INTERRUPT is not a viable micro-op for tier 2 because it is replaced */ case _GET_LEN: { _PyStackRef obj; @@ -4401,7 +4398,7 @@ oparg = CURRENT_OPARG(); null_or_index = stack_pointer[-1]; iter = stack_pointer[-2]; - TIER2_JUMPBY(1 + INLINE_CACHE_ENTRIES_FOR_ITER); + TIER2_STORE_IP(1 + INLINE_CACHE_ENTRIES_FOR_ITER); _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -4409,7 +4406,7 @@ if (PyStackRef_IsError(item)) { JUMP_TO_ERROR(); } - TIER2_JUMPBY(oparg + 1); + TIER2_STORE_IP(oparg + 1); if (true) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -6766,7 +6763,9 @@ _PyInterpreterFrame *prev = frame->previous; _PyThreadState_PopFrame(tstate, frame); frame = tstate->current_frame = prev; + #if TIER_ONE LOAD_IP(frame->return_offset); + #endif #if TIER_TWO frame->instr_ptr += (frame->return_offset); #endif diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index f8230170a25e8f..9c9eaa29ed8141 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -7432,9 +7432,11 @@ frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); stack_pointer = _PyFrame_GetStackPointer(frame); + #if TIER_ONE LOAD_IP(frame->return_offset); + #endif #if TIER_TWO - frame->instr_ptr += frame->return_offset; + TIER2_STORE_IP(frame->return_offset); #endif res = temp; LLTRACE_RESUME_FRAME(); @@ -7503,9 +7505,11 @@ _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); #endif stack_pointer = _PyFrame_GetStackPointer(frame); + #if TIER_ONE LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + #endif #if TIER_TWO - frame->instr_ptr += (1 + INLINE_CACHE_ENTRIES_SEND); + TIER2_STORE_IP(1 + INLINE_CACHE_ENTRIES_SEND); #endif value = PyStackRef_MakeHeapSafe(temp); LLTRACE_RESUME_FRAME(); @@ -7617,9 +7621,7 @@ } // _JUMP_BACKWARD_NO_INTERRUPT { - #if TIER_ONE assert(oparg <= INSTR_OFFSET()); - #endif JUMPBY(-oparg); } DISPATCH(); @@ -7648,9 +7650,7 @@ } // _JUMP_BACKWARD_NO_INTERRUPT { - #if TIER_ONE assert(oparg <= INSTR_OFFSET()); - #endif JUMPBY(-oparg); } // _JIT @@ -7691,9 +7691,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); - #if TIER_ONE assert(oparg <= INSTR_OFFSET()); - #endif JUMPBY(-oparg); DISPATCH(); } @@ -7719,9 +7717,7 @@ } // _JUMP_BACKWARD_NO_INTERRUPT { - #if TIER_ONE assert(oparg <= INSTR_OFFSET()); - #endif JUMPBY(-oparg); } DISPATCH(); @@ -10366,7 +10362,9 @@ _PyInterpreterFrame *prev = frame->previous; _PyThreadState_PopFrame(tstate, frame); frame = tstate->current_frame = prev; + #if TIER_ONE LOAD_IP(frame->return_offset); + #endif #if TIER_TWO frame->instr_ptr += (frame->return_offset); #endif @@ -10401,9 +10399,11 @@ frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); stack_pointer = _PyFrame_GetStackPointer(frame); + #if TIER_ONE LOAD_IP(frame->return_offset); + #endif #if TIER_TWO - frame->instr_ptr += frame->return_offset; + TIER2_STORE_IP(frame->return_offset); #endif res = temp; LLTRACE_RESUME_FRAME(); @@ -12092,9 +12092,11 @@ _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); #endif stack_pointer = _PyFrame_GetStackPointer(frame); + #if TIER_ONE LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + #endif #if TIER_TWO - frame->instr_ptr += (1 + INLINE_CACHE_ENTRIES_SEND); + TIER2_STORE_IP(1 + INLINE_CACHE_ENTRIES_SEND); #endif value = PyStackRef_MakeHeapSafe(temp); LLTRACE_RESUME_FRAME(); diff --git a/Python/generated_tracer_cases.c.h b/Python/generated_tracer_cases.c.h index 52364a8743b35c..48e71c54c7a55a 100644 --- a/Python/generated_tracer_cases.c.h +++ b/Python/generated_tracer_cases.c.h @@ -8431,9 +8431,11 @@ frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); stack_pointer = _PyFrame_GetStackPointer(frame); + #if TIER_ONE LOAD_IP(frame->return_offset); + #endif #if TIER_TWO - frame->instr_ptr += frame->return_offset; + TIER2_STORE_IP(frame->return_offset); #endif res = temp; LLTRACE_RESUME_FRAME(); @@ -8510,9 +8512,11 @@ _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); #endif stack_pointer = _PyFrame_GetStackPointer(frame); + #if TIER_ONE LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + #endif #if TIER_TWO - frame->instr_ptr += (1 + INLINE_CACHE_ENTRIES_SEND); + TIER2_STORE_IP(1 + INLINE_CACHE_ENTRIES_SEND); #endif value = PyStackRef_MakeHeapSafe(temp); LLTRACE_RESUME_FRAME(); @@ -8652,9 +8656,7 @@ } // _JUMP_BACKWARD_NO_INTERRUPT { - #if TIER_ONE assert(oparg <= INSTR_OFFSET()); - #endif JUMPBY(-oparg); } TRACING_DISPATCH(); @@ -8691,9 +8693,7 @@ } // _JUMP_BACKWARD_NO_INTERRUPT { - #if TIER_ONE assert(oparg <= INSTR_OFFSET()); - #endif JUMPBY(-oparg); } // _JIT @@ -8744,9 +8744,7 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; - #if TIER_ONE assert(oparg <= INSTR_OFFSET()); - #endif JUMPBY(-oparg); TRACING_DISPATCH(); } @@ -8782,9 +8780,7 @@ } // _JUMP_BACKWARD_NO_INTERRUPT { - #if TIER_ONE assert(oparg <= INSTR_OFFSET()); - #endif JUMPBY(-oparg); } TRACING_DISPATCH(); @@ -12011,7 +12007,9 @@ _PyInterpreterFrame *prev = frame->previous; _PyThreadState_PopFrame(tstate, frame); frame = tstate->current_frame = prev; + #if TIER_ONE LOAD_IP(frame->return_offset); + #endif #if TIER_TWO frame->instr_ptr += (frame->return_offset); #endif @@ -12056,9 +12054,11 @@ frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); stack_pointer = _PyFrame_GetStackPointer(frame); + #if TIER_ONE LOAD_IP(frame->return_offset); + #endif #if TIER_TWO - frame->instr_ptr += frame->return_offset; + TIER2_STORE_IP(frame->return_offset); #endif res = temp; LLTRACE_RESUME_FRAME(); @@ -14087,9 +14087,11 @@ _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); #endif stack_pointer = _PyFrame_GetStackPointer(frame); + #if TIER_ONE LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + #endif #if TIER_TWO - frame->instr_ptr += (1 + INLINE_CACHE_ENTRIES_SEND); + TIER2_STORE_IP(1 + INLINE_CACHE_ENTRIES_SEND); #endif value = PyStackRef_MakeHeapSafe(temp); LLTRACE_RESUME_FRAME(); diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 325acc34a1b5ee..85fb73aaf098ee 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -18,7 +18,7 @@ #include "pycore_tuple.h" // _PyTuple_FromArraySteal() #include "opcode_ids.h" -#include "../Include/internal/pycore_optimizer.h" +#include "pycore_optimizer.h" /* Uncomment this to dump debugging output when assertions fail */ diff --git a/Python/optimizer.c b/Python/optimizer.c index bb3154dd0f2ca0..824f67d0a126ed 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -701,6 +701,20 @@ _PyJIT_translate_single_bytecode_to_trace( break; } case JUMP_BACKWARD_JIT: + // This is possible as the JIT might have re-activated after it was disabled. // if (next_uop->opcode != _START_EXECUTOR) { + // if (next_uop->format == UOP_FORMAT_TARGET) { + // _Py_CODEUNIT *aim = _PyFrame_GetBytecode(frame) + next_uop->target; + // printf(" aim=[%s]\n", _PyOpcode_OpName[aim->op.code]); + // } + // else if (next_uop->format == UOP_FORMAT_JUMP) { + // _PyUOpInstruction *aim_uop = current_executor->trace + next_uop->jump_target; + // if (aim_uop->format == UOP_FORMAT_TARGET) { + // _Py_CODEUNIT *aim = _PyFrame_GetBytecode(frame) + aim_uop->target; + // printf(" aim=[%s]\n", _PyOpcode_OpName[aim->op.code]); + // } + // } + // } + case JUMP_BACKWARD_NO_JIT: case JUMP_BACKWARD: ADD_TO_TRACE(_CHECK_PERIODIC, 0, 0, target); _Py_FALLTHROUGH; diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index f5bb8a5585af93..99d8d8ae33397d 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -2107,9 +2107,7 @@ break; } - case _JUMP_BACKWARD_NO_INTERRUPT: { - break; - } + /* _JUMP_BACKWARD_NO_INTERRUPT is not a viable micro-op for tier 2 */ case _GET_LEN: { JitOptRef obj; diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index 28adb132874d7a..87b26b1c732599 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -147,7 +147,7 @@ def jumpby( if storage.spilled: raise analysis_error("stack_pointer needs reloading before dispatch", tkn) storage.stack.flush(self.out) - self.emit("TIER2_JUMPBY") + self.emit("TIER2_STORE_IP") emit_to(self.out, tkn_iter, "SEMI") self.emit(";\n") return True From 460fb39d431079e1805b871b70ea063b58bec3a8 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Sat, 18 Oct 2025 21:50:49 +0000 Subject: [PATCH 057/190] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by?= =?UTF-8?q?=20blurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2025-10-18-21-50-44.gh-issue-139109.9QQOzN.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-50-44.gh-issue-139109.9QQOzN.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-50-44.gh-issue-139109.9QQOzN.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-50-44.gh-issue-139109.9QQOzN.rst new file mode 100644 index 00000000000000..01e2efa4df4590 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-50-44.gh-issue-139109.9QQOzN.rst @@ -0,0 +1 @@ +A new tracing frontend for the JIT compiler has been implemented. Patch by Ken Jin. Design for CPython by Brandt Bucher, Mark Shannon and Ken Jin. From a8762c25c35b4276b2d96c7941f8388477063eec Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 18 Oct 2025 23:04:11 +0100 Subject: [PATCH 058/190] Disable windows CI for now, simplify --- .github/workflows/jit.yml | 24 ++++++++++++------------ Python/generated_tracer_cases.c.h | 5 +++-- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index c32bf4fd63cc8f..48b2e943e818c8 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -57,9 +57,9 @@ jobs: fail-fast: false matrix: target: - - i686-pc-windows-msvc/msvc - - x86_64-pc-windows-msvc/msvc - - aarch64-pc-windows-msvc/msvc +# - i686-pc-windows-msvc/msvc +# - x86_64-pc-windows-msvc/msvc +# - aarch64-pc-windows-msvc/msvc - x86_64-apple-darwin/clang - aarch64-apple-darwin/clang - x86_64-unknown-linux-gnu/gcc @@ -70,15 +70,15 @@ jobs: llvm: - 19 include: - - target: i686-pc-windows-msvc/msvc - architecture: Win32 - runner: windows-2022 - - target: x86_64-pc-windows-msvc/msvc - architecture: x64 - runner: windows-2022 - - target: aarch64-pc-windows-msvc/msvc - architecture: ARM64 - runner: windows-11-arm +# - target: i686-pc-windows-msvc/msvc +# architecture: Win32 +# runner: windows-2022 +# - target: x86_64-pc-windows-msvc/msvc +# architecture: x64 +# runner: windows-2022 +# - target: aarch64-pc-windows-msvc/msvc +# architecture: ARM64 +# runner: windows-11-arm - target: x86_64-apple-darwin/clang architecture: x86_64 runner: macos-15-intel diff --git a/Python/generated_tracer_cases.c.h b/Python/generated_tracer_cases.c.h index 48e71c54c7a55a..67c240669b9dd8 100644 --- a/Python/generated_tracer_cases.c.h +++ b/Python/generated_tracer_cases.c.h @@ -8637,9 +8637,10 @@ /* Skip 1 cache entry */ // _SPECIALIZE_JUMP_BACKWARD { - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT if (this_instr->op.code == JUMP_BACKWARD) { - this_instr->op.code = (tstate->interp->jit && !PyStackRef_IsNull(frame->f_funcobj)) ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; + uint8_t desired = tstate->interp->jit ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; + FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, desired); next_instr = this_instr; DISPATCH_SAME_OPARG(); } From 72c22423c8dd92b674ecdab3a1063595cf6add27 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 18 Oct 2025 23:12:55 +0100 Subject: [PATCH 059/190] restore non-jit builds --- Python/opcode_targets.h | 2 ++ Tools/cases_generator/target_generator.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 299739764919bb..66abd6d4623e1c 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -257,6 +257,7 @@ static void *opcode_targets_table[256] = { &&TARGET_INSTRUMENTED_LINE, &&TARGET_ENTER_EXECUTOR, }; +#if _Py_TIER2 static void *opcode_tracing_targets_table[256] = { &&TARGET_TRACING_CACHE, &&TARGET_TRACING_BINARY_SLICE, @@ -515,6 +516,7 @@ static void *opcode_tracing_targets_table[256] = { &&TARGET_TRACING_INSTRUMENTED_LINE, &&TARGET_TRACING_ENTER_EXECUTOR, }; +#endif #else /* _Py_TAIL_CALL_INTERP */ static py_tail_call_funcptr instruction_funcptr_handler_table[256]; diff --git a/Tools/cases_generator/target_generator.py b/Tools/cases_generator/target_generator.py index b6570d202d847d..8e5dc10f759a15 100644 --- a/Tools/cases_generator/target_generator.py +++ b/Tools/cases_generator/target_generator.py @@ -35,10 +35,12 @@ def write_opcode_targets(analysis: Analysis, out: CWriter) -> None: for name, op in analysis.opmap.items(): if op < 256: targets[op] = f"&&TARGET_TRACING_{name},\n" + out.emit("#if _Py_TIER2\n") out.emit("static void *opcode_tracing_targets_table[256] = {\n") for target in targets: out.emit(target) out.emit("};\n") + out.emit(f"#endif\n") out.emit("#else /* _Py_TAIL_CALL_INTERP */\n") def function_proto(name: str) -> str: From d76dc85f4b7ccf18eb28510f36e2b8fcd7ce2bff Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 18 Oct 2025 23:17:27 +0100 Subject: [PATCH 060/190] make mypy happy --- Tools/cases_generator/tier2_generator.py | 2 +- Tools/cases_generator/tracer_generator.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index 87b26b1c732599..0f7284fadf79e5 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -93,7 +93,7 @@ def deopt_if( self.emit("}\n") return not always_true(first_tkn) - def exit_if( + def exit_if( # type: ignore[override] self, tkn: Token, tkn_iter: TokenIterator, diff --git a/Tools/cases_generator/tracer_generator.py b/Tools/cases_generator/tracer_generator.py index ebc9a63664e8f2..5594d0bd4b7c03 100644 --- a/Tools/cases_generator/tracer_generator.py +++ b/Tools/cases_generator/tracer_generator.py @@ -117,7 +117,7 @@ def deopt_if( def generate_tracer_cases( analysis: Analysis, out: CWriter -): +) -> None: out.emit(f"#ifdef _Py_TIER2 /* BEGIN TRACING INSTRUCTIONS */\n") generate_tier1_cases(analysis, out, TracerEmitter(out, analysis.labels), is_tracing=True) out.emit(f"#endif /* END TRACING INSTRUCTIONS */\n") From 87c0b72d075ccf93f2649279f5d739986297eb5f Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 18 Oct 2025 23:21:36 +0100 Subject: [PATCH 061/190] fix linter and mypy? --- Tools/cases_generator/tier2_generator.py | 2 +- Tools/cases_generator/tracer_generator.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index 0f7284fadf79e5..87b26b1c732599 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -93,7 +93,7 @@ def deopt_if( self.emit("}\n") return not always_true(first_tkn) - def exit_if( # type: ignore[override] + def exit_if( self, tkn: Token, tkn_iter: TokenIterator, diff --git a/Tools/cases_generator/tracer_generator.py b/Tools/cases_generator/tracer_generator.py index 5594d0bd4b7c03..30a34a91d65b2e 100644 --- a/Tools/cases_generator/tracer_generator.py +++ b/Tools/cases_generator/tracer_generator.py @@ -165,4 +165,4 @@ def generate_tracer_from_files( args.input.append(DEFAULT_INPUT) data = analyze_files(args.input) with open(args.output, "w") as outfile: - generate_tracer(args.input, data, outfile, args.emit_line_directives) \ No newline at end of file + generate_tracer(args.input, data, outfile, args.emit_line_directives) From 24cd7f9f7a16562a10a8c72d4213df084ef58358 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 18 Oct 2025 23:26:00 +0100 Subject: [PATCH 062/190] more cleanup to fix CI --- Python/ceval.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Python/ceval.c b/Python/ceval.c index 0d63c312278928..b16fcdb51cd9b4 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -984,6 +984,7 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch) } } +#if _Py_TIER2 // 1 for trace full, 0 for successful write. static int add_to_code_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, PyCodeObject *old_code, PyFunctionObject *old_func, _Py_CODEUNIT *this_instr, _Py_CODEUNIT *next_instr, int opcode, int oparg, int jump_taken) @@ -992,6 +993,7 @@ add_to_code_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, PyCodeObjec assert(tstate->interp->jit_state.jit_tracer_code_curr_size < UOP_MAX_TRACE_LENGTH); return !_PyJIT_translate_single_bytecode_to_trace(tstate, frame, this_instr, next_instr, old_code, old_func, opcode, oparg, jump_taken); } +#endif /* _PyEval_EvalFrameDefault is too large to optimize for speed with PGO on MSVC. */ From f38ef69e2cf1865f4803a5724ae472c9deadf216 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 19 Oct 2025 00:35:34 +0100 Subject: [PATCH 063/190] Fix lltrace on jit debug builds --- Tools/jit/template.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Tools/jit/template.c b/Tools/jit/template.c index d40217bc87ff51..471e052c2d8d52 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -69,9 +69,11 @@ do { \ } while (0) #undef LLTRACE_RESUME_FRAME -#define LLTRACE_RESUME_FRAME() \ - do { \ - } while (0) +#ifdef Py_DEBUG +#define LLTRACE_RESUME_FRAME() frame->lltrace = 0; +#else +#define LLTRACE_RESUME_FRAME() +#endif #define PATCH_JUMP(ALIAS) \ do { \ From 960d647c75ce80b1787b9f8dcb6f8e0b1d6528f9 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 19 Oct 2025 14:53:33 +0100 Subject: [PATCH 064/190] Turn off tracing on dynamic exit --- Python/bytecodes.c | 11 +---------- Python/executor_cases.c.h | 11 +---------- 2 files changed, 2 insertions(+), 20 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index ce2ff2c666f805..38ac989e6652fb 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5482,16 +5482,7 @@ dummy_func( TIER2_TO_TIER2(executor); } else { - if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) { - GOTO_TIER_ONE(target, 0); - } - if (!backoff_counter_triggers(temperature)) { - exit->temperature = advance_backoff_counter(temperature); - GOTO_TIER_ONE(target, 0); - } - exit->temperature = initial_temperature_backoff_counter(); - _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), 0, NULL); - GOTO_TIER_ONE(target, 1); + GOTO_TIER_ONE(target, 0); } } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index ade852159afb42..8b2b6b89927454 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7546,16 +7546,7 @@ TIER2_TO_TIER2(executor); } else { - if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) { - GOTO_TIER_ONE(target, 0); - } - if (!backoff_counter_triggers(temperature)) { - exit->temperature = advance_backoff_counter(temperature); - GOTO_TIER_ONE(target, 0); - } - exit->temperature = initial_temperature_backoff_counter(); - _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), 0, NULL); - GOTO_TIER_ONE(target, 1); + GOTO_TIER_ONE(target, 0); } break; } From 1798ab1b338399bd8559819faceb54fd71c2bf90 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 19 Oct 2025 15:51:26 +0100 Subject: [PATCH 065/190] Fix _CHECK_PERIODIC insertion --- Python/bytecodes.c | 6 ++++-- Python/generated_cases.c.h | 5 ++++- Python/generated_tracer_cases.c.h | 3 +++ Python/optimizer.c | 15 +-------------- 4 files changed, 12 insertions(+), 17 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 38ac989e6652fb..e75f1d05a4849a 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2981,8 +2981,10 @@ dummy_func( _PyJIT_InitializeTracing(tstate, frame, this_instr, STACK_LEVEL(), 0, NULL); ENTER_TRACING(); } - // Don't add the JUMP_BACKWARD_JIT instruction to the trace. - DISPATCH(); + int _jump_taken = false; + PyCodeObject *old_code = _PyFrame_GetCode(frame); + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + TRACING_DISPATCH(); } else { ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 5080df123dc00a..7b7d10e526505d 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -7674,7 +7674,10 @@ _PyJIT_InitializeTracing(tstate, frame, this_instr, STACK_LEVEL(), 0, NULL); ENTER_TRACING(); } - DISPATCH(); + int _jump_taken = false; + PyCodeObject *old_code = _PyFrame_GetCode(frame); + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + TRACING_DISPATCH(); } else { ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); diff --git a/Python/generated_tracer_cases.c.h b/Python/generated_tracer_cases.c.h index 67c240669b9dd8..3feeb0d05362fd 100644 --- a/Python/generated_tracer_cases.c.h +++ b/Python/generated_tracer_cases.c.h @@ -8717,6 +8717,9 @@ _PyJIT_InitializeTracing(tstate, frame, this_instr, STACK_LEVEL(), 0, NULL); ENTER_TRACING(); } + int _jump_taken = false; + PyCodeObject *old_code = _PyFrame_GetCode(frame); + PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); TRACING_DISPATCH(); } else { diff --git a/Python/optimizer.c b/Python/optimizer.c index a6d9c7904ee638..18b7d1d5aa29ff 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -673,7 +673,6 @@ _PyJIT_translate_single_bytecode_to_trace( // Loop back to the start if (is_first_instr && tstate->interp->jit_state.jit_tracer_code_curr_size > 2) { - ADD_TO_TRACE(_CHECK_PERIODIC, 0, 0, target); ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0, 0); goto done; } @@ -705,19 +704,7 @@ _PyJIT_translate_single_bytecode_to_trace( break; } case JUMP_BACKWARD_JIT: - // This is possible as the JIT might have re-activated after it was disabled. // if (next_uop->opcode != _START_EXECUTOR) { - // if (next_uop->format == UOP_FORMAT_TARGET) { - // _Py_CODEUNIT *aim = _PyFrame_GetBytecode(frame) + next_uop->target; - // printf(" aim=[%s]\n", _PyOpcode_OpName[aim->op.code]); - // } - // else if (next_uop->format == UOP_FORMAT_JUMP) { - // _PyUOpInstruction *aim_uop = current_executor->trace + next_uop->jump_target; - // if (aim_uop->format == UOP_FORMAT_TARGET) { - // _Py_CODEUNIT *aim = _PyFrame_GetBytecode(frame) + aim_uop->target; - // printf(" aim=[%s]\n", _PyOpcode_OpName[aim->op.code]); - // } - // } - // } + // This is possible as the JIT might have re-activated after it was disabled case JUMP_BACKWARD_NO_JIT: case JUMP_BACKWARD: ADD_TO_TRACE(_CHECK_PERIODIC, 0, 0, target); From 681485ff25db7336925d88552111641b89035262 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 19 Oct 2025 16:06:03 +0100 Subject: [PATCH 066/190] Increase uop length to compensate --- Include/internal/pycore_uop.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/internal/pycore_uop.h b/Include/internal/pycore_uop.h index 6effe12915423a..86bea31204ca83 100644 --- a/Include/internal/pycore_uop.h +++ b/Include/internal/pycore_uop.h @@ -36,7 +36,7 @@ typedef struct _PyUOpInstruction{ } _PyUOpInstruction; // This is the length of the trace we translate initially. -#define UOP_MAX_TRACE_LENGTH 1200 +#define UOP_MAX_TRACE_LENGTH 1400 #define UOP_BUFFER_SIZE (UOP_MAX_TRACE_LENGTH * sizeof(_PyUOpInstruction)) /* Bloom filter with m = 256 From 9bb03a890e42d08fcb6cb2768cdbb8c251bcabc6 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 19 Oct 2025 20:26:01 +0100 Subject: [PATCH 067/190] Handle EXTENDED_ARG --- Python/bytecodes.c | 4 +- Python/ceval_macros.h | 2 +- Python/executor_cases.c.h | 3 +- Python/generated_tracer_cases.c.h | 1 + Python/optimizer.c | 67 +++++++++++++++++++------------ 5 files changed, 47 insertions(+), 30 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index e75f1d05a4849a..68a8d380643fe0 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1378,6 +1378,7 @@ dummy_func( if (err == 0) { assert(retval_o != NULL); JUMPBY(oparg); + RECORD_JUMP_TAKEN(); } else { PyStackRef_CLOSE(v); @@ -5464,9 +5465,8 @@ dummy_func( // from a single exit! tier2 op(_DYNAMIC_EXIT, (exit_p/4 --)) { _Py_CODEUNIT *target = frame->instr_ptr; - _PyExitData *exit = (_PyExitData *)exit_p; - _Py_BackoffCounter temperature = exit->temperature; #if defined(Py_DEBUG) && !defined(_Py_JIT) + _PyExitData *exit = (_PyExitData *)exit_p; OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); if (frame->lltrace >= 2) { printf("DYNAMIC EXIT: [UOp "); diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index dab0ff33c9ae03..3adbadab4c5344 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -426,7 +426,7 @@ do { \ frame = tstate->current_frame; \ stack_pointer = _PyFrame_GetStackPointer(frame); \ int keep_tracing_bit = (uintptr_t)next_instr & 1; \ - next_instr = (_Py_CODEUNIT *)(((uintptr_t)next_instr) >> 1 << 1); \ + next_instr = (_Py_CODEUNIT *)(((uintptr_t)next_instr) & (~1)); \ if (next_instr == NULL) { \ next_instr = frame->instr_ptr; \ JUMP_TO_LABEL(error); \ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 8b2b6b89927454..e32c144b9e9132 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7524,9 +7524,8 @@ case _DYNAMIC_EXIT: { PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); _Py_CODEUNIT *target = frame->instr_ptr; - _PyExitData *exit = (_PyExitData *)exit_p; - _Py_BackoffCounter temperature = exit->temperature; #if defined(Py_DEBUG) && !defined(_Py_JIT) + _PyExitData *exit = (_PyExitData *)exit_p; OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); if (frame->lltrace >= 2) { _PyFrame_SetStackPointer(frame, stack_pointer); diff --git a/Python/generated_tracer_cases.c.h b/Python/generated_tracer_cases.c.h index 3feeb0d05362fd..4c5a9c33fb963c 100644 --- a/Python/generated_tracer_cases.c.h +++ b/Python/generated_tracer_cases.c.h @@ -12162,6 +12162,7 @@ if (err == 0) { assert(retval_o != NULL); JUMPBY(oparg); + RECORD_JUMP_TAKEN(); } else { stack_pointer += -1; diff --git a/Python/optimizer.c b/Python/optimizer.c index 18b7d1d5aa29ff..a11244d4b54f74 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -570,10 +570,9 @@ _PyJIT_translate_single_bytecode_to_trace( { int is_first_instr = tstate->interp->jit_state.jit_tracer_initial_instr == this_instr; - bool progress_needed = (tstate->interp->jit_state.jit_tracer_initial_chain_depth % MAX_CHAIN_DEPTH) == 0 && is_first_instr;; + bool progress_needed = (tstate->interp->jit_state.jit_tracer_initial_chain_depth % MAX_CHAIN_DEPTH) == 0;; _PyBloomFilter *dependencies = &tstate->interp->jit_state.jit_tracer_dependencies; _Py_BloomFilter_Add(dependencies, old_code); - _Py_CODEUNIT *target_instr = this_instr; int trace_length = tstate->interp->jit_state.jit_tracer_code_curr_size; _PyUOpInstruction *trace = tstate->interp->jit_state.jit_tracer_code_buffer; int max_length = tstate->interp->jit_state.jit_tracer_code_max_size; @@ -585,11 +584,26 @@ _PyJIT_translate_single_bytecode_to_trace( lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that } #endif - + _Py_CODEUNIT *target_instr = this_instr; uint32_t target = 0; target = INSTR_IP(target_instr, old_code); + // Rewind EXTENDED_ARG so that we see the whole thing. + // We must point to the first EXTENDED_ARG when deopting. + int rewind_oparg = oparg; + while (rewind_oparg > 255) { + rewind_oparg >>= 8; + target--; + } +#ifdef Py_DEBUG + if (oparg > 255) { + assert(_Py_GetBaseCodeUnit(old_code, target).op.code == EXTENDED_ARG); + } +#endif + + DPRINTF(2, "%p %d: %s(%d) %d\n", old_code, target, _PyOpcode_OpName[opcode], oparg, progress_needed); + bool needs_guard_ip = _PyOpcode_NeedsGuardIp[opcode] && !(opcode == FOR_ITER_RANGE || opcode == FOR_ITER_LIST || opcode == FOR_ITER_TUPLE) && !(opcode == JUMP_BACKWARD_NO_INTERRUPT || opcode == JUMP_BACKWARD || opcode == JUMP_BACKWARD_JIT) && @@ -600,8 +614,7 @@ _PyJIT_translate_single_bytecode_to_trace( // This happens when a recursive call happens that we can't trace. Such as Python -> C -> Python calls // If we haven't guarded the IP, then it's untraceable. (frame != tstate->interp->jit_state.jit_tracer_current_frame && !needs_guard_ip) || - // TODO handle extended args. - oparg > 255 || opcode == EXTENDED_ARG || + (oparg > 0xFFFF) || // TODO handle BINARY_OP_INPLACE_ADD_UNICODE opcode == BINARY_OP_INPLACE_ADD_UNICODE || // TODO (gh-140277): The constituent uops are invalid. @@ -633,8 +646,6 @@ _PyJIT_translate_single_bytecode_to_trace( tstate->interp->jit_state.jit_tracer_current_frame = frame; - DPRINTF(2, "%p %d: %s(%d)\n", old_code, target, _PyOpcode_OpName[opcode], oparg); - if (opcode == NOP) { return 1; } @@ -643,6 +654,10 @@ _PyJIT_translate_single_bytecode_to_trace( return 1; } + if (opcode == EXTENDED_ARG) { + return 1; + } + // One for possible _DEOPT, one because _CHECK_VALIDITY itself might _DEOPT max_length -= 2; @@ -663,7 +678,7 @@ _PyJIT_translate_single_bytecode_to_trace( /* Special case the first instruction, * so that we can guarantee forward progress */ - if (progress_needed && is_first_instr) { + if (progress_needed && tstate->interp->jit_state.jit_tracer_code_curr_size <= 2) { if (OPCODE_HAS_EXIT(opcode) || OPCODE_HAS_DEOPT(opcode)) { opcode = _PyOpcode_Deopt[opcode]; } @@ -695,12 +710,13 @@ _PyJIT_translate_single_bytecode_to_trace( case POP_JUMP_IF_FALSE: case POP_JUMP_IF_TRUE: { - RESERVE(1); - _Py_CODEUNIT *computed_next_instr = target_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; - _Py_CODEUNIT *computed_jump_instr = computed_next_instr + oparg; - int jump_likely = computed_jump_instr == next_instr; - uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_likely]; - ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(jump_likely ? computed_next_instr : computed_jump_instr, old_code)); + _Py_CODEUNIT *computed_next_instr_without_modifiers = target_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; + _Py_CODEUNIT *computed_next_instr = computed_next_instr_without_modifiers + (computed_next_instr_without_modifiers->op.code == NOT_TAKEN); + _Py_CODEUNIT *computed_jump_instr = computed_next_instr_without_modifiers + oparg; + assert(next_instr == computed_next_instr || next_instr == computed_jump_instr); + int jump_happened = computed_jump_instr == next_instr; + uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_happened]; + ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(jump_happened ? computed_next_instr : computed_jump_instr, old_code)); break; } case JUMP_BACKWARD_JIT: @@ -731,8 +747,10 @@ _PyJIT_translate_single_bytecode_to_trace( assert(nuops > 0); RESERVE(nuops + 1); /* One extra for exit */ uint32_t orig_oparg = oparg; // For OPARG_TOP/BOTTOM + uint32_t orig_target = target; for (int i = 0; i < nuops; i++) { oparg = orig_oparg; + target = orig_target; uint32_t uop = expansion->uops[i].uop; uint64_t operand = 0; // Add one to account for the actual opcode/oparg pair: @@ -751,9 +769,11 @@ _PyJIT_translate_single_bytecode_to_trace( operand = read_u64(&this_instr[offset].cache); break; case OPARG_TOP: // First half of super-instr + assert(orig_oparg <= 255); oparg = orig_oparg >> 4; break; case OPARG_BOTTOM: // Second half of super-instr + assert(orig_oparg <= 255); oparg = orig_oparg & 0xF; break; case OPARG_SAVE_RETURN_OFFSET: // op=_SAVE_RETURN_OFFSET; oparg=return_offset @@ -768,13 +788,15 @@ _PyJIT_translate_single_bytecode_to_trace( if (uop == _TIER2_RESUME_CHECK) { target = next_inst; } -#ifdef Py_DEBUG else if (uop != _FOR_ITER_TIER_TWO) { - uint32_t jump_target = next_inst + oparg; + int extended_arg = orig_oparg > 255; + uint32_t jump_target = next_inst + orig_oparg + extended_arg; assert(_Py_GetBaseCodeUnit(old_code, jump_target).op.code == END_FOR); assert(_Py_GetBaseCodeUnit(old_code, jump_target+1).op.code == POP_ITER); + if (is_for_iter_test[uop]) { + target = jump_target + 1; + } } -#endif break; case OPERAND1_1: assert(trace[trace_length-1].opcode == uop); @@ -859,11 +881,12 @@ _PyJIT_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_ lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that } DPRINTF(2, - "Tracing %s (%s:%d) at byte offset %d\n", + "Tracing %s (%s:%d) at byte offset %d at chain depth %d\n", PyUnicode_AsUTF8(code->co_qualname), PyUnicode_AsUTF8(code->co_filename), code->co_firstlineno, - 2 * INSTR_IP(next_instr, code)); + 2 * INSTR_IP(next_instr, code), + chain_depth); #endif add_to_trace(tstate->interp->jit_state.jit_tracer_code_buffer, 0, _START_EXECUTOR, 0, (uintptr_t)next_instr, INSTR_IP(next_instr, code)); add_to_trace(tstate->interp->jit_state.jit_tracer_code_buffer, 1, _MAKE_WARM, 0, 0, 0); @@ -976,12 +999,6 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) exit_op = _DYNAMIC_EXIT; unique_target = true; } - if (is_for_iter_test[opcode]) { - /* Target the POP_TOP immediately after the END_FOR, - * leaving only the iterator on the stack. */ - int32_t next_inst = target + 1 + INLINE_CACHE_ENTRIES_FOR_ITER; - jump_target = next_inst + inst->oparg + 1; - } if (unique_target || jump_target != current_jump_target || current_exit_op != exit_op) { make_exit(&buffer[next_spare], exit_op, jump_target); current_exit_op = exit_op; From 4b26cdebacdbabbd0be3559ba5d2d26631e63660 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 20 Oct 2025 00:35:48 +0100 Subject: [PATCH 068/190] Handle unstable branches --- Python/bytecodes.c | 2 ++ Python/generated_cases.c.h | 12 ++++++++++++ Python/generated_tracer_cases.c.h | 4 ++++ Python/optimizer.c | 10 +++++++++- 4 files changed, 27 insertions(+), 1 deletion(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 68a8d380643fe0..42e323e3a4181a 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3063,6 +3063,7 @@ dummy_func( assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsFalse(cond); DEAD(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } @@ -3070,6 +3071,7 @@ dummy_func( assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsTrue(cond); DEAD(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 7b7d10e526505d..bd043ae7285d95 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -9977,6 +9977,8 @@ int opcode = POP_JUMP_IF_FALSE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_FALSE); @@ -9985,6 +9987,7 @@ cond = stack_pointer[-1]; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsFalse(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -9996,6 +9999,8 @@ int opcode = POP_JUMP_IF_NONE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_NONE); @@ -10024,6 +10029,7 @@ cond = b; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsTrue(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } stack_pointer += -1; @@ -10036,6 +10042,8 @@ int opcode = POP_JUMP_IF_NOT_NONE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); @@ -10064,6 +10072,7 @@ cond = b; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsFalse(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } stack_pointer += -1; @@ -10076,6 +10085,8 @@ int opcode = POP_JUMP_IF_TRUE; (void)(opcode); #endif + _Py_CODEUNIT* const this_instr = next_instr; + (void)this_instr; frame->instr_ptr = next_instr; next_instr += 2; INSTRUCTION_STATS(POP_JUMP_IF_TRUE); @@ -10084,6 +10095,7 @@ cond = stack_pointer[-1]; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsTrue(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); diff --git a/Python/generated_tracer_cases.c.h b/Python/generated_tracer_cases.c.h index 4c5a9c33fb963c..75e86d9c76d3cd 100644 --- a/Python/generated_tracer_cases.c.h +++ b/Python/generated_tracer_cases.c.h @@ -11518,6 +11518,7 @@ cond = stack_pointer[-1]; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsFalse(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -11567,6 +11568,7 @@ cond = b; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsTrue(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } stack_pointer += -1; @@ -11617,6 +11619,7 @@ cond = b; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsFalse(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); } stack_pointer += -1; @@ -11647,6 +11650,7 @@ cond = stack_pointer[-1]; assert(PyStackRef_BoolCheck(cond)); int flag = PyStackRef_IsTrue(cond); + RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); diff --git a/Python/optimizer.c b/Python/optimizer.c index a11244d4b54f74..c46c23aa128825 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -710,11 +710,19 @@ _PyJIT_translate_single_bytecode_to_trace( case POP_JUMP_IF_FALSE: case POP_JUMP_IF_TRUE: { + int counter = target_instr[1].cache; + int bitcount = _Py_popcount32(counter); + int jump_likely = bitcount > 8; _Py_CODEUNIT *computed_next_instr_without_modifiers = target_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; _Py_CODEUNIT *computed_next_instr = computed_next_instr_without_modifiers + (computed_next_instr_without_modifiers->op.code == NOT_TAKEN); _Py_CODEUNIT *computed_jump_instr = computed_next_instr_without_modifiers + oparg; assert(next_instr == computed_next_instr || next_instr == computed_jump_instr); int jump_happened = computed_jump_instr == next_instr; + // Jump is likely but it did not happen this time. Indicates we are likely tracing + // an uncommmon path. Stop the trace early here to prevent trace explosion. + if (jump_likely != jump_happened) { + goto unsupported; + } uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_happened]; ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(jump_happened ? computed_next_instr : computed_jump_instr, old_code)); break; @@ -1226,7 +1234,7 @@ uop_optimize( int curr_stackentries = tstate->interp->jit_state.jit_tracer_initial_stack_depth; int length = interp->jit_state.jit_tracer_code_curr_size; // Trace too short, don't bother. - if (length <= 5) { + if (length <= 20) { return 0; } assert(length > 0); From d820e22378966553002e718930c77c1edfa50997 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 20 Oct 2025 00:43:35 +0100 Subject: [PATCH 069/190] Don't JIT short traces except if they end in a loop --- Python/optimizer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/optimizer.c b/Python/optimizer.c index c46c23aa128825..53f3c6e8bfc887 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1234,7 +1234,7 @@ uop_optimize( int curr_stackentries = tstate->interp->jit_state.jit_tracer_initial_stack_depth; int length = interp->jit_state.jit_tracer_code_curr_size; // Trace too short, don't bother. - if (length <= 20) { + if (length <= 20 && buffer[length-1].opcode != _JUMP_TO_TOP) { return 0; } assert(length > 0); From 00c81fae5a13e5af7c76d920dae2c4404148e588 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 20 Oct 2025 02:31:30 +0100 Subject: [PATCH 070/190] revert last 2 changes --- Python/optimizer.c | 10 +--------- Python/optimizer_analysis.c | 6 +----- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/Python/optimizer.c b/Python/optimizer.c index 53f3c6e8bfc887..a11244d4b54f74 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -710,19 +710,11 @@ _PyJIT_translate_single_bytecode_to_trace( case POP_JUMP_IF_FALSE: case POP_JUMP_IF_TRUE: { - int counter = target_instr[1].cache; - int bitcount = _Py_popcount32(counter); - int jump_likely = bitcount > 8; _Py_CODEUNIT *computed_next_instr_without_modifiers = target_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; _Py_CODEUNIT *computed_next_instr = computed_next_instr_without_modifiers + (computed_next_instr_without_modifiers->op.code == NOT_TAKEN); _Py_CODEUNIT *computed_jump_instr = computed_next_instr_without_modifiers + oparg; assert(next_instr == computed_next_instr || next_instr == computed_jump_instr); int jump_happened = computed_jump_instr == next_instr; - // Jump is likely but it did not happen this time. Indicates we are likely tracing - // an uncommmon path. Stop the trace early here to prevent trace explosion. - if (jump_likely != jump_happened) { - goto unsupported; - } uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_happened]; ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(jump_happened ? computed_next_instr : computed_jump_instr, old_code)); break; @@ -1234,7 +1226,7 @@ uop_optimize( int curr_stackentries = tstate->interp->jit_state.jit_tracer_initial_stack_depth; int length = interp->jit_state.jit_tracer_code_curr_size; // Trace too short, don't bother. - if (length <= 20 && buffer[length-1].opcode != _JUMP_TO_TOP) { + if (length <= 5) { return 0; } assert(length > 0); diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 7cee20bd8ee2a0..9f2a12efee6799 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -528,14 +528,10 @@ _Py_uop_analyze_and_optimize( { OPT_STAT_INC(optimizer_attempts); - int err = optimize_uops( + optimize_uops( initial_func, buffer, length, curr_stacklen, dependencies); - if (err == 0) { - return err; - } - assert(length > 0); length = remove_unneeded_uops(buffer, length); From ba64a5ba4a640753c286296ed41d92642a8b3405 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 20 Oct 2025 02:37:40 +0100 Subject: [PATCH 071/190] Support BINARY_OP_INPLACE_ADD_UNICODE --- Python/optimizer.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Python/optimizer.c b/Python/optimizer.c index a11244d4b54f74..24e3d0614a2a76 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -615,8 +615,6 @@ _PyJIT_translate_single_bytecode_to_trace( // If we haven't guarded the IP, then it's untraceable. (frame != tstate->interp->jit_state.jit_tracer_current_frame && !needs_guard_ip) || (oparg > 0xFFFF) || - // TODO handle BINARY_OP_INPLACE_ADD_UNICODE - opcode == BINARY_OP_INPLACE_ADD_UNICODE || // TODO (gh-140277): The constituent uops are invalid. opcode == BINARY_OP_SUBSCR_GETITEM || // Exception stuff, could be handled in the future maybe? @@ -838,6 +836,12 @@ _PyJIT_translate_single_bytecode_to_trace( operand = 0; } } + if (uop == _BINARY_OP_INPLACE_ADD_UNICODE) { + assert(i + 1 == nuops); + _Py_CODEUNIT *next_instr = target_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; + assert(next_instr->op.code == STORE_FAST); + operand = next_instr->op.arg; + } // All other instructions ADD_TO_TRACE(uop, oparg, operand, target); } From b00252ec02aee6c5be02d04d2394188272359e77 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 20 Oct 2025 02:52:00 +0100 Subject: [PATCH 072/190] Trace through BINARY_OP_SUBSCR_GETITEM --- Include/internal/pycore_optimizer.h | 1 + Python/bytecodes.c | 1 + Python/ceval.c | 6 +- Python/ceval_macros.h | 2 +- Python/generated_cases.c.h | 1 + Python/generated_tracer_cases.c.h | 451 +++++++++++++++++++++++ Python/optimizer.c | 5 +- Tools/cases_generator/tier1_generator.py | 2 + 8 files changed, 463 insertions(+), 6 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index ca9f9702df4cdf..d9a47cf056f96b 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -367,6 +367,7 @@ _PyJIT_translate_single_bytecode_to_trace( _Py_CODEUNIT *next_instr, PyCodeObject *code, PyFunctionObject *func, + int old_stack_level, int opcode, int oparg, int jump_taken); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 42e323e3a4181a..9ef58978d88f17 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2985,6 +2985,7 @@ dummy_func( int _jump_taken = false; PyCodeObject *old_code = _PyFrame_GetCode(frame); PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + int _old_stack_level = 0; TRACING_DISPATCH(); } else { diff --git a/Python/ceval.c b/Python/ceval.c index b16fcdb51cd9b4..08add3f75cd529 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -986,12 +986,12 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch) #if _Py_TIER2 // 1 for trace full, 0 for successful write. -static int -add_to_code_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, PyCodeObject *old_code, PyFunctionObject *old_func, _Py_CODEUNIT *this_instr, _Py_CODEUNIT *next_instr, int opcode, int oparg, int jump_taken) +static inline int +add_to_code_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, PyCodeObject *old_code, PyFunctionObject *old_func, int old_stack_level, _Py_CODEUNIT *this_instr, _Py_CODEUNIT *next_instr, int opcode, int oparg, int jump_taken) { assert(frame != NULL); assert(tstate->interp->jit_state.jit_tracer_code_curr_size < UOP_MAX_TRACE_LENGTH); - return !_PyJIT_translate_single_bytecode_to_trace(tstate, frame, this_instr, next_instr, old_code, old_func, opcode, oparg, jump_taken); + return !_PyJIT_translate_single_bytecode_to_trace(tstate, frame, this_instr, next_instr, old_code, old_func, old_stack_level, opcode, oparg, jump_taken); } #endif diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 3adbadab4c5344..18ccc3dc7bf88c 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -164,7 +164,7 @@ if (_is_sys_tracing) { \ LEAVE_TRACING(); \ } \ - else if ((IS_JIT_TRACING() && add_to_code_trace(tstate, frame, old_code, old_func, this_instr, next_instr, opcode, oparg, _jump_taken))) { \ + else if ((IS_JIT_TRACING() && add_to_code_trace(tstate, frame, old_code, old_func, _old_stack_level, this_instr, next_instr, opcode, oparg, _jump_taken))) { \ BAIL_TRACING_NO_DISPATCH(); \ } \ } while (0); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index bd043ae7285d95..b9ba4ca57cbef5 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -7677,6 +7677,7 @@ int _jump_taken = false; PyCodeObject *old_code = _PyFrame_GetCode(frame); PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + int _old_stack_level = 0; TRACING_DISPATCH(); } else { diff --git a/Python/generated_tracer_cases.c.h b/Python/generated_tracer_cases.c.h index 75e86d9c76d3cd..ff7b91cc239e7e 100644 --- a/Python/generated_tracer_cases.c.h +++ b/Python/generated_tracer_cases.c.h @@ -26,6 +26,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef lhs; _PyStackRef rhs; _PyStackRef res; @@ -96,6 +98,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef left; @@ -162,6 +166,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef left; @@ -230,6 +236,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; @@ -298,6 +306,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef left; _PyStackRef right; @@ -370,6 +380,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; @@ -461,6 +473,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef left; @@ -527,6 +541,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef left; @@ -595,6 +611,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef nos; _PyStackRef dict_st; @@ -669,6 +687,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef container; _PyStackRef getitem; @@ -761,6 +781,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; @@ -859,6 +881,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef tos; _PyStackRef nos; @@ -939,6 +963,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; @@ -1025,6 +1051,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; @@ -1106,6 +1134,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef left; @@ -1172,6 +1202,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); _PyStackRef value; _PyStackRef left; @@ -1240,6 +1272,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef container; _PyStackRef start; _PyStackRef stop; @@ -1306,6 +1340,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef value; _PyStackRef str; _PyStackRef *format; @@ -1374,6 +1410,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef *values; _PyStackRef list; values = &stack_pointer[-oparg]; @@ -1408,6 +1446,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef *values; _PyStackRef map; values = &stack_pointer[-oparg*2]; @@ -1470,6 +1510,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef *values; _PyStackRef set; values = &stack_pointer[-oparg]; @@ -1537,6 +1579,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef *args; _PyStackRef slice; args = &stack_pointer[-oparg]; @@ -1582,6 +1626,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef *pieces; _PyStackRef str; pieces = &stack_pointer[-oparg]; @@ -1639,6 +1685,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef strings; _PyStackRef interpolations; _PyStackRef template; @@ -1687,6 +1735,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef *values; _PyStackRef tup; values = &stack_pointer[-oparg]; @@ -1719,6 +1769,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); assert(0 && "Executing a cache."); Py_FatalError("Executing a cache."); TRACING_DISPATCH(); @@ -1743,6 +1795,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); opcode = CALL; _PyStackRef callable; _PyStackRef self_or_null; @@ -1928,6 +1982,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -2057,6 +2113,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef null; @@ -2208,6 +2266,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef null; @@ -2344,6 +2404,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -2457,6 +2519,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -2574,6 +2638,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -2691,6 +2757,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -2783,6 +2851,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); opcode = CALL_FUNCTION_EX; _PyStackRef func; _PyStackRef callargs; @@ -2958,6 +3028,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; @@ -2998,6 +3070,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef value2_st; _PyStackRef value1_st; _PyStackRef res; @@ -3047,6 +3121,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef null; _PyStackRef callable; @@ -3132,6 +3208,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); opcode = CALL_KW; _PyStackRef callable; _PyStackRef self_or_null; @@ -3321,6 +3399,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef null; @@ -3459,6 +3539,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); opcode = CALL_KW_NON_PY; static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); _PyStackRef callable; @@ -3594,6 +3676,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -3712,6 +3796,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef null; _PyStackRef callable; @@ -3792,6 +3878,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef nos; @@ -3888,6 +3976,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -4017,6 +4107,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -4148,6 +4240,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -4249,6 +4343,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -4363,6 +4459,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); opcode = CALL_NON_PY_GENERAL; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; @@ -4486,6 +4584,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -4607,6 +4707,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -4715,6 +4817,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef null; _PyStackRef callable; @@ -4795,6 +4899,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef null; _PyStackRef callable; @@ -4875,6 +4981,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef null; _PyStackRef callable; @@ -4938,6 +5046,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef exc_value_st; _PyStackRef match_type_st; _PyStackRef rest; @@ -5019,6 +5129,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef left; _PyStackRef right; _PyStackRef b; @@ -5066,6 +5178,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef sub_iter; _PyStackRef last_sent_val; _PyStackRef exc_value_st; @@ -5134,6 +5248,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef left; _PyStackRef right; _PyStackRef res; @@ -5214,6 +5330,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); _PyStackRef value; _PyStackRef left; @@ -5277,6 +5395,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); _PyStackRef value; _PyStackRef left; @@ -5344,6 +5464,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; @@ -5412,6 +5534,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef right; _PyStackRef left; _PyStackRef b; @@ -5479,6 +5603,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); _PyStackRef tos; _PyStackRef left; @@ -5545,6 +5671,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); _PyStackRef tos; _PyStackRef left; @@ -5611,6 +5739,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef value; _PyStackRef result; value = stack_pointer[-1]; @@ -5653,6 +5783,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef bottom; _PyStackRef top; bottom = stack_pointer[-1 - (oparg-1)]; @@ -5681,6 +5813,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); PyCodeObject *co = _PyFrame_GetCode(frame); assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -5712,6 +5846,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef owner; owner = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -5747,6 +5883,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); if (oldobj == NULL) { @@ -5779,6 +5917,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef v = GETLOCAL(oparg); if (PyStackRef_IsNull(v)) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -5815,6 +5955,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); int err = PyDict_Pop(GLOBALS(), name, NULL); @@ -5850,6 +5992,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); PyObject *ns = LOCALS(); int err; @@ -5892,6 +6036,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef container; _PyStackRef sub; sub = stack_pointer[-1]; @@ -5934,6 +6080,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef callable; _PyStackRef dict; _PyStackRef update; @@ -5983,6 +6131,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef dict; _PyStackRef update; update = stack_pointer[-1]; @@ -6036,6 +6186,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef awaitable_st; _PyStackRef exc_st; exc_st = stack_pointer[-1]; @@ -6088,6 +6240,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef value; value = stack_pointer[-1]; stack_pointer += -1; @@ -6116,6 +6270,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef receiver; _PyStackRef value; _PyStackRef val; @@ -6149,6 +6305,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); opcode = ENTER_EXECUTOR; #ifdef _Py_TIER2 PyCodeObject *code = _PyFrame_GetCode(frame); @@ -6196,6 +6354,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef should_be_none; should_be_none = stack_pointer[-1]; if (!PyStackRef_IsNone(should_be_none)) { @@ -6229,6 +6389,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); opcode = EXTENDED_ARG; assert(oparg); opcode = next_instr->op.code; @@ -6255,6 +6417,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; @@ -6301,6 +6465,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef value; _PyStackRef fmt_spec; _PyStackRef res; @@ -6348,6 +6514,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef iter; _PyStackRef null_or_index; _PyStackRef next; @@ -6410,6 +6578,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; _PyStackRef gen_frame; @@ -6489,6 +6659,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; _PyStackRef null_or_index; @@ -6582,6 +6754,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; _PyStackRef next; @@ -6658,6 +6832,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); _PyStackRef iter; _PyStackRef null_or_index; @@ -6723,6 +6899,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef obj; _PyStackRef iter; obj = stack_pointer[-1]; @@ -6794,6 +6972,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef aiter; _PyStackRef awaitable; aiter = stack_pointer[-1]; @@ -6828,6 +7008,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef iterable; _PyStackRef iter; iterable = stack_pointer[-1]; @@ -6867,6 +7049,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef iterable; _PyStackRef iter; _PyStackRef index_or_null; @@ -6923,6 +7107,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef obj; _PyStackRef len; obj = stack_pointer[-1]; @@ -6961,6 +7147,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef iterable; _PyStackRef iter; iterable = stack_pointer[-1]; @@ -7016,6 +7204,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef from; _PyStackRef res; from = stack_pointer[-1]; @@ -7051,6 +7241,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef level; _PyStackRef fromlist; _PyStackRef res; @@ -7100,6 +7292,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); opcode = INSTRUMENTED_CALL; _PyStackRef callable; _PyStackRef self_or_null; @@ -7296,6 +7490,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); opcode = INSTRUMENTED_CALL_FUNCTION_EX; _PyStackRef func; _PyStackRef callargs; @@ -7471,6 +7667,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); opcode = INSTRUMENTED_CALL_KW; _PyStackRef callable; _PyStackRef self_or_null; @@ -7665,6 +7863,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef awaitable_st; _PyStackRef exc_st; // _MONITOR_END_ASYNC_FOR @@ -7725,6 +7925,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef receiver; _PyStackRef value; value = stack_pointer[-1]; @@ -7763,6 +7965,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef receiver; _PyStackRef value; _PyStackRef val; @@ -7805,6 +8009,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef iter; _PyStackRef null_or_index; _PyStackRef next; @@ -7849,6 +8055,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); opcode = INSTRUMENTED_INSTRUCTION; _PyFrame_SetStackPointer(frame, stack_pointer); int next_opcode = _Py_call_instrumentation_instruction( @@ -7884,6 +8092,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); /* Skip 1 cache entry */ // _CHECK_PERIODIC { @@ -7919,6 +8129,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); TRACING_DISPATCH(); } @@ -7942,6 +8154,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); opcode = INSTRUMENTED_LINE; int original_opcode = 0; if (tstate->tracing) { @@ -7989,6 +8203,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); opcode = INSTRUMENTED_LOAD_SUPER_ATTR; _PyStackRef global_super_st; _PyStackRef class_st; @@ -8118,6 +8334,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); (void)this_instr; INSTRUMENTED_JUMP(prev_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); TRACING_DISPATCH(); @@ -8142,6 +8360,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef iter; _PyStackRef index_or_null; index_or_null = stack_pointer[-1]; @@ -8174,6 +8394,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef cond; /* Skip 1 cache entry */ cond = stack_pointer[-1]; @@ -8206,6 +8428,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef value; /* Skip 1 cache entry */ value = stack_pointer[-1]; @@ -8245,6 +8469,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef value; /* Skip 1 cache entry */ value = stack_pointer[-1]; @@ -8282,6 +8508,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef cond; /* Skip 1 cache entry */ cond = stack_pointer[-1]; @@ -8314,6 +8542,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); // _LOAD_BYTECODE { #ifdef Py_GIL_DISABLED @@ -8402,6 +8632,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef val; _PyStackRef retval; _PyStackRef res; @@ -8464,6 +8696,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef val; _PyStackRef retval; _PyStackRef value; @@ -8545,6 +8779,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef retval; retval = stack_pointer[-1]; assert(frame->owner == FRAME_OWNED_BY_INTERPRETER); @@ -8590,6 +8826,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef left; _PyStackRef right; _PyStackRef b; @@ -8634,6 +8872,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); /* Skip 1 cache entry */ // _SPECIALIZE_JUMP_BACKWARD { @@ -8681,6 +8921,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(1 == 1, "incorrect cache size"); /* Skip 1 cache entry */ // _CHECK_PERIODIC @@ -8720,6 +8962,7 @@ int _jump_taken = false; PyCodeObject *old_code = _PyFrame_GetCode(frame); PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + int _old_stack_level = 0; TRACING_DISPATCH(); } else { @@ -8748,6 +8991,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); assert(oparg <= INSTR_OFFSET()); JUMPBY(-oparg); TRACING_DISPATCH(); @@ -8771,6 +9016,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(1 == 1, "incorrect cache size"); /* Skip 1 cache entry */ // _CHECK_PERIODIC @@ -8808,6 +9055,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); JUMPBY(oparg); TRACING_DISPATCH(); } @@ -8830,6 +9079,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef list; _PyStackRef v; v = stack_pointer[-1]; @@ -8862,6 +9113,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef list_st; _PyStackRef iterable_st; iterable_st = stack_pointer[-1]; @@ -8920,6 +9173,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef owner; _PyStackRef *attr; _PyStackRef *self_or_null; @@ -9008,6 +9263,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -9074,6 +9331,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -9150,6 +9409,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; /* Skip 1 cache entry */ @@ -9216,6 +9477,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -9304,6 +9567,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -9368,6 +9633,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -9422,6 +9689,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -9497,6 +9766,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -9579,6 +9850,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -9634,6 +9907,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -9710,6 +9985,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef new_frame; @@ -9812,6 +10089,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -9889,6 +10168,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); _PyStackRef owner; _PyStackRef attr; @@ -10007,6 +10288,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef bc; PyObject *bc_o; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -10047,6 +10330,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef value; assert(oparg < NUM_COMMON_CONSTANTS); value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); @@ -10074,6 +10359,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef value; PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); value = PyStackRef_FromPyObjectBorrow(obj); @@ -10101,6 +10388,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef value; PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -10139,6 +10428,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef value; assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_DUP(GETLOCAL(oparg)); @@ -10166,6 +10457,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef value; value = GETLOCAL(oparg); GETLOCAL(oparg) = PyStackRef_NULL; @@ -10193,6 +10486,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef value; assert(!PyStackRef_IsNull(GETLOCAL(oparg))); value = PyStackRef_Borrow(GETLOCAL(oparg)); @@ -10220,6 +10515,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef value1; _PyStackRef value2; uint32_t oparg1 = oparg >> 4; @@ -10251,6 +10548,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef value; _PyStackRef value_s = GETLOCAL(oparg); if (PyStackRef_IsNull(value_s)) { @@ -10287,6 +10586,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef value1; _PyStackRef value2; uint32_t oparg1 = oparg >> 4; @@ -10318,6 +10619,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef class_dict_st; _PyStackRef value; class_dict_st = stack_pointer[-1]; @@ -10373,6 +10676,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef mod_or_class_dict; _PyStackRef v; mod_or_class_dict = stack_pointer[-1]; @@ -10459,6 +10764,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef *res; _PyStackRef *null; // _SPECIALIZE_LOAD_GLOBAL @@ -10522,6 +10829,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); _PyStackRef res; _PyStackRef *null; @@ -10610,6 +10919,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); _PyStackRef res; _PyStackRef *null; @@ -10685,6 +10996,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef locals; PyObject *l = LOCALS(); if (l == NULL) { @@ -10719,6 +11032,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef v; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -10752,6 +11067,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef value; assert(oparg < _PY_NSMALLPOSINTS); PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; @@ -10780,6 +11097,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef self; _PyStackRef *method_and_self; // _INSERT_NULL @@ -10837,6 +11156,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); opcode = LOAD_SUPER_ATTR; _PyStackRef global_super_st; _PyStackRef class_st; @@ -10981,6 +11302,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); _PyStackRef global_super_st; _PyStackRef class_st; @@ -11051,6 +11374,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); _PyStackRef global_super_st; _PyStackRef class_st; @@ -11137,6 +11462,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); PyObject *cell = PyCell_New(initial); if (cell == NULL) { @@ -11168,6 +11495,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef codeobj_st; _PyStackRef func; codeobj_st = stack_pointer[-1]; @@ -11211,6 +11540,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef dict_st; _PyStackRef key; _PyStackRef value; @@ -11252,6 +11583,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef subject; _PyStackRef type; _PyStackRef names; @@ -11314,6 +11647,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef subject; _PyStackRef keys; _PyStackRef values_or_none; @@ -11351,6 +11686,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef subject; _PyStackRef res; subject = stack_pointer[-1]; @@ -11380,6 +11717,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef subject; _PyStackRef res; subject = stack_pointer[-1]; @@ -11409,6 +11748,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); TRACING_DISPATCH(); } @@ -11430,6 +11771,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); TRACING_DISPATCH(); } @@ -11451,6 +11794,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef exc_value; exc_value = stack_pointer[-1]; _PyErr_StackItem *exc_info = tstate->exc_info; @@ -11482,6 +11827,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef iter; _PyStackRef index_or_null; index_or_null = stack_pointer[-1]; @@ -11513,6 +11860,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef cond; /* Skip 1 cache entry */ cond = stack_pointer[-1]; @@ -11543,6 +11892,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef value; _PyStackRef b; _PyStackRef cond; @@ -11594,6 +11945,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef value; _PyStackRef b; _PyStackRef cond; @@ -11645,6 +11998,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef cond; /* Skip 1 cache entry */ cond = stack_pointer[-1]; @@ -11675,6 +12030,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef value; value = stack_pointer[-1]; stack_pointer += -1; @@ -11703,6 +12060,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef exc; _PyStackRef prev_exc; _PyStackRef new_exc; @@ -11742,6 +12101,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef res; res = PyStackRef_NULL; stack_pointer[0] = res; @@ -11768,6 +12129,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef *args; args = &stack_pointer[-oparg]; assert(oparg < 3); @@ -11805,6 +12168,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef *values; _PyStackRef exc_st; exc_st = stack_pointer[-1]; @@ -11841,6 +12206,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); assert(0 && "Executing RESERVED instruction."); Py_FatalError("Executing RESERVED instruction."); TRACING_DISPATCH(); @@ -11865,6 +12232,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); // _LOAD_BYTECODE { #ifdef Py_GIL_DISABLED @@ -11948,6 +12317,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(0 == 0, "incorrect cache size"); #if defined(__EMSCRIPTEN__) if (_Py_emscripten_signal_clock == 0) { @@ -11994,6 +12365,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef res; assert(PyStackRef_FunctionCheck(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); @@ -12048,6 +12421,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef retval; _PyStackRef res; retval = stack_pointer[-1]; @@ -12095,6 +12470,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef receiver; _PyStackRef v; _PyStackRef retval; @@ -12208,6 +12585,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); _PyStackRef receiver; _PyStackRef v; @@ -12285,6 +12664,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); PyObject *ann_dict; if (LOCALS() == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -12341,6 +12722,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef set; _PyStackRef v; v = stack_pointer[-1]; @@ -12375,6 +12758,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef attr_st; _PyStackRef func_in; _PyStackRef func_out; @@ -12413,6 +12798,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef set; _PyStackRef iterable; iterable = stack_pointer[-1]; @@ -12451,6 +12838,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef owner; _PyStackRef v; // _SPECIALIZE_STORE_ATTR @@ -12515,6 +12904,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); _PyStackRef owner; _PyStackRef value; @@ -12599,6 +12990,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); _PyStackRef owner; _PyStackRef value; @@ -12658,6 +13051,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); _PyStackRef owner; _PyStackRef value; @@ -12754,6 +13149,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef v; v = stack_pointer[-1]; PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); @@ -12783,6 +13180,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef value; value = stack_pointer[-1]; _PyStackRef tmp = GETLOCAL(oparg); @@ -12813,6 +13212,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef value1; _PyStackRef value2; value1 = stack_pointer[-1]; @@ -12846,6 +13247,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef value2; _PyStackRef value1; value1 = stack_pointer[-1]; @@ -12887,6 +13290,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef v; v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -12922,6 +13327,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef v; v = stack_pointer[-1]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); @@ -12978,6 +13385,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef v; _PyStackRef container; _PyStackRef start; @@ -13049,6 +13458,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef container; _PyStackRef sub; _PyStackRef v; @@ -13115,6 +13526,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); _PyStackRef nos; _PyStackRef value; @@ -13174,6 +13587,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); _PyStackRef value; _PyStackRef nos; @@ -13263,6 +13678,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef bottom; _PyStackRef top; top = stack_pointer[-1]; @@ -13294,6 +13711,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef value; _PyStackRef res; // _SPECIALIZE_TO_BOOL @@ -13353,6 +13772,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef owner; _PyStackRef value; @@ -13404,6 +13825,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef value; /* Skip 1 cache entry */ @@ -13436,6 +13859,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef value; _PyStackRef res; @@ -13484,6 +13909,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef tos; _PyStackRef value; @@ -13535,6 +13962,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef value; _PyStackRef res; @@ -13570,6 +13999,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); _PyStackRef value; _PyStackRef res; @@ -13626,6 +14057,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; @@ -13665,6 +14098,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; @@ -13704,6 +14139,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef value; _PyStackRef res; value = stack_pointer[-1]; @@ -13732,6 +14169,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef seq; _PyStackRef *top; seq = stack_pointer[-1]; @@ -13770,6 +14209,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef seq; _PyStackRef *top; // _SPECIALIZE_UNPACK_SEQUENCE @@ -13828,6 +14269,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); _PyStackRef tos; _PyStackRef seq; @@ -13895,6 +14338,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); _PyStackRef tos; _PyStackRef seq; @@ -13953,6 +14398,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); _PyStackRef tos; _PyStackRef seq; @@ -14012,6 +14459,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef exit_func; _PyStackRef exit_self; _PyStackRef lasti; @@ -14067,6 +14516,8 @@ (void)old_func; int _jump_taken = false; (void)_jump_taken; + int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; + (void)(_old_stack_level); _PyStackRef retval; _PyStackRef value; retval = stack_pointer[-1]; diff --git a/Python/optimizer.c b/Python/optimizer.c index 24e3d0614a2a76..f20b669cfb859d 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -564,6 +564,7 @@ _PyJIT_translate_single_bytecode_to_trace( _Py_CODEUNIT *next_instr, PyCodeObject *old_code, PyFunctionObject *func, + int old_stack_level, int opcode, int oparg, int jump_taken) @@ -615,8 +616,8 @@ _PyJIT_translate_single_bytecode_to_trace( // If we haven't guarded the IP, then it's untraceable. (frame != tstate->interp->jit_state.jit_tracer_current_frame && !needs_guard_ip) || (oparg > 0xFFFF) || - // TODO (gh-140277): The constituent uops are invalid. - opcode == BINARY_OP_SUBSCR_GETITEM || + // TODO (gh-140277): The constituent use one extra stack slot. So we need to check for heaedroom. + (opcode == BINARY_OP_SUBSCR_GETITEM && old_stack_level + 1 > old_code->co_stacksize)|| // Exception stuff, could be handled in the future maybe? opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO || frame->owner >= FRAME_OWNED_BY_INTERPRETER diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 939035332cba72..c77074f656c9e0 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -260,6 +260,8 @@ def generate_tier1_cases( out.emit(f"(void)old_func;\n") out.emit(f"int _jump_taken = false;\n") out.emit(f"(void)_jump_taken;\n") + out.emit(f"int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0;\n") + out.emit(f"(void)(_old_stack_level);\n") if inst.properties.uses_opcode: out.emit(f"opcode = {name};\n") if inst.family is not None: From 754b3b78fe9275229f4ee653eb261af3cf69ca7f Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 20 Oct 2025 03:51:41 +0100 Subject: [PATCH 073/190] Close loops --- Include/internal/pycore_interp_structs.h | 3 ++- Include/internal/pycore_optimizer.h | 2 +- Python/bytecodes.c | 4 ++-- Python/executor_cases.c.h | 2 +- Python/generated_cases.c.h | 2 +- Python/generated_tracer_cases.c.h | 2 +- Python/optimizer.c | 15 ++++++++------- Python/pystate.c | 1 - 8 files changed, 16 insertions(+), 15 deletions(-) diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 44f8a23fd47278..fdbba611c2dec2 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -763,7 +763,8 @@ typedef struct _PyJitState { _PyBloomFilter jit_tracer_dependencies; bool jit_tracer_dependencies_still_valid; _PyUOpInstruction *jit_tracer_code_buffer; - _Py_CODEUNIT *jit_tracer_initial_instr; + _Py_CODEUNIT *jit_tracer_insert_exec_instr; + _Py_CODEUNIT *jit_tracer_close_loop_instr; int jit_tracer_initial_stack_depth; int jit_tracer_initial_chain_depth; PyCodeObject *jit_tracer_initial_code; // Strong diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index d9a47cf056f96b..b06593377324e3 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -373,7 +373,7 @@ _PyJIT_translate_single_bytecode_to_trace( int jump_taken); void -_PyJIT_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit); +_PyJIT_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *insert_exec_instr, _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit); void _PyJIT_FinalizeTracing(PyThreadState *tstate); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 9ef58978d88f17..0ba8286fc467d7 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2979,7 +2979,7 @@ dummy_func( } int _is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL); if (!_is_sys_tracing) { - _PyJIT_InitializeTracing(tstate, frame, this_instr, STACK_LEVEL(), 0, NULL); + _PyJIT_InitializeTracing(tstate, frame, this_instr, next_instr, STACK_LEVEL(), 0, NULL); ENTER_TRACING(); } int _jump_taken = false; @@ -5451,7 +5451,7 @@ dummy_func( _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); assert(tstate->current_executor == (PyObject *)previous_executor); int chain_depth = previous_executor->vm_data.chain_depth + 1; - _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), chain_depth, exit); + _PyJIT_InitializeTracing(tstate, frame, target, target, STACK_LEVEL(), chain_depth, exit); exit->temperature = initial_temperature_backoff_counter(); GOTO_TIER_ONE(target, 1); } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index e32c144b9e9132..03846f48d538da 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7502,7 +7502,7 @@ _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); assert(tstate->current_executor == (PyObject *)previous_executor); int chain_depth = previous_executor->vm_data.chain_depth + 1; - _PyJIT_InitializeTracing(tstate, frame, target, STACK_LEVEL(), chain_depth, exit); + _PyJIT_InitializeTracing(tstate, frame, target, target, STACK_LEVEL(), chain_depth, exit); exit->temperature = initial_temperature_backoff_counter(); GOTO_TIER_ONE(target, 1); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index b9ba4ca57cbef5..28691b2c5c836d 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -7671,7 +7671,7 @@ } int _is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL); if (!_is_sys_tracing) { - _PyJIT_InitializeTracing(tstate, frame, this_instr, STACK_LEVEL(), 0, NULL); + _PyJIT_InitializeTracing(tstate, frame, this_instr, next_instr, STACK_LEVEL(), 0, NULL); ENTER_TRACING(); } int _jump_taken = false; diff --git a/Python/generated_tracer_cases.c.h b/Python/generated_tracer_cases.c.h index ff7b91cc239e7e..cdd7246fbb4a06 100644 --- a/Python/generated_tracer_cases.c.h +++ b/Python/generated_tracer_cases.c.h @@ -8956,7 +8956,7 @@ } int _is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL); if (!_is_sys_tracing) { - _PyJIT_InitializeTracing(tstate, frame, this_instr, STACK_LEVEL(), 0, NULL); + _PyJIT_InitializeTracing(tstate, frame, this_instr, next_instr, STACK_LEVEL(), 0, NULL); ENTER_TRACING(); } int _jump_taken = false; diff --git a/Python/optimizer.c b/Python/optimizer.c index f20b669cfb859d..2fcf6b763691ac 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -131,7 +131,7 @@ _PyOptimizer_Optimize( chain_depth %= MAX_CHAIN_DEPTH; bool progress_needed = chain_depth == 0; PyCodeObject *code = (PyCodeObject *)tstate->interp->jit_state.jit_tracer_initial_code; - _Py_CODEUNIT *start = tstate->interp->jit_state.jit_tracer_initial_instr; + _Py_CODEUNIT *start = tstate->interp->jit_state.jit_tracer_insert_exec_instr; // A recursive trace might've cleared the values. In that case, bail. if (code == NULL) { interp->compiling = false; @@ -570,7 +570,7 @@ _PyJIT_translate_single_bytecode_to_trace( int jump_taken) { - int is_first_instr = tstate->interp->jit_state.jit_tracer_initial_instr == this_instr; + int is_first_instr = tstate->interp->jit_state.jit_tracer_close_loop_instr == this_instr; bool progress_needed = (tstate->interp->jit_state.jit_tracer_initial_chain_depth % MAX_CHAIN_DEPTH) == 0;; _PyBloomFilter *dependencies = &tstate->interp->jit_state.jit_tracer_dependencies; _Py_BloomFilter_Add(dependencies, old_code); @@ -686,7 +686,7 @@ _PyJIT_translate_single_bytecode_to_trace( } // Loop back to the start - if (is_first_instr && tstate->interp->jit_state.jit_tracer_code_curr_size > 2) { + if (is_first_instr && tstate->interp->jit_state.jit_tracer_code_curr_size > 5) { ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0, 0); goto done; } @@ -876,7 +876,7 @@ _PyJIT_translate_single_bytecode_to_trace( } void -_PyJIT_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit) +_PyJIT_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *insert_exec_instr, _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit) { PyCodeObject *code = _PyFrame_GetCode(frame); #ifdef Py_DEBUG @@ -890,14 +890,15 @@ _PyJIT_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_ PyUnicode_AsUTF8(code->co_qualname), PyUnicode_AsUTF8(code->co_filename), code->co_firstlineno, - 2 * INSTR_IP(next_instr, code), + 2 * INSTR_IP(close_loop_instr, code), chain_depth); #endif - add_to_trace(tstate->interp->jit_state.jit_tracer_code_buffer, 0, _START_EXECUTOR, 0, (uintptr_t)next_instr, INSTR_IP(next_instr, code)); + add_to_trace(tstate->interp->jit_state.jit_tracer_code_buffer, 0, _START_EXECUTOR, 0, (uintptr_t)insert_exec_instr, INSTR_IP(insert_exec_instr, code)); add_to_trace(tstate->interp->jit_state.jit_tracer_code_buffer, 1, _MAKE_WARM, 0, 0, 0); tstate->interp->jit_state.jit_tracer_code_curr_size = 2; tstate->interp->jit_state.jit_tracer_code_max_size = UOP_MAX_TRACE_LENGTH; - tstate->interp->jit_state.jit_tracer_initial_instr = next_instr; + tstate->interp->jit_state.jit_tracer_insert_exec_instr = insert_exec_instr; + tstate->interp->jit_state.jit_tracer_close_loop_instr = close_loop_instr; tstate->interp->jit_state.jit_tracer_initial_code = (PyCodeObject *)Py_NewRef(code); tstate->interp->jit_state.jit_tracer_initial_func = (PyFunctionObject *)Py_NewRef(_PyFrame_GetFunction(frame)); tstate->interp->jit_state.jit_tracer_previous_exit = exit; diff --git a/Python/pystate.c b/Python/pystate.c index 01eab6d3f5ea1b..f0790322abd1ff 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -557,7 +557,6 @@ init_interpreter(PyInterpreterState *interp, #ifdef _Py_TIER2 interp->jit_state.jit_tracer_code_buffer = NULL; - interp->jit_state.jit_tracer_initial_instr = NULL; interp->jit_state.jit_tracer_initial_stack_depth = -1; interp->jit_state.jit_tracer_initial_chain_depth = -1; interp->jit_state.jit_tracer_initial_code = NULL; From 6045a677ac736652e43bacb313f2fa2ab56c7cda Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 20 Oct 2025 17:16:25 +0100 Subject: [PATCH 074/190] Specialize on deopt when tracing --- Include/internal/pycore_interp_structs.h | 1 + Python/ceval_macros.h | 20 ++++++++++++++- Python/generated_tracer_cases.c.h | 30 +++++++++++------------ Python/optimizer.c | 3 ++- Tools/cases_generator/tracer_generator.py | 21 ++++++++++++++++ 5 files changed, 58 insertions(+), 17 deletions(-) diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index fdbba611c2dec2..142bcbece6220b 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -765,6 +765,7 @@ typedef struct _PyJitState { _PyUOpInstruction *jit_tracer_code_buffer; _Py_CODEUNIT *jit_tracer_insert_exec_instr; _Py_CODEUNIT *jit_tracer_close_loop_instr; + _Py_CODEUNIT *last_specialized_instr; int jit_tracer_initial_stack_depth; int jit_tracer_initial_chain_depth; PyCodeObject *jit_tracer_initial_code; // Strong diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 18ccc3dc7bf88c..256175b476dc55 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -137,6 +137,9 @@ #if _Py_TAIL_CALL_INTERP || USE_COMPUTED_GOTOS # define IS_JIT_TRACING() (DISPATCH_TABLE_VAR == TRACING_DISPATCH_TABLE) +// tstate->interp->jit_state.last_specialized_instr != this_instr is required to not get stuck in infinite +// specialization loops due to specialization failure. +# define IS_JIT_TRACING_MAKING_PROGRESS() (IS_JIT_TRACING() && tstate->interp->jit_state.last_specialized_instr != this_instr) # define ENTER_TRACING() \ DISPATCH_TABLE_VAR = TRACING_DISPATCH_TABLE; # define LEAVE_TRACING() \ @@ -214,6 +217,14 @@ do { \ DISPATCH_GOTO(); \ } +#define TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() \ +{ \ + tstate->interp->jit_state.last_specialized_instr = this_instr; \ + opcode = next_instr->op.code; \ + PRE_DISPATCH_GOTO(); \ + DISPATCH_GOTO(); \ +} + #define DISPATCH_INLINED(NEW_FRAME) \ do { \ assert(tstate->interp->eval_frame == NULL); \ @@ -225,11 +236,13 @@ do { \ } while (0) #define TRACING_DISPATCH_INLINED(NEW_FRAME) \ + tstate->interp->jit_state.last_specialized_instr = this_instr; \ RECORD_TRACE_NO_DISPATCH(); \ DISPATCH_INLINED(NEW_FRAME); #define TRACING_DISPATCH() \ { \ + tstate->interp->jit_state.last_specialized_instr = this_instr; \ assert(frame->stackpointer == NULL); \ RECORD_TRACE_NO_DISPATCH(); \ NEXTOPARG(); \ @@ -337,9 +350,14 @@ GETITEM(PyObject *v, Py_ssize_t i) { /* This takes a uint16_t instead of a _Py_BackoffCounter, * because it is used directly on the cache entry in generated code, * which is always an integral type. */ +#if _Py_TIER2 +// Force re-specialization when tracing a side exit to get good side exits. +#define ADAPTIVE_COUNTER_TRIGGERS(COUNTER) \ + backoff_counter_triggers(forge_backoff_counter((COUNTER))) || IS_JIT_TRACING_MAKING_PROGRESS() +#else #define ADAPTIVE_COUNTER_TRIGGERS(COUNTER) \ backoff_counter_triggers(forge_backoff_counter((COUNTER))) - +#endif #define ADVANCE_ADAPTIVE_COUNTER(COUNTER) \ do { \ (COUNTER) = advance_backoff_counter((COUNTER)); \ diff --git a/Python/generated_tracer_cases.c.h b/Python/generated_tracer_cases.c.h index cdd7246fbb4a06..9d739dc00db983 100644 --- a/Python/generated_tracer_cases.c.h +++ b/Python/generated_tracer_cases.c.h @@ -43,7 +43,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() } OPCODE_DEFERRED_INC(BINARY_OP); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); @@ -1814,7 +1814,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_Call(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() } OPCODE_DEFERRED_INC(CALL); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); @@ -3228,7 +3228,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() } OPCODE_DEFERRED_INC(CALL_KW); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); @@ -5265,7 +5265,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_CompareOp(left, right, next_instr, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() } OPCODE_DEFERRED_INC(COMPARE_OP); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); @@ -5550,7 +5550,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_ContainsOp(right, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() } OPCODE_DEFERRED_INC(CONTAINS_OP); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); @@ -6531,7 +6531,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_ForIter(iter, null_or_index, next_instr, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() } OPCODE_DEFERRED_INC(FOR_ITER); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); @@ -8882,7 +8882,7 @@ uint8_t desired = tstate->interp->jit ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, desired); next_instr = this_instr; - DISPATCH_SAME_OPARG(); + DISPATCH_SAME_OPARG() } #endif } @@ -9190,7 +9190,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_LoadAttr(owner, next_instr, name); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() } OPCODE_DEFERRED_INC(LOAD_ATTR); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); @@ -10779,7 +10779,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() } OPCODE_DEFERRED_INC(LOAD_GLOBAL); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); @@ -11177,7 +11177,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() } OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); @@ -12486,7 +12486,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_Send(receiver, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() } OPCODE_DEFERRED_INC(SEND); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); @@ -12854,7 +12854,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_StoreAttr(owner, next_instr, name); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() } OPCODE_DEFERRED_INC(STORE_ATTR); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); @@ -13475,7 +13475,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_StoreSubscr(container, sub, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() } OPCODE_DEFERRED_INC(STORE_SUBSCR); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); @@ -13726,7 +13726,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_ToBool(value, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() } OPCODE_DEFERRED_INC(TO_BOOL); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); @@ -14224,7 +14224,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_UnpackSequence(seq, next_instr, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - DISPATCH_SAME_OPARG(); + TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() } OPCODE_DEFERRED_INC(UNPACK_SEQUENCE); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); diff --git a/Python/optimizer.c b/Python/optimizer.c index 2fcf6b763691ac..ca9967e4272d42 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -904,9 +904,10 @@ _PyJIT_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_ tstate->interp->jit_state.jit_tracer_previous_exit = exit; _Py_BloomFilter_Init(&tstate->interp->jit_state.jit_tracer_dependencies); tstate->interp->jit_state.jit_tracer_initial_stack_depth = curr_stackdepth; - tstate->interp->jit_state.jit_tracer_initial_chain_depth = chain_depth; + tstate->interp->jit_state.jit_tracer_initial_chain_depth = chain_depth % MAX_CHAIN_DEPTH; tstate->interp->jit_state.jit_tracer_current_frame = frame; tstate->interp->jit_state.jit_tracer_dependencies_still_valid = true; + tstate->interp->jit_state.last_specialized_instr = NULL; } void diff --git a/Tools/cases_generator/tracer_generator.py b/Tools/cases_generator/tracer_generator.py index 30a34a91d65b2e..7039079dd895f3 100644 --- a/Tools/cases_generator/tracer_generator.py +++ b/Tools/cases_generator/tracer_generator.py @@ -44,6 +44,7 @@ def __init__(self, out: CWriter, labels: dict[str, Label], cannot_escape: bool = **self._replacers, "DISPATCH": self.dispatch, "DISPATCH_INLINED": self.dispatch_inlined, + "DISPATCH_SAME_OPARG": self.dispatch_same_oparg, } def dispatch( @@ -75,6 +76,26 @@ def dispatch_inlined( self.out.start_line() self.emit("TRACING_DISPATCH_INLINED") return False + + def dispatch_same_oparg( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ) -> bool: + if storage.spilled: + raise analysis_error("stack_pointer needs reloading before dispatch", tkn) + storage.stack.flush(self.out) + self.out.start_line() + if "specializing" in uop.annotations: + self.emit("TRACING_SPECIALIZE_DISPATCH_SAME_OPARG") + else: + self.emit(tkn) + emit_to(self.out, tkn_iter, "SEMI") + return False + def record_jump_taken( self, tkn: Token, From ec2971fb9aea51099cb86455afb8523f97a603ca Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 20 Oct 2025 17:18:43 +0100 Subject: [PATCH 075/190] make mypy happy --- Tools/cases_generator/tracer_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/cases_generator/tracer_generator.py b/Tools/cases_generator/tracer_generator.py index 7039079dd895f3..912056b6212a04 100644 --- a/Tools/cases_generator/tracer_generator.py +++ b/Tools/cases_generator/tracer_generator.py @@ -89,7 +89,7 @@ def dispatch_same_oparg( raise analysis_error("stack_pointer needs reloading before dispatch", tkn) storage.stack.flush(self.out) self.out.start_line() - if "specializing" in uop.annotations: + if isinstance(uop, Uop) and "specializing" in uop.annotations: self.emit("TRACING_SPECIALIZE_DISPATCH_SAME_OPARG") else: self.emit(tkn) From d49e367b37337c91c89b0dabd8eb98c3362da500 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 20 Oct 2025 21:45:20 +0100 Subject: [PATCH 076/190] remedies against trace explosion --- Python/optimizer.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/Python/optimizer.c b/Python/optimizer.c index ca9967e4272d42..f8c6666c0ec79d 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -709,13 +709,29 @@ _PyJIT_translate_single_bytecode_to_trace( case POP_JUMP_IF_FALSE: case POP_JUMP_IF_TRUE: { + int counter = target_instr[1].cache; + int direction = _Py_popcount32(counter); _Py_CODEUNIT *computed_next_instr_without_modifiers = target_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; _Py_CODEUNIT *computed_next_instr = computed_next_instr_without_modifiers + (computed_next_instr_without_modifiers->op.code == NOT_TAKEN); _Py_CODEUNIT *computed_jump_instr = computed_next_instr_without_modifiers + oparg; assert(next_instr == computed_next_instr || next_instr == computed_jump_instr); int jump_happened = computed_jump_instr == next_instr; uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_happened]; - ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(jump_happened ? computed_next_instr : computed_jump_instr, old_code)); + ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(jump_happened ? computed_next_instr : computed_jump_instr, old_code)); + // Branch is biased to jumping, but jump did not happen. + // We are likely in a bad trace. So we should retrace later. + if ((direction > 10 && !jump_happened) || + // Branch is biased to not jumping, but jump did not happen. + // We are likely in a bad trace. So we should retrace later. + (direction < 6 && jump_happened) || + // Finally, branch is just not heavily biased. + // So we should not trace through it anyways + // This prevents trace explosion. + (direction >= 6 && direction <= 10) + ) { + ADD_TO_TRACE(_EXIT_TRACE, 0, 0, INSTR_IP(next_instr, old_code)); + goto full; + } break; } case JUMP_BACKWARD_JIT: From 55892a4ed73795fff20315183fd83d2d44df7e62 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 20 Oct 2025 21:48:00 +0100 Subject: [PATCH 077/190] lint --- Python/optimizer.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Python/optimizer.c b/Python/optimizer.c index f8c6666c0ec79d..f67eccda716985 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -717,19 +717,19 @@ _PyJIT_translate_single_bytecode_to_trace( assert(next_instr == computed_next_instr || next_instr == computed_jump_instr); int jump_happened = computed_jump_instr == next_instr; uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_happened]; - ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(jump_happened ? computed_next_instr : computed_jump_instr, old_code)); + ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(jump_happened ? computed_next_instr : computed_jump_instr, old_code)); // Branch is biased to jumping, but jump did not happen. // We are likely in a bad trace. So we should retrace later. if ((direction > 10 && !jump_happened) || // Branch is biased to not jumping, but jump did not happen. - // We are likely in a bad trace. So we should retrace later. + // We are likely in a bad trace. So we should retrace later (direction < 6 && jump_happened) || // Finally, branch is just not heavily biased. // So we should not trace through it anyways - // This prevents trace explosion. + // This prevents trace explosion (direction >= 6 && direction <= 10) ) { - ADD_TO_TRACE(_EXIT_TRACE, 0, 0, INSTR_IP(next_instr, old_code)); + ADD_TO_TRACE(_EXIT_TRACE, 0, 0, INSTR_IP(next_instr, old_code)); goto full; } break; From dd0e16f7e1eeb32de05d335502cd1a628508f8c2 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 21 Oct 2025 00:52:56 +0100 Subject: [PATCH 078/190] Fix a bug with where the executors get inserted during EXTENDED_ARG --- Include/internal/pycore_uop.h | 2 +- Python/bytecodes.c | 8 +++++++- Python/generated_cases.c.h | 7 ++++++- Python/generated_tracer_cases.c.h | 7 ++++++- 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_uop.h b/Include/internal/pycore_uop.h index 86bea31204ca83..bccc0b00a4d210 100644 --- a/Include/internal/pycore_uop.h +++ b/Include/internal/pycore_uop.h @@ -36,7 +36,7 @@ typedef struct _PyUOpInstruction{ } _PyUOpInstruction; // This is the length of the trace we translate initially. -#define UOP_MAX_TRACE_LENGTH 1400 +#define UOP_MAX_TRACE_LENGTH 1500 #define UOP_BUFFER_SIZE (UOP_MAX_TRACE_LENGTH * sizeof(_PyUOpInstruction)) /* Bloom filter with m = 256 diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 0ba8286fc467d7..7df899dc50e8d9 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2979,7 +2979,13 @@ dummy_func( } int _is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL); if (!_is_sys_tracing) { - _PyJIT_InitializeTracing(tstate, frame, this_instr, next_instr, STACK_LEVEL(), 0, NULL); + /* Back up over EXTENDED_ARGs so executor is inserted at the corret place */ + _Py_CODEUNIT *insert_exec_at = this_instr; + while (oparg > 255) { + oparg >>= 8; + insert_exec_at--; + } + _PyJIT_InitializeTracing(tstate, frame, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL); ENTER_TRACING(); } int _jump_taken = false; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 28691b2c5c836d..5cd7142adb3314 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -7671,7 +7671,12 @@ } int _is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL); if (!_is_sys_tracing) { - _PyJIT_InitializeTracing(tstate, frame, this_instr, next_instr, STACK_LEVEL(), 0, NULL); + _Py_CODEUNIT *insert_exec_at = this_instr; + while (oparg > 255) { + oparg >>= 8; + insert_exec_at--; + } + _PyJIT_InitializeTracing(tstate, frame, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL); ENTER_TRACING(); } int _jump_taken = false; diff --git a/Python/generated_tracer_cases.c.h b/Python/generated_tracer_cases.c.h index 9d739dc00db983..759bfd13cb83dc 100644 --- a/Python/generated_tracer_cases.c.h +++ b/Python/generated_tracer_cases.c.h @@ -8956,7 +8956,12 @@ } int _is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL); if (!_is_sys_tracing) { - _PyJIT_InitializeTracing(tstate, frame, this_instr, next_instr, STACK_LEVEL(), 0, NULL); + _Py_CODEUNIT *insert_exec_at = this_instr; + while (oparg > 255) { + oparg >>= 8; + insert_exec_at--; + } + _PyJIT_InitializeTracing(tstate, frame, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL); ENTER_TRACING(); } int _jump_taken = false; From d18c1a139c89d58cfe919828d057ea7ffbcf1b54 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 21 Oct 2025 00:53:41 +0100 Subject: [PATCH 079/190] Revert remedies against trace explosion --- Python/optimizer.c | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/Python/optimizer.c b/Python/optimizer.c index f67eccda716985..ca9967e4272d42 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -709,8 +709,6 @@ _PyJIT_translate_single_bytecode_to_trace( case POP_JUMP_IF_FALSE: case POP_JUMP_IF_TRUE: { - int counter = target_instr[1].cache; - int direction = _Py_popcount32(counter); _Py_CODEUNIT *computed_next_instr_without_modifiers = target_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; _Py_CODEUNIT *computed_next_instr = computed_next_instr_without_modifiers + (computed_next_instr_without_modifiers->op.code == NOT_TAKEN); _Py_CODEUNIT *computed_jump_instr = computed_next_instr_without_modifiers + oparg; @@ -718,20 +716,6 @@ _PyJIT_translate_single_bytecode_to_trace( int jump_happened = computed_jump_instr == next_instr; uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_happened]; ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(jump_happened ? computed_next_instr : computed_jump_instr, old_code)); - // Branch is biased to jumping, but jump did not happen. - // We are likely in a bad trace. So we should retrace later. - if ((direction > 10 && !jump_happened) || - // Branch is biased to not jumping, but jump did not happen. - // We are likely in a bad trace. So we should retrace later - (direction < 6 && jump_happened) || - // Finally, branch is just not heavily biased. - // So we should not trace through it anyways - // This prevents trace explosion - (direction >= 6 && direction <= 10) - ) { - ADD_TO_TRACE(_EXIT_TRACE, 0, 0, INSTR_IP(next_instr, old_code)); - goto full; - } break; } case JUMP_BACKWARD_JIT: From 7d17741473d641cdddd3626c032e93479bc98cbd Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 21 Oct 2025 21:21:29 +0100 Subject: [PATCH 080/190] First half of reviews --- Include/internal/pycore_interp_structs.h | 6 ++-- Include/internal/pycore_opcode_metadata.h | 23 +++++---------- Include/internal/pycore_optimizer.h | 1 - Python/bytecodes.c | 13 ++++----- Python/ceval_macros.h | 4 +-- Python/executor_cases.c.h | 3 +- Python/generated_cases.c.h | 2 +- Python/generated_tracer_cases.c.h | 12 ++++---- Python/optimizer.c | 33 ++++++++-------------- Python/optimizer_analysis.c | 6 +++- Tools/cases_generator/analyzer.py | 2 +- Tools/cases_generator/generators_common.py | 4 +-- Tools/cases_generator/tracer_generator.py | 2 +- 13 files changed, 46 insertions(+), 65 deletions(-) diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 142bcbece6220b..5285233b6b1342 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -757,7 +757,7 @@ struct _Py_unique_id_pool { typedef _Py_CODEUNIT *(*_PyJitEntryFuncPtr)(struct _PyExecutorObject *exec, _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate); -typedef struct _PyJitState { +typedef struct _PyJitTracerState { int jit_tracer_code_max_size; int jit_tracer_code_curr_size; _PyBloomFilter jit_tracer_dependencies; @@ -772,7 +772,7 @@ typedef struct _PyJitState { PyFunctionObject *jit_tracer_initial_func; // Strong struct _PyExitData *jit_tracer_previous_exit; _PyInterpreterFrame *jit_tracer_current_frame; -} _PyJitState; +} _PyJitTracerState; /* PyInterpreterState holds the global state for one of the runtime's interpreters. Typically the initial (main) interpreter is the only one. @@ -949,7 +949,7 @@ struct _is { struct types_state types; struct callable_cache callable_cache; PyObject *common_consts[NUM_COMMON_CONSTANTS]; - _PyJitState jit_state; + _PyJitTracerState jit_state; bool jit; bool compiling; struct _PyExecutorObject *executor_list_head; diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 5d642604515e46..e8e0c59874a546 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1777,30 +1777,17 @@ const uint8_t _PyOpcode_NeedsGuardIp[256] = { [INTERPRETER_EXIT] = 1, [RETURN_VALUE] = 1, [YIELD_VALUE] = 1, - [JUMP_FORWARD] = 1, - [JUMP_BACKWARD_NO_INTERRUPT] = 1, - [INSTRUMENTED_FOR_ITER] = 1, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = 1, [RETURN_GENERATOR] = 1, [BINARY_OP_SUBSCR_GETITEM] = 1, [INSTRUMENTED_RETURN_VALUE] = 1, [SEND] = 1, [SEND_GEN] = 1, [INSTRUMENTED_YIELD_VALUE] = 1, - [INSTRUMENTED_END_ASYNC_FOR] = 1, - [END_ASYNC_FOR] = 1, [LOAD_ATTR_PROPERTY] = 1, - [JUMP_BACKWARD] = 1, - [JUMP_BACKWARD_NO_JIT] = 1, - [JUMP_BACKWARD_JIT] = 1, - [POP_JUMP_IF_TRUE] = 1, - [POP_JUMP_IF_FALSE] = 1, - [POP_JUMP_IF_NONE] = 1, - [POP_JUMP_IF_NOT_NONE] = 1, - [FOR_ITER] = 1, - [FOR_ITER_LIST] = 1, - [FOR_ITER_TUPLE] = 1, - [FOR_ITER_RANGE] = 1, [FOR_ITER_GEN] = 1, + [CALL] = 1, + [INSTRUMENTED_CALL] = 1, [CALL_PY_GENERAL] = 1, [CALL_BOUND_METHOD_GENERAL] = 1, [CALL_BOUND_METHOD_EXACT_ARGS] = 1, @@ -1808,6 +1795,10 @@ const uint8_t _PyOpcode_NeedsGuardIp[256] = { [CALL_ALLOC_AND_ENTER_INIT] = 1, [CALL_KW_PY] = 1, [CALL_KW_BOUND_METHOD] = 1, + [CALL_KW] = 1, + [INSTRUMENTED_CALL_KW] = 1, + [CALL_FUNCTION_EX] = 1, + [INSTRUMENTED_CALL_FUNCTION_EX] = 1, }; #endif diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index b06593377324e3..0acf1940276966 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -37,7 +37,6 @@ typedef struct { typedef struct _PyExitData { uint32_t target; uint16_t index; - char is_dynamic; _Py_BackoffCounter temperature; struct _PyExecutorObject *executor; } _PyExitData; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 7df899dc50e8d9..ab5649281d9025 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1378,7 +1378,7 @@ dummy_func( if (err == 0) { assert(retval_o != NULL); JUMPBY(oparg); - RECORD_JUMP_TAKEN(); + RECORD_DYNAMIC_JUMP_TAKEN(); } else { PyStackRef_CLOSE(v); @@ -3241,7 +3241,7 @@ dummy_func( } // Jump forward by oparg and skip the following END_FOR JUMPBY(oparg + 1); - RECORD_JUMP_TAKEN(); + RECORD_DYNAMIC_JUMP_TAKEN(); DISPATCH(); } next = item; @@ -3303,7 +3303,7 @@ dummy_func( null_or_index = PyStackRef_TagInt(-1); /* Jump forward oparg, then skip following END_FOR instruction */ JUMPBY(oparg + 1); - RECORD_JUMP_TAKEN(); + RECORD_DYNAMIC_JUMP_TAKEN(); DISPATCH(); } #endif @@ -3381,7 +3381,7 @@ dummy_func( null_or_index = PyStackRef_TagInt(-1); /* Jump forward oparg, then skip following END_FOR instruction */ JUMPBY(oparg + 1); - RECORD_JUMP_TAKEN(); + RECORD_DYNAMIC_JUMP_TAKEN(); DISPATCH(); } } @@ -3426,7 +3426,7 @@ dummy_func( if (r->len <= 0) { // Jump over END_FOR instruction. JUMPBY(oparg + 1); - RECORD_JUMP_TAKEN(); + RECORD_DYNAMIC_JUMP_TAKEN(); DISPATCH(); } } @@ -5011,7 +5011,7 @@ dummy_func( LOAD_IP(frame->return_offset); #endif #if TIER_TWO - frame->instr_ptr += (frame->return_offset); + TIER2_STORE_IP(frame->return_offset); #endif RELOAD_STACK(); res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); @@ -5278,7 +5278,6 @@ dummy_func( tier2 op(_EXIT_TRACE, (exit_p/4 --)) { _PyExitData *exit = (_PyExitData *)exit_p; - assert(!exit->is_dynamic); #if defined(Py_DEBUG) && !defined(_Py_JIT) _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 256175b476dc55..e85adf88c679af 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -130,7 +130,7 @@ #endif #define TRACING_JUMP_TO_LABEL(label) \ - RECORD_JUMP_TAKEN() \ + RECORD_DYNAMIC_JUMP_TAKEN() \ RECORD_TRACE_NO_DISPATCH() \ assert(!IS_JIT_TRACING()); \ JUMP_TO_LABEL(label); @@ -381,7 +381,7 @@ GETITEM(PyObject *v, Py_ssize_t i) { #define RECORD_BRANCH_TAKEN(bitset, flag) #endif -#define RECORD_JUMP_TAKEN() _jump_taken = 1; +#define RECORD_DYNAMIC_JUMP_TAKEN() _jump_taken = 1; #define UNBOUNDLOCAL_ERROR_MSG \ "cannot access local variable '%s' where it is not associated with a value" diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 03846f48d538da..7e76e0a0719e7f 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -6767,7 +6767,7 @@ LOAD_IP(frame->return_offset); #endif #if TIER_TWO - frame->instr_ptr += (frame->return_offset); + TIER2_STORE_IP(frame->return_offset); #endif stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); @@ -7127,7 +7127,6 @@ case _EXIT_TRACE: { PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); _PyExitData *exit = (_PyExitData *)exit_p; - assert(!exit->is_dynamic); #if defined(Py_DEBUG) && !defined(_Py_JIT) _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 5cd7142adb3314..0aeff340806f17 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -10388,7 +10388,7 @@ LOAD_IP(frame->return_offset); #endif #if TIER_TWO - frame->instr_ptr += (frame->return_offset); + TIER2_STORE_IP(frame->return_offset); #endif stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); diff --git a/Python/generated_tracer_cases.c.h b/Python/generated_tracer_cases.c.h index 759bfd13cb83dc..123315df9e5ffe 100644 --- a/Python/generated_tracer_cases.c.h +++ b/Python/generated_tracer_cases.c.h @@ -6547,7 +6547,7 @@ TRACING_JUMP_TO_LABEL(error); } JUMPBY(oparg + 1); - RECORD_JUMP_TAKEN(); + RECORD_DYNAMIC_JUMP_TAKEN(); stack_pointer[-1] = null_or_index; TRACING_DISPATCH(); } @@ -6696,7 +6696,7 @@ if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { null_or_index = PyStackRef_TagInt(-1); JUMPBY(oparg + 1); - RECORD_JUMP_TAKEN(); + RECORD_DYNAMIC_JUMP_TAKEN(); stack_pointer[-1] = null_or_index; TRACING_DISPATCH(); } @@ -6787,7 +6787,7 @@ STAT_INC(FOR_ITER, hit); if (r->len <= 0) { JUMPBY(oparg + 1); - RECORD_JUMP_TAKEN(); + RECORD_DYNAMIC_JUMP_TAKEN(); TRACING_DISPATCH(); } } @@ -6860,7 +6860,7 @@ if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { null_or_index = PyStackRef_TagInt(-1); JUMPBY(oparg + 1); - RECORD_JUMP_TAKEN(); + RECORD_DYNAMIC_JUMP_TAKEN(); stack_pointer[-1] = null_or_index; TRACING_DISPATCH(); } @@ -12397,7 +12397,7 @@ LOAD_IP(frame->return_offset); #endif #if TIER_TWO - frame->instr_ptr += (frame->return_offset); + TIER2_STORE_IP(frame->return_offset); #endif stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); @@ -12548,7 +12548,7 @@ if (err == 0) { assert(retval_o != NULL); JUMPBY(oparg); - RECORD_JUMP_TAKEN(); + RECORD_DYNAMIC_JUMP_TAKEN(); } else { stack_pointer += -1; diff --git a/Python/optimizer.c b/Python/optimizer.c index ca9967e4272d42..c7967d09093666 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -141,13 +141,7 @@ _PyOptimizer_Optimize( interp->compiling = false; return 0; } - // We are the only one still holding a reference to this code object that - // is practically dead. - if (_PyObject_IsUniquelyReferenced((PyObject *)code) || _PyObject_IsUniquelyReferenced((PyObject *)tstate->interp->jit_state.jit_tracer_initial_func)) { - interp->compiling = false; - return 0; - } - // One of our depencies while tracing was invalidated. Not worth compiling. + // One of our dependencies while tracing was invalidated. Not worth compiling. if (!tstate->interp->jit_state.jit_tracer_dependencies_still_valid) { interp->compiling = false; return 0; @@ -603,12 +597,13 @@ _PyJIT_translate_single_bytecode_to_trace( } #endif + if (!tstate->interp->jit_state.jit_tracer_dependencies_still_valid) { + goto done; + } + DPRINTF(2, "%p %d: %s(%d) %d\n", old_code, target, _PyOpcode_OpName[opcode], oparg, progress_needed); - bool needs_guard_ip = _PyOpcode_NeedsGuardIp[opcode] && - !(opcode == FOR_ITER_RANGE || opcode == FOR_ITER_LIST || opcode == FOR_ITER_TUPLE) && - !(opcode == JUMP_BACKWARD_NO_INTERRUPT || opcode == JUMP_BACKWARD || opcode == JUMP_BACKWARD_JIT) && - !(opcode == POP_JUMP_IF_TRUE || opcode == POP_JUMP_IF_FALSE || opcode == POP_JUMP_IF_NONE || opcode == POP_JUMP_IF_NOT_NONE); + bool needs_guard_ip = _PyOpcode_NeedsGuardIp[opcode]; // Strange control-flow, unsupported opcode, etc. if (jump_taken || @@ -616,7 +611,7 @@ _PyJIT_translate_single_bytecode_to_trace( // If we haven't guarded the IP, then it's untraceable. (frame != tstate->interp->jit_state.jit_tracer_current_frame && !needs_guard_ip) || (oparg > 0xFFFF) || - // TODO (gh-140277): The constituent use one extra stack slot. So we need to check for heaedroom. + // TODO (gh-140277): The constituent use one extra stack slot. So we need to check for headroom. (opcode == BINARY_OP_SUBSCR_GETITEM && old_stack_level + 1 > old_code->co_stacksize)|| // Exception stuff, could be handled in the future maybe? opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO || @@ -1155,7 +1150,6 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil _PyExitData *exit = &executor->exits[next_exit]; exit->target = buffer[i].target; dest->operand0 = (uint64_t)exit; - exit->is_dynamic = (char)(opcode == _DYNAMIC_EXIT); next_exit--; } } @@ -1561,20 +1555,15 @@ _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj, int is } void -_Py_JITTracer_InvalidateDependency(PyThreadState *old_tstate, void *obj) +_Py_JITTracer_InvalidateDependency(PyThreadState *tstate, void *obj) { _PyBloomFilter obj_filter; _Py_BloomFilter_Init(&obj_filter); _Py_BloomFilter_Add(&obj_filter, obj); - PyInterpreterState *interp = old_tstate->interp; - - _Py_FOR_EACH_TSTATE_UNLOCKED(interp, tstate) { - if (bloom_filter_may_contain(&tstate->interp->jit_state.jit_tracer_dependencies, &obj_filter)) - { - tstate->interp->jit_state.jit_tracer_dependencies_still_valid = false; - } - + if (bloom_filter_may_contain(&tstate->interp->jit_state.jit_tracer_dependencies, &obj_filter)) + { + tstate->interp->jit_state.jit_tracer_dependencies_still_valid = false; } } /* Invalidate all executors */ diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 9f2a12efee6799..2d3f1d95d5ab14 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -528,10 +528,14 @@ _Py_uop_analyze_and_optimize( { OPT_STAT_INC(optimizer_attempts); - optimize_uops( + length = optimize_uops( initial_func, buffer, length, curr_stacklen, dependencies); + if (length == 0) { + return length; + } + assert(length > 0); length = remove_unneeded_uops(buffer, length); diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index facdabef83e72b..720bb5f0ec61f5 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -937,7 +937,7 @@ def compute_properties(op: parser.CodeDef) -> Properties: no_save_ip=no_save_ip, tier=tier_variable(op), needs_prev=variable_used(op, "prev_instr"), - needs_guard_ip=variable_used(op, "JUMPBY") or variable_used(op, "LLTRACE_RESUME_FRAME"), + needs_guard_ip=variable_used(op, "TIER2_STORE_IP") or variable_used(op, "LLTRACE_RESUME_FRAME") or variable_used(op, "DISPATCH_INLINED"), ) def expand(items: list[StackItem], oparg: int) -> list[StackItem]: diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 8a95e4ea6a345b..2d5f4607ab463b 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -128,7 +128,7 @@ def __init__(self, out: CWriter, labels: dict[str, Label], cannot_escape: bool = "DISPATCH": self.dispatch, "INSTRUCTION_SIZE": self.instruction_size, "stack_pointer": self.stack_pointer, - "RECORD_JUMP_TAKEN": self.record_jump_taken, + "RECORD_DYNAMIC_JUMP_TAKEN": self.record_dynamic_jump_taken, } self.out = out self.labels = labels @@ -476,7 +476,7 @@ def instruction_size(self, self.out.emit(f" {uop.instruction_size}u ") return True - def record_jump_taken( + def record_dynamic_jump_taken( self, tkn: Token, tkn_iter: TokenIterator, diff --git a/Tools/cases_generator/tracer_generator.py b/Tools/cases_generator/tracer_generator.py index 912056b6212a04..7227cca90f5129 100644 --- a/Tools/cases_generator/tracer_generator.py +++ b/Tools/cases_generator/tracer_generator.py @@ -96,7 +96,7 @@ def dispatch_same_oparg( emit_to(self.out, tkn_iter, "SEMI") return False - def record_jump_taken( + def record_dynamic_jump_taken( self, tkn: Token, tkn_iter: TokenIterator, From 8ebb6cbdd0ab0ae033dc864aee0d7681f4d3c82a Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 21 Oct 2025 21:30:07 +0100 Subject: [PATCH 081/190] Fix naming of things --- Include/internal/pycore_interp_structs.h | 26 +++--- Include/internal/pycore_optimizer.h | 9 +- Objects/codeobject.c | 2 +- Objects/frameobject.c | 2 +- Objects/funcobject.c | 4 +- Python/bytecodes.c | 10 +- Python/ceval.c | 4 +- Python/ceval_macros.h | 6 +- Python/executor_cases.c.h | 2 +- Python/generated_cases.c.h | 8 +- Python/generated_tracer_cases.c.h | 8 +- Python/instrumentation.c | 2 +- Python/optimizer.c | 112 +++++++++++------------ Python/pystate.c | 16 ++-- Tools/cases_generator/analyzer.py | 2 +- 15 files changed, 107 insertions(+), 106 deletions(-) diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 5285233b6b1342..ac1bcdf74ae417 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -758,20 +758,20 @@ struct _Py_unique_id_pool { typedef _Py_CODEUNIT *(*_PyJitEntryFuncPtr)(struct _PyExecutorObject *exec, _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate); typedef struct _PyJitTracerState { - int jit_tracer_code_max_size; - int jit_tracer_code_curr_size; - _PyBloomFilter jit_tracer_dependencies; - bool jit_tracer_dependencies_still_valid; - _PyUOpInstruction *jit_tracer_code_buffer; - _Py_CODEUNIT *jit_tracer_insert_exec_instr; - _Py_CODEUNIT *jit_tracer_close_loop_instr; + bool dependencies_still_valid; + int code_max_size; + int code_curr_size; + int initial_stack_depth; + int initial_chain_depth; + _PyUOpInstruction *code_buffer; + _Py_CODEUNIT *insert_exec_instr; + _Py_CODEUNIT *close_loop_instr; _Py_CODEUNIT *last_specialized_instr; - int jit_tracer_initial_stack_depth; - int jit_tracer_initial_chain_depth; - PyCodeObject *jit_tracer_initial_code; // Strong - PyFunctionObject *jit_tracer_initial_func; // Strong - struct _PyExitData *jit_tracer_previous_exit; - _PyInterpreterFrame *jit_tracer_current_frame; + PyCodeObject *initial_code; // Strong + PyFunctionObject *initial_func; // Strong + struct _PyExitData *previous_exit; + _PyInterpreterFrame *current_frame; + _PyBloomFilter dependencies; } _PyJitTracerState; /* PyInterpreterState holds the global state for one of the runtime's diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 0acf1940276966..a87cce85e9798b 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -359,7 +359,7 @@ extern void _Py_ClearExecutorDeletionList(PyInterpreterState *interp); #endif int -_PyJIT_translate_single_bytecode_to_trace( +_PyJit_translate_single_bytecode_to_trace( PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *this_instr, @@ -372,11 +372,12 @@ _PyJIT_translate_single_bytecode_to_trace( int jump_taken); void -_PyJIT_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *insert_exec_instr, _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit); +_PyJit_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *insert_exec_instr + _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit); -void _PyJIT_FinalizeTracing(PyThreadState *tstate); +void _PyJit_FinalizeTracing(PyThreadState *tstate); -void _Py_JITTracer_InvalidateDependency(PyThreadState *old_tstate, void *obj); +void _PyJit_Tracer_InvalidateDependency(PyThreadState *old_tstate, void *obj); #ifdef __cplusplus } diff --git a/Objects/codeobject.c b/Objects/codeobject.c index d0dd65300a1d8e..3a13cfaee2a5fc 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -2415,7 +2415,7 @@ code_dealloc(PyObject *self) PyMem_Free(co_extra); } #ifdef _Py_TIER2 - _Py_JITTracer_InvalidateDependency(tstate, self); + _PyJit_Tracer_InvalidateDependency(tstate, self); if (co->co_executors != NULL) { clear_executors(co); } diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 156daef7f9c6ad..abed4547ffd053 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -264,7 +264,7 @@ framelocalsproxy_setitem(PyObject *self, PyObject *key, PyObject *value) #if _Py_TIER2 _Py_Executors_InvalidateDependency(PyInterpreterState_Get(), co, 1); - _Py_JITTracer_InvalidateDependency(PyThreadState_GET(), co); + _PyJit_Tracer_InvalidateDependency(PyThreadState_GET(), co); #endif _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 5a801b616fef17..fcd79c7e4f4ea1 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -11,7 +11,7 @@ #include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_stats.h" #include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() -#include "pycore_optimizer.h" // _Py_JITTracer_InvalidateDependency +#include "pycore_optimizer.h" // _PyJit_Tracer_InvalidateDependency static const char * func_event_name(PyFunction_WatchEvent event) { @@ -1152,7 +1152,7 @@ func_dealloc(PyObject *self) return; } #if _Py_TIER2 - _Py_JITTracer_InvalidateDependency(PyThreadState_GET(), self); + _PyJit_Tracer_InvalidateDependency(PyThreadState_GET(), self); #endif _PyObject_GC_UNTRACK(op); FT_CLEAR_WEAKREFS(self, op->func_weakreflist); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index ab5649281d9025..f8742e6212d795 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2970,9 +2970,9 @@ dummy_func( if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD_JIT && next_instr->op.code != ENTER_EXECUTOR) { - if (tstate->interp->jit_state.jit_tracer_code_buffer == NULL) { - tstate->interp->jit_state.jit_tracer_code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); - if (tstate->interp->jit_state.jit_tracer_code_buffer == NULL) { + if (tstate->interp->jit_state.code_buffer == NULL) { + tstate->interp->jit_state.code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); + if (tstate->interp->jit_state.code_buffer == NULL) { // Don't error, just go to next instruction. DISPATCH(); } @@ -2985,7 +2985,7 @@ dummy_func( oparg >>= 8; insert_exec_at--; } - _PyJIT_InitializeTracing(tstate, frame, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL); + _PyJit_InitializeTracing(tstate, frame, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL); ENTER_TRACING(); } int _jump_taken = false; @@ -5456,7 +5456,7 @@ dummy_func( _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); assert(tstate->current_executor == (PyObject *)previous_executor); int chain_depth = previous_executor->vm_data.chain_depth + 1; - _PyJIT_InitializeTracing(tstate, frame, target, target, STACK_LEVEL(), chain_depth, exit); + _PyJit_InitializeTracing(tstate, frame, target, target, STACK_LEVEL(), chain_depth, exit); exit->temperature = initial_temperature_backoff_counter(); GOTO_TIER_ONE(target, 1); } diff --git a/Python/ceval.c b/Python/ceval.c index 08add3f75cd529..e8f1ed5f88b28d 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -990,8 +990,8 @@ static inline int add_to_code_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, PyCodeObject *old_code, PyFunctionObject *old_func, int old_stack_level, _Py_CODEUNIT *this_instr, _Py_CODEUNIT *next_instr, int opcode, int oparg, int jump_taken) { assert(frame != NULL); - assert(tstate->interp->jit_state.jit_tracer_code_curr_size < UOP_MAX_TRACE_LENGTH); - return !_PyJIT_translate_single_bytecode_to_trace(tstate, frame, this_instr, next_instr, old_code, old_func, old_stack_level, opcode, oparg, jump_taken); + assert(tstate->interp->jit_state.code_curr_size < UOP_MAX_TRACE_LENGTH); + return !_PyJit_translate_single_bytecode_to_trace(tstate, frame, this_instr, next_instr, old_code, old_func, old_stack_level, opcode, oparg, jump_taken); } #endif diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index e85adf88c679af..1d359461ffa067 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -150,7 +150,7 @@ if (!_PyErr_Occurred(tstate) && !_is_sys_tracing) { \ _PyFrame_SetStackPointer(frame, stack_pointer); \ int _err = _PyOptimizer_Optimize(frame, tstate); \ - _PyJIT_FinalizeTracing(tstate); \ + _PyJit_FinalizeTracing(tstate); \ stack_pointer = _PyFrame_GetStackPointer(frame); \ if (_err < 0) { \ JUMP_TO_LABEL(error); \ @@ -158,7 +158,7 @@ } \ else { \ _PyFrame_SetStackPointer(frame, stack_pointer); \ - _PyJIT_FinalizeTracing(tstate); \ + _PyJit_FinalizeTracing(tstate); \ stack_pointer = _PyFrame_GetStackPointer(frame); \ } \ } while (0); @@ -451,7 +451,7 @@ do { \ } \ if (keep_tracing_bit) { \ assert(next_instr->op.code != ENTER_EXECUTOR); \ - assert(tstate->interp->jit_state.jit_tracer_code_curr_size == 2); \ + assert(tstate->interp->jit_state.code_curr_size == 2); \ ENTER_TRACING(); \ } \ DISPATCH(); \ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 7e76e0a0719e7f..72927d8378c3bd 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7501,7 +7501,7 @@ _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); assert(tstate->current_executor == (PyObject *)previous_executor); int chain_depth = previous_executor->vm_data.chain_depth + 1; - _PyJIT_InitializeTracing(tstate, frame, target, target, STACK_LEVEL(), chain_depth, exit); + _PyJit_InitializeTracing(tstate, frame, target, target, STACK_LEVEL(), chain_depth, exit); exit->temperature = initial_temperature_backoff_counter(); GOTO_TIER_ONE(target, 1); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 0aeff340806f17..4fb61a06cdfe04 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -7661,11 +7661,11 @@ if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD_JIT && next_instr->op.code != ENTER_EXECUTOR) { - if (tstate->interp->jit_state.jit_tracer_code_buffer == NULL) { + if (tstate->interp->jit_state.code_buffer == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); - tstate->interp->jit_state.jit_tracer_code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); + tstate->interp->jit_state.code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); stack_pointer = _PyFrame_GetStackPointer(frame); - if (tstate->interp->jit_state.jit_tracer_code_buffer == NULL) { + if (tstate->interp->jit_state.code_buffer == NULL) { DISPATCH(); } } @@ -7676,7 +7676,7 @@ oparg >>= 8; insert_exec_at--; } - _PyJIT_InitializeTracing(tstate, frame, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL); + _PyJit_InitializeTracing(tstate, frame, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL); ENTER_TRACING(); } int _jump_taken = false; diff --git a/Python/generated_tracer_cases.c.h b/Python/generated_tracer_cases.c.h index 123315df9e5ffe..6317b02ddb9343 100644 --- a/Python/generated_tracer_cases.c.h +++ b/Python/generated_tracer_cases.c.h @@ -8946,11 +8946,11 @@ if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD_JIT && next_instr->op.code != ENTER_EXECUTOR) { - if (tstate->interp->jit_state.jit_tracer_code_buffer == NULL) { + if (tstate->interp->jit_state.code_buffer == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); - tstate->interp->jit_state.jit_tracer_code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); + tstate->interp->jit_state.code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); stack_pointer = _PyFrame_GetStackPointer(frame); - if (tstate->interp->jit_state.jit_tracer_code_buffer == NULL) { + if (tstate->interp->jit_state.code_buffer == NULL) { TRACING_DISPATCH(); } } @@ -8961,7 +8961,7 @@ oparg >>= 8; insert_exec_at--; } - _PyJIT_InitializeTracing(tstate, frame, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL); + _PyJit_InitializeTracing(tstate, frame, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL); ENTER_TRACING(); } int _jump_taken = false; diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 85fb73aaf098ee..81e46a331e0b9e 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1786,7 +1786,7 @@ force_instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp) _PyCode_Clear_Executors(code); } _Py_Executors_InvalidateDependency(interp, code, 1); - _Py_JITTracer_InvalidateDependency(PyThreadState_GET(), code); + _PyJit_Tracer_InvalidateDependency(PyThreadState_GET(), code); #endif int code_len = (int)Py_SIZE(code); /* Exit early to avoid creating instrumentation diff --git a/Python/optimizer.c b/Python/optimizer.c index c7967d09093666..8482162d4a325c 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -118,10 +118,10 @@ _PyOptimizer_Optimize( _PyInterpreterFrame *frame, PyThreadState *tstate) { PyInterpreterState *interp = _PyInterpreterState_GET(); - int chain_depth = tstate->interp->jit_state.jit_tracer_initial_chain_depth; + int chain_depth = tstate->interp->jit_state.initial_chain_depth; assert(interp->jit); assert(!interp->compiling); - assert(tstate->interp->jit_state.jit_tracer_initial_stack_depth >= 0); + assert(tstate->interp->jit_state.initial_stack_depth >= 0); #ifndef Py_GIL_DISABLED interp->compiling = true; // The first executor in a chain and the MAX_CHAIN_DEPTH'th executor *must* @@ -130,8 +130,8 @@ _PyOptimizer_Optimize( // this is true, since a deopt won't infinitely re-enter the executor: chain_depth %= MAX_CHAIN_DEPTH; bool progress_needed = chain_depth == 0; - PyCodeObject *code = (PyCodeObject *)tstate->interp->jit_state.jit_tracer_initial_code; - _Py_CODEUNIT *start = tstate->interp->jit_state.jit_tracer_insert_exec_instr; + PyCodeObject *code = (PyCodeObject *)tstate->interp->jit_state.initial_code; + _Py_CODEUNIT *start = tstate->interp->jit_state.insert_exec_instr; // A recursive trace might've cleared the values. In that case, bail. if (code == NULL) { interp->compiling = false; @@ -142,7 +142,7 @@ _PyOptimizer_Optimize( return 0; } // One of our dependencies while tracing was invalidated. Not worth compiling. - if (!tstate->interp->jit_state.jit_tracer_dependencies_still_valid) { + if (!tstate->interp->jit_state.dependencies_still_valid) { interp->compiling = false; return 0; } @@ -172,7 +172,7 @@ _PyOptimizer_Optimize( executor->vm_data.code = NULL; } if (chain_depth > 0) { - _PyExitData *prev_exit = tstate->interp->jit_state.jit_tracer_previous_exit; + _PyExitData *prev_exit = tstate->interp->jit_state.previous_exit; assert(prev_exit != NULL); prev_exit->executor = executor;; } @@ -551,7 +551,7 @@ add_to_trace( /* Returns 1 on success (added to trace), 0 on trace end. */ int -_PyJIT_translate_single_bytecode_to_trace( +_PyJit_translate_single_bytecode_to_trace( PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *this_instr, @@ -564,13 +564,13 @@ _PyJIT_translate_single_bytecode_to_trace( int jump_taken) { - int is_first_instr = tstate->interp->jit_state.jit_tracer_close_loop_instr == this_instr; - bool progress_needed = (tstate->interp->jit_state.jit_tracer_initial_chain_depth % MAX_CHAIN_DEPTH) == 0;; - _PyBloomFilter *dependencies = &tstate->interp->jit_state.jit_tracer_dependencies; + int is_first_instr = tstate->interp->jit_state.close_loop_instr == this_instr; + bool progress_needed = (tstate->interp->jit_state.initial_chain_depth % MAX_CHAIN_DEPTH) == 0;; + _PyBloomFilter *dependencies = &tstate->interp->jit_state.dependencies; _Py_BloomFilter_Add(dependencies, old_code); - int trace_length = tstate->interp->jit_state.jit_tracer_code_curr_size; - _PyUOpInstruction *trace = tstate->interp->jit_state.jit_tracer_code_buffer; - int max_length = tstate->interp->jit_state.jit_tracer_code_max_size; + int trace_length = tstate->interp->jit_state.code_curr_size; + _PyUOpInstruction *trace = tstate->interp->jit_state.code_buffer; + int max_length = tstate->interp->jit_state.code_max_size; #ifdef Py_DEBUG char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); @@ -597,7 +597,7 @@ _PyJIT_translate_single_bytecode_to_trace( } #endif - if (!tstate->interp->jit_state.jit_tracer_dependencies_still_valid) { + if (!tstate->interp->jit_state.dependencies_still_valid) { goto done; } @@ -609,7 +609,7 @@ _PyJIT_translate_single_bytecode_to_trace( if (jump_taken || // This happens when a recursive call happens that we can't trace. Such as Python -> C -> Python calls // If we haven't guarded the IP, then it's untraceable. - (frame != tstate->interp->jit_state.jit_tracer_current_frame && !needs_guard_ip) || + (frame != tstate->interp->jit_state.current_frame && !needs_guard_ip) || (oparg > 0xFFFF) || // TODO (gh-140277): The constituent use one extra stack slot. So we need to check for headroom. (opcode == BINARY_OP_SUBSCR_GETITEM && old_stack_level + 1 > old_code->co_stacksize)|| @@ -638,7 +638,7 @@ _PyJIT_translate_single_bytecode_to_trace( } } - tstate->interp->jit_state.jit_tracer_current_frame = frame; + tstate->interp->jit_state.current_frame = frame; if (opcode == NOP) { return 1; @@ -672,7 +672,7 @@ _PyJIT_translate_single_bytecode_to_trace( /* Special case the first instruction, * so that we can guarantee forward progress */ - if (progress_needed && tstate->interp->jit_state.jit_tracer_code_curr_size <= 2) { + if (progress_needed && tstate->interp->jit_state.code_curr_size <= 2) { if (OPCODE_HAS_EXIT(opcode) || OPCODE_HAS_DEOPT(opcode)) { opcode = _PyOpcode_Deopt[opcode]; } @@ -681,7 +681,7 @@ _PyJIT_translate_single_bytecode_to_trace( } // Loop back to the start - if (is_first_instr && tstate->interp->jit_state.jit_tracer_code_curr_size > 5) { + if (is_first_instr && tstate->interp->jit_state.code_curr_size > 5) { ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0, 0); goto done; } @@ -849,29 +849,29 @@ _PyJIT_translate_single_bytecode_to_trace( if (needs_guard_ip) { ADD_TO_TRACE(_GUARD_IP, 0, (uintptr_t)next_instr, 0); } - tstate->interp->jit_state.jit_tracer_code_curr_size = trace_length; - tstate->interp->jit_state.jit_tracer_code_max_size = max_length; + tstate->interp->jit_state.code_curr_size = trace_length; + tstate->interp->jit_state.code_max_size = max_length; return 1; done: - tstate->interp->jit_state.jit_tracer_code_curr_size = trace_length; - tstate->interp->jit_state.jit_tracer_code_max_size = max_length; + tstate->interp->jit_state.code_curr_size = trace_length; + tstate->interp->jit_state.code_max_size = max_length; return 0; full: - if (!is_terminator(&tstate->interp->jit_state.jit_tracer_code_buffer[trace_length-1])) { + if (!is_terminator(&tstate->interp->jit_state.code_buffer[trace_length-1])) { // Undo the last few instructions. - trace_length = tstate->interp->jit_state.jit_tracer_code_curr_size; - max_length = tstate->interp->jit_state.jit_tracer_code_max_size; + trace_length = tstate->interp->jit_state.code_curr_size; + max_length = tstate->interp->jit_state.code_max_size; // We previously reversed one. max_length += 1; ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); } - tstate->interp->jit_state.jit_tracer_code_curr_size = trace_length; - tstate->interp->jit_state.jit_tracer_code_max_size = max_length; + tstate->interp->jit_state.code_curr_size = trace_length; + tstate->interp->jit_state.code_max_size = max_length; return 0; } void -_PyJIT_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *insert_exec_instr, _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit) +_PyJit_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *insert_exec_instr, _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit) { PyCodeObject *code = _PyFrame_GetCode(frame); #ifdef Py_DEBUG @@ -888,30 +888,30 @@ _PyJIT_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_ 2 * INSTR_IP(close_loop_instr, code), chain_depth); #endif - add_to_trace(tstate->interp->jit_state.jit_tracer_code_buffer, 0, _START_EXECUTOR, 0, (uintptr_t)insert_exec_instr, INSTR_IP(insert_exec_instr, code)); - add_to_trace(tstate->interp->jit_state.jit_tracer_code_buffer, 1, _MAKE_WARM, 0, 0, 0); - tstate->interp->jit_state.jit_tracer_code_curr_size = 2; - tstate->interp->jit_state.jit_tracer_code_max_size = UOP_MAX_TRACE_LENGTH; - tstate->interp->jit_state.jit_tracer_insert_exec_instr = insert_exec_instr; - tstate->interp->jit_state.jit_tracer_close_loop_instr = close_loop_instr; - tstate->interp->jit_state.jit_tracer_initial_code = (PyCodeObject *)Py_NewRef(code); - tstate->interp->jit_state.jit_tracer_initial_func = (PyFunctionObject *)Py_NewRef(_PyFrame_GetFunction(frame)); - tstate->interp->jit_state.jit_tracer_previous_exit = exit; - _Py_BloomFilter_Init(&tstate->interp->jit_state.jit_tracer_dependencies); - tstate->interp->jit_state.jit_tracer_initial_stack_depth = curr_stackdepth; - tstate->interp->jit_state.jit_tracer_initial_chain_depth = chain_depth % MAX_CHAIN_DEPTH; - tstate->interp->jit_state.jit_tracer_current_frame = frame; - tstate->interp->jit_state.jit_tracer_dependencies_still_valid = true; + add_to_trace(tstate->interp->jit_state.code_buffer, 0, _START_EXECUTOR, 0, (uintptr_t)insert_exec_instr, INSTR_IP(insert_exec_instr, code)); + add_to_trace(tstate->interp->jit_state.code_buffer, 1, _MAKE_WARM, 0, 0, 0); + tstate->interp->jit_state.code_curr_size = 2; + tstate->interp->jit_state.code_max_size = UOP_MAX_TRACE_LENGTH; + tstate->interp->jit_state.insert_exec_instr = insert_exec_instr; + tstate->interp->jit_state.close_loop_instr = close_loop_instr; + tstate->interp->jit_state.initial_code = (PyCodeObject *)Py_NewRef(code); + tstate->interp->jit_state.initial_func = (PyFunctionObject *)Py_NewRef(_PyFrame_GetFunction(frame)); + tstate->interp->jit_state.previous_exit = exit; + _Py_BloomFilter_Init(&tstate->interp->jit_state.dependencies); + tstate->interp->jit_state.initial_stack_depth = curr_stackdepth; + tstate->interp->jit_state.initial_chain_depth = chain_depth % MAX_CHAIN_DEPTH; + tstate->interp->jit_state.current_frame = frame; + tstate->interp->jit_state.dependencies_still_valid = true; tstate->interp->jit_state.last_specialized_instr = NULL; } void -_PyJIT_FinalizeTracing(PyThreadState *tstate) +_PyJit_FinalizeTracing(PyThreadState *tstate) { - Py_CLEAR(tstate->interp->jit_state.jit_tracer_initial_code); - Py_CLEAR(tstate->interp->jit_state.jit_tracer_initial_func); - tstate->interp->jit_state.jit_tracer_code_curr_size = 2; - tstate->interp->jit_state.jit_tracer_code_max_size = UOP_MAX_TRACE_LENGTH - 1; + Py_CLEAR(tstate->interp->jit_state.initial_code); + Py_CLEAR(tstate->interp->jit_state.initial_func); + tstate->interp->jit_state.code_curr_size = 2; + tstate->interp->jit_state.code_max_size = UOP_MAX_TRACE_LENGTH - 1; } @@ -1215,17 +1215,17 @@ uop_optimize( _PyExecutorObject **exec_ptr, bool progress_needed) { - _PyBloomFilter *dependencies = &tstate->interp->jit_state.jit_tracer_dependencies; + _PyBloomFilter *dependencies = &tstate->interp->jit_state.dependencies; PyInterpreterState *interp = _PyInterpreterState_GET(); - _PyUOpInstruction *buffer = interp->jit_state.jit_tracer_code_buffer; + _PyUOpInstruction *buffer = interp->jit_state.code_buffer; OPT_STAT_INC(attempts); char *env_var = Py_GETENV("PYTHON_UOPS_OPTIMIZE"); bool is_noopt = true; if (env_var == NULL || *env_var == '\0' || *env_var > '0') { is_noopt = false; } - int curr_stackentries = tstate->interp->jit_state.jit_tracer_initial_stack_depth; - int length = interp->jit_state.jit_tracer_code_curr_size; + int curr_stackentries = tstate->interp->jit_state.initial_stack_depth; + int length = interp->jit_state.code_curr_size; // Trace too short, don't bother. if (length <= 5) { return 0; @@ -1234,7 +1234,7 @@ uop_optimize( assert(length < UOP_MAX_TRACE_LENGTH); OPT_STAT_INC(traces_created); if (!is_noopt) { - length = _Py_uop_analyze_and_optimize(tstate->interp->jit_state.jit_tracer_initial_func, buffer, + length = _Py_uop_analyze_and_optimize(tstate->interp->jit_state.initial_func, buffer, length, curr_stackentries, dependencies); if (length <= 0) { @@ -1259,7 +1259,7 @@ uop_optimize( OPT_HIST(effective_trace_length(buffer, length), optimized_trace_length_hist); length = prepare_for_execution(buffer, length); assert(length <= UOP_MAX_TRACE_LENGTH); - _PyExecutorObject *executor = make_executor_from_uops(buffer, length, dependencies, tstate->interp->jit_state.jit_tracer_initial_chain_depth); + _PyExecutorObject *executor = make_executor_from_uops(buffer, length, dependencies, tstate->interp->jit_state.initial_chain_depth); if (executor == NULL) { return -1; } @@ -1555,15 +1555,15 @@ _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj, int is } void -_Py_JITTracer_InvalidateDependency(PyThreadState *tstate, void *obj) +_PyJit_Tracer_InvalidateDependency(PyThreadState *tstate, void *obj) { _PyBloomFilter obj_filter; _Py_BloomFilter_Init(&obj_filter); _Py_BloomFilter_Add(&obj_filter, obj); - if (bloom_filter_may_contain(&tstate->interp->jit_state.jit_tracer_dependencies, &obj_filter)) + if (bloom_filter_may_contain(&tstate->interp->jit_state.dependencies, &obj_filter)) { - tstate->interp->jit_state.jit_tracer_dependencies_still_valid = false; + tstate->interp->jit_state.dependencies_still_valid = false; } } /* Invalidate all executors */ diff --git a/Python/pystate.c b/Python/pystate.c index f0790322abd1ff..ebb37fc8400eb2 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -556,11 +556,11 @@ init_interpreter(PyInterpreterState *interp, #endif #ifdef _Py_TIER2 - interp->jit_state.jit_tracer_code_buffer = NULL; - interp->jit_state.jit_tracer_initial_stack_depth = -1; - interp->jit_state.jit_tracer_initial_chain_depth = -1; - interp->jit_state.jit_tracer_initial_code = NULL; - interp->jit_state.jit_tracer_initial_func = NULL; + interp->jit_state.code_buffer = NULL; + interp->jit_state.initial_stack_depth = -1; + interp->jit_state.initial_chain_depth = -1; + interp->jit_state.initial_code = NULL; + interp->jit_state.initial_func = NULL; #endif llist_init(&interp->mem_free_queue.head); llist_init(&interp->asyncio_tasks_head); @@ -811,9 +811,9 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) #ifdef _Py_TIER2 _Py_ClearExecutorDeletionList(interp); - if (interp->jit_state.jit_tracer_code_buffer != NULL) { - _PyObject_VirtualFree(interp->jit_state.jit_tracer_code_buffer, UOP_BUFFER_SIZE); - interp->jit_state.jit_tracer_code_buffer = NULL; + if (interp->jit_state.code_buffer != NULL) { + _PyObject_VirtualFree(interp->jit_state.code_buffer, UOP_BUFFER_SIZE); + interp->jit_state.code_buffer = NULL; } #endif _PyAST_Fini(interp); diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 720bb5f0ec61f5..7a5e8786f2a804 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -696,7 +696,7 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "PyStackRef_Unwrap", "_PyLong_CheckExactAndCompact", "_PyExecutor_FromExit", - "_PyJIT_InitializeTracing", + "_PyJit_InitializeTracing", ) From c23e591db02e1676ed79a01211e075d325b7ac19 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 21 Oct 2025 21:34:48 +0100 Subject: [PATCH 082/190] restore optimizer code --- Include/internal/pycore_optimizer.h | 2 +- Python/optimizer_bytecodes.c | 12 ++++++++++-- Python/optimizer_cases.c.h | 11 +++++++++-- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index a87cce85e9798b..9c3aaf1edaf550 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -372,7 +372,7 @@ _PyJit_translate_single_bytecode_to_trace( int jump_taken); void -_PyJit_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *insert_exec_instr +_PyJit_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *insert_exec_instr, _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit); void _PyJit_FinalizeTracing(PyThreadState *tstate); diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 734a9193af7f89..09c5ae764e5ac3 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -794,10 +794,18 @@ dummy_func(void) { op(_RETURN_GENERATOR, ( -- res)) { SYNC_SP(); + PyCodeObject *co = get_current_code_object(ctx); + ctx->frame->stack_pointer = stack_pointer; + frame_pop(ctx); stack_pointer = ctx->frame->stack_pointer; res = sym_new_unknown(ctx); - ctx->done = true; - ctx->out_of_space = true; + /* Stack space handling */ + assert(corresponding_check_stack == NULL); + assert(co != NULL); + int framesize = co->co_framesize; + assert(framesize > 0); + assert(framesize <= curr_space); + curr_space -= framesize; } op(_YIELD_VALUE, (unused -- value)) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 99d8d8ae33397d..002da75ea38481 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -3011,10 +3011,17 @@ case _RETURN_GENERATOR: { JitOptRef res; + PyCodeObject *co = get_current_code_object(ctx); + ctx->frame->stack_pointer = stack_pointer; + frame_pop(ctx); stack_pointer = ctx->frame->stack_pointer; res = sym_new_unknown(ctx); - ctx->done = true; - ctx->out_of_space = true; + assert(corresponding_check_stack == NULL); + assert(co != NULL); + int framesize = co->co_framesize; + assert(framesize > 0); + assert(framesize <= curr_space); + curr_space -= framesize; stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); From a62fe40ccb6101f3795dbd3bdf5c5d9fdb692edb Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 21 Oct 2025 22:11:44 +0100 Subject: [PATCH 083/190] Clean up macros --- Python/bytecodes.c | 2 +- Python/ceval.c | 14 ++++++++++++++ Python/ceval_macros.h | 35 +++++++++++------------------------ Python/optimizer.c | 5 +++++ 4 files changed, 31 insertions(+), 25 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index f8742e6212d795..388da83dbc24bf 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2979,7 +2979,7 @@ dummy_func( } int _is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL); if (!_is_sys_tracing) { - /* Back up over EXTENDED_ARGs so executor is inserted at the corret place */ + /* Back up over EXTENDED_ARGs so executor is inserted at the correct place */ _Py_CODEUNIT *insert_exec_at = this_instr; while (oparg > 255) { oparg >>= 8; diff --git a/Python/ceval.c b/Python/ceval.c index e8f1ed5f88b28d..3ca8378294998c 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -993,6 +993,20 @@ add_to_code_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, PyCodeObjec assert(tstate->interp->jit_state.code_curr_size < UOP_MAX_TRACE_LENGTH); return !_PyJit_translate_single_bytecode_to_trace(tstate, frame, this_instr, next_instr, old_code, old_func, old_stack_level, opcode, oparg, jump_taken); } + + +// 0 for success, -1 for error. +static int +bail_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame) +{ + int _is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL); + int err = 0; + if (!_PyErr_Occurred(tstate) && !_is_sys_tracing) { + err = _PyOptimizer_Optimize(frame, tstate); + } + _PyJit_FinalizeTracing(tstate); + return err; +} #endif /* _PyEval_EvalFrameDefault is too large to optimize for speed with PGO on MSVC. diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 1d359461ffa067..e19a614f28dec6 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -144,32 +144,19 @@ DISPATCH_TABLE_VAR = TRACING_DISPATCH_TABLE; # define LEAVE_TRACING() \ DISPATCH_TABLE_VAR = DISPATCH_TABLE; -# define BAIL_TRACING_NO_DISPATCH() \ - do { \ - LEAVE_TRACING(); \ - if (!_PyErr_Occurred(tstate) && !_is_sys_tracing) { \ - _PyFrame_SetStackPointer(frame, stack_pointer); \ - int _err = _PyOptimizer_Optimize(frame, tstate); \ - _PyJit_FinalizeTracing(tstate); \ - stack_pointer = _PyFrame_GetStackPointer(frame); \ - if (_err < 0) { \ - JUMP_TO_LABEL(error); \ - } \ - } \ - else { \ - _PyFrame_SetStackPointer(frame, stack_pointer); \ - _PyJit_FinalizeTracing(tstate); \ - stack_pointer = _PyFrame_GetStackPointer(frame); \ - } \ - } while (0); # define RECORD_TRACE_NO_DISPATCH() do { \ - int _is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL); \ - if (_is_sys_tracing) { \ - LEAVE_TRACING(); \ - } \ - else if ((IS_JIT_TRACING() && add_to_code_trace(tstate, frame, old_code, old_func, _old_stack_level, this_instr, next_instr, opcode, oparg, _jump_taken))) { \ - BAIL_TRACING_NO_DISPATCH(); \ + int err = 0; \ + _PyFrame_SetStackPointer(frame, stack_pointer); \ + /* We need to check once more here in case it swapped out halfway. */ \ + if (IS_JIT_TRACING()) { \ + int full = add_to_code_trace(tstate, frame, old_code, old_func, _old_stack_level, this_instr, next_instr, opcode, oparg, _jump_taken); \ + if (full) { \ + LEAVE_TRACING(); \ + err = bail_tracing_and_jit(tstate, frame); \ + } \ } \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ + if (err < 0) { JUMP_TO_LABEL(error); } \ } while (0); #endif diff --git a/Python/optimizer.c b/Python/optimizer.c index 8482162d4a325c..f77ff500e2f9d4 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -572,6 +572,11 @@ _PyJit_translate_single_bytecode_to_trace( _PyUOpInstruction *trace = tstate->interp->jit_state.code_buffer; int max_length = tstate->interp->jit_state.code_max_size; + int is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL); + if (is_sys_tracing) { + goto full; + } + #ifdef Py_DEBUG char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); int lltrace = 0; From e4f162432afd966090f9e2cdf7de99b4614121ba Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 21 Oct 2025 22:36:15 +0100 Subject: [PATCH 084/190] Clean up the cases generator --- Python/generated_tracer_cases.c.h | 14 ---- Tools/cases_generator/generators_common.py | 14 ++-- Tools/cases_generator/tier1_generator.py | 24 ++---- Tools/cases_generator/tracer_generator.py | 91 +++++++++++++++------- 4 files changed, 74 insertions(+), 69 deletions(-) diff --git a/Python/generated_tracer_cases.c.h b/Python/generated_tracer_cases.c.h index 6317b02ddb9343..932a57a22e5110 100644 --- a/Python/generated_tracer_cases.c.h +++ b/Python/generated_tracer_cases.c.h @@ -1797,7 +1797,6 @@ (void)_jump_taken; int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; (void)(_old_stack_level); - opcode = CALL; _PyStackRef callable; _PyStackRef self_or_null; _PyStackRef *args; @@ -2853,7 +2852,6 @@ (void)_jump_taken; int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; (void)(_old_stack_level); - opcode = CALL_FUNCTION_EX; _PyStackRef func; _PyStackRef callargs; _PyStackRef func_st; @@ -3210,7 +3208,6 @@ (void)_jump_taken; int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; (void)(_old_stack_level); - opcode = CALL_KW; _PyStackRef callable; _PyStackRef self_or_null; _PyStackRef *args; @@ -3541,7 +3538,6 @@ (void)_jump_taken; int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; (void)(_old_stack_level); - opcode = CALL_KW_NON_PY; static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -4461,7 +4457,6 @@ (void)_jump_taken; int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; (void)(_old_stack_level); - opcode = CALL_NON_PY_GENERAL; static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); _PyStackRef callable; _PyStackRef self_or_null; @@ -6307,7 +6302,6 @@ (void)_jump_taken; int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; (void)(_old_stack_level); - opcode = ENTER_EXECUTOR; #ifdef _Py_TIER2 PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; @@ -6391,7 +6385,6 @@ (void)_jump_taken; int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; (void)(_old_stack_level); - opcode = EXTENDED_ARG; assert(oparg); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; @@ -7294,7 +7287,6 @@ (void)_jump_taken; int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; (void)(_old_stack_level); - opcode = INSTRUMENTED_CALL; _PyStackRef callable; _PyStackRef self_or_null; _PyStackRef func; @@ -7492,7 +7484,6 @@ (void)_jump_taken; int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; (void)(_old_stack_level); - opcode = INSTRUMENTED_CALL_FUNCTION_EX; _PyStackRef func; _PyStackRef callargs; _PyStackRef func_st; @@ -7669,7 +7660,6 @@ (void)_jump_taken; int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; (void)(_old_stack_level); - opcode = INSTRUMENTED_CALL_KW; _PyStackRef callable; _PyStackRef self_or_null; _PyStackRef *args; @@ -8057,7 +8047,6 @@ (void)_jump_taken; int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; (void)(_old_stack_level); - opcode = INSTRUMENTED_INSTRUCTION; _PyFrame_SetStackPointer(frame, stack_pointer); int next_opcode = _Py_call_instrumentation_instruction( tstate, frame, this_instr); @@ -8156,7 +8145,6 @@ (void)_jump_taken; int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; (void)(_old_stack_level); - opcode = INSTRUMENTED_LINE; int original_opcode = 0; if (tstate->tracing) { PyCodeObject *code = _PyFrame_GetCode(frame); @@ -8205,7 +8193,6 @@ (void)_jump_taken; int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; (void)(_old_stack_level); - opcode = INSTRUMENTED_LOAD_SUPER_ATTR; _PyStackRef global_super_st; _PyStackRef class_st; _PyStackRef self_st; @@ -11163,7 +11150,6 @@ (void)_jump_taken; int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; (void)(_old_stack_level); - opcode = LOAD_SUPER_ATTR; _PyStackRef global_super_st; _PyStackRef class_st; _PyStackRef self_st; diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 2d5f4607ab463b..3edefb110834a9 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -107,9 +107,9 @@ class Emitter: labels: dict[str, Label] _replacers: dict[str, ReplacementFunctionType] cannot_escape: bool - tracing: str + jump_prefix: str - def __init__(self, out: CWriter, labels: dict[str, Label], cannot_escape: bool = False, is_tracing: bool = False): + def __init__(self, out: CWriter, labels: dict[str, Label], cannot_escape: bool = False, jump_prefix: str = ""): self._replacers = { "EXIT_IF": self.exit_if, "AT_END_EXIT_IF": self.exit_if_after, @@ -133,7 +133,7 @@ def __init__(self, out: CWriter, labels: dict[str, Label], cannot_escape: bool = self.out = out self.labels = labels self.cannot_escape = cannot_escape - self.tracing = "TRACING_" if is_tracing else "" + self.jump_prefix = jump_prefix def dispatch( self, @@ -170,7 +170,7 @@ def deopt_if( family_name = inst.family.name self.emit(f"UPDATE_MISS_STATS({family_name});\n") self.emit(f"assert(_PyOpcode_Deopt[opcode] == ({family_name}));\n") - self.emit(f"JUMP_TO_PREDICTED({family_name});\n") + self.emit(f"JUMP_TO_PREDICTED({self.jump_prefix}{family_name});\n") self.emit("}\n") return not always_true(first_tkn) @@ -201,10 +201,10 @@ def exit_if_after( def goto_error(self, offset: int, storage: Storage) -> str: if offset > 0: - return f"{self.tracing}JUMP_TO_LABEL(pop_{offset}_error);" + return f"{self.jump_prefix}JUMP_TO_LABEL(pop_{offset}_error);" if offset < 0: storage.copy().flush(self.out) - return f"{self.tracing}JUMP_TO_LABEL(error);" + return f"{self.jump_prefix}JUMP_TO_LABEL(error);" def error_if( self, @@ -424,7 +424,7 @@ def goto_label(self, goto: Token, label: Token, storage: Storage) -> None: elif storage.spilled: raise analysis_error("Cannot jump from spilled label without reloading the stack pointer", goto) self.out.start_line() - self.out.emit(f"{self.tracing}JUMP_TO_LABEL(") + self.out.emit(f"{self.jump_prefix}JUMP_TO_LABEL(") self.out.emit(label) self.out.emit(")") diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index c77074f656c9e0..25a7442a0ef611 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -217,15 +217,12 @@ def get_popped(inst: Instruction, analysis: Analysis) -> str: return (-stack.base_offset).to_c() def generate_tier1_cases( - analysis: Analysis, out: CWriter, emitter: Emitter, is_tracing: bool = False + analysis: Analysis, out: CWriter, emitter: Emitter ) -> None: - tracing_prepend = "TRACING_" if is_tracing else "" out.emit("\n") for name, inst in sorted(analysis.instructions.items()): out.emit("\n") - out.emit(f"{tracing_prepend}TARGET({name}) {{\n") - if is_tracing: - out.emit(f"assert(IS_JIT_TRACING());\n") + out.emit(f"TARGET({name}) {{\n") popped = get_popped(inst, analysis) # We need to ifdef it because this breaks platforms # without computed gotos/tail calling. @@ -233,7 +230,7 @@ def generate_tier1_cases( out.emit(f"int opcode = {name};\n") out.emit(f"(void)(opcode);\n") out.emit(f"#endif\n") - needs_this = is_tracing or uses_this(inst) + needs_this = uses_this(inst) unused_guard = "(void)this_instr;\n" if inst.properties.needs_prev: out.emit(f"_Py_CODEUNIT* const prev_instr = frame->instr_ptr;\n") @@ -247,21 +244,10 @@ def generate_tier1_cases( out.emit(f"next_instr += {inst.size};\n") out.emit(f"INSTRUCTION_STATS({name});\n") if inst.is_target: - out.emit(f"PREDICTED_{tracing_prepend}{name}:;\n") + out.emit(f"PREDICTED_{name}:;\n") if needs_this: out.emit(f"_Py_CODEUNIT* const this_instr = next_instr - {inst.size};\n") out.emit(unused_guard) - if is_tracing: - # This is required so that the predicted ops reflect the correct opcode. - out.emit(f"opcode = {name};\n") - out.emit(f"PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable);\n") - out.emit(f"(void)old_code;\n") - out.emit(f"PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj);\n") - out.emit(f"(void)old_func;\n") - out.emit(f"int _jump_taken = false;\n") - out.emit(f"(void)_jump_taken;\n") - out.emit(f"int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0;\n") - out.emit(f"(void)(_old_stack_level);\n") if inst.properties.uses_opcode: out.emit(f"opcode = {name};\n") if inst.family is not None: @@ -279,7 +265,7 @@ def generate_tier1_cases( out.start_line() if reachable: # type: ignore[possibly-undefined] stack.flush(out) - out.emit(f"{tracing_prepend}DISPATCH();\n") + out.emit(f"DISPATCH();\n") out.start_line() out.emit("}") out.emit("\n") diff --git a/Tools/cases_generator/tracer_generator.py b/Tools/cases_generator/tracer_generator.py index 7227cca90f5129..1b7d9a5acea6bc 100644 --- a/Tools/cases_generator/tracer_generator.py +++ b/Tools/cases_generator/tracer_generator.py @@ -28,7 +28,7 @@ from typing import TextIO from lexer import Token from stack import Local, Stack, StackError, get_stack_effect, Storage -from tier1_generator import generate_tier1_cases +from tier1_generator import get_popped, declare_variables, write_uop DEFAULT_OUTPUT = ROOT / "Python/generated_tracer_cases.c.h" @@ -39,7 +39,7 @@ class TracerEmitter(Emitter): cannot_escape: bool def __init__(self, out: CWriter, labels: dict[str, Label], cannot_escape: bool = False): - super().__init__(out, labels, cannot_escape, is_tracing=True) + super().__init__(out, labels, cannot_escape, jump_prefix="TRACING_") self._replacers = { **self._replacers, "DISPATCH": self.dispatch, @@ -109,38 +109,71 @@ def record_dynamic_jump_taken( self.out.emit(";\n") return True - def deopt_if( - self, - tkn: Token, - tkn_iter: TokenIterator, - uop: CodeSection, - storage: Storage, - inst: Instruction | None, - ) -> bool: - self.out.start_line() - self.out.emit("if (") - lparen = next(tkn_iter) - assert lparen.kind == "LPAREN" - first_tkn = tkn_iter.peek() - emit_to(self.out, tkn_iter, "RPAREN") - self.emit(") {\n") - next(tkn_iter) # Semi colon - assert inst is not None - assert inst.family is not None - family_name = inst.family.name - self.emit(f"UPDATE_MISS_STATS({family_name});\n") - self.emit(f"assert(_PyOpcode_Deopt[opcode] == ({family_name}));\n") - self.emit(f"JUMP_TO_PREDICTED(TRACING_{family_name});\n") - self.emit("}\n") - return not always_true(first_tkn) - - exit_if = deopt_if +def generate_tier1_tracer_cases( + analysis: Analysis, out: CWriter, emitter: Emitter +) -> None: + out.emit("\n") + for name, inst in sorted(analysis.instructions.items()): + out.emit("\n") + out.emit(f"TRACING_TARGET({name}) {{\n") + out.emit(f"assert(IS_JIT_TRACING());\n") + # We need to ifdef it because this breaks platforms + # without computed gotos/tail calling. + out.emit(f"#if _Py_TAIL_CALL_INTERP\n") + out.emit(f"int opcode = {name};\n") + out.emit(f"(void)(opcode);\n") + out.emit(f"#endif\n") + unused_guard = "(void)this_instr;\n" + if inst.properties.needs_prev: + out.emit(f"_Py_CODEUNIT* const prev_instr = frame->instr_ptr;\n") + if not inst.is_target: + out.emit(f"_Py_CODEUNIT* const this_instr = next_instr;\n") + out.emit(unused_guard) + if not inst.properties.no_save_ip: + out.emit(f"frame->instr_ptr = next_instr;\n") + + out.emit(f"next_instr += {inst.size};\n") + out.emit(f"INSTRUCTION_STATS({name});\n") + if inst.is_target: + out.emit(f"PREDICTED_TRACING_{name}:;\n") + out.emit(f"_Py_CODEUNIT* const this_instr = next_instr - {inst.size};\n") + out.emit(unused_guard) + # This is required so that the predicted ops reflect the correct opcode. + out.emit(f"opcode = {name};\n") + out.emit(f"PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable);\n") + out.emit(f"(void)old_code;\n") + out.emit(f"PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj);\n") + out.emit(f"(void)old_func;\n") + out.emit(f"int _jump_taken = false;\n") + out.emit(f"(void)_jump_taken;\n") + out.emit(f"int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0;\n") + out.emit(f"(void)(_old_stack_level);\n") + if inst.family is not None: + out.emit( + f"static_assert({inst.family.size} == {inst.size-1}" + ', "incorrect cache size");\n' + ) + declare_variables(inst, out) + offset = 1 # The instruction itself + stack = Stack() + for part in inst.parts: + # Only emit braces if more than one uop + insert_braces = len([p for p in inst.parts if isinstance(p, Uop)]) > 1 + reachable, offset, stack = write_uop(part, emitter, offset, stack, inst, insert_braces) + out.start_line() + if reachable: # type: ignore[possibly-undefined] + stack.flush(out) + out.emit(f"TRACING_DISPATCH();\n") + out.start_line() + out.emit("}") + out.emit("\n") + def generate_tracer_cases( analysis: Analysis, out: CWriter ) -> None: out.emit(f"#ifdef _Py_TIER2 /* BEGIN TRACING INSTRUCTIONS */\n") - generate_tier1_cases(analysis, out, TracerEmitter(out, analysis.labels), is_tracing=True) + generate_tier1_tracer_cases(analysis, out, TracerEmitter(out, analysis.labels)) out.emit(f"#endif /* END TRACING INSTRUCTIONS */\n") def generate_tracer( From a7fcf249b8de30c1fb9fa9acaa9c2a772ba3fdae Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 23 Oct 2025 01:53:47 +0100 Subject: [PATCH 085/190] Close loops properly, don't trace into nested loops --- Python/optimizer.c | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/Python/optimizer.c b/Python/optimizer.c index f77ff500e2f9d4..4d9e9e0ae4237f 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -564,7 +564,6 @@ _PyJit_translate_single_bytecode_to_trace( int jump_taken) { - int is_first_instr = tstate->interp->jit_state.close_loop_instr == this_instr; bool progress_needed = (tstate->interp->jit_state.initial_chain_depth % MAX_CHAIN_DEPTH) == 0;; _PyBloomFilter *dependencies = &tstate->interp->jit_state.dependencies; _Py_BloomFilter_Add(dependencies, old_code); @@ -685,12 +684,6 @@ _PyJit_translate_single_bytecode_to_trace( assert(!OPCODE_HAS_DEOPT(opcode)); } - // Loop back to the start - if (is_first_instr && tstate->interp->jit_state.code_curr_size > 5) { - ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0, 0); - goto done; - } - if (OPCODE_HAS_EXIT(opcode)) { // Make space for side exit and final _EXIT_TRACE: max_length--; @@ -725,7 +718,21 @@ _PyJit_translate_single_bytecode_to_trace( ADD_TO_TRACE(_CHECK_PERIODIC, 0, 0, target); _Py_FALLTHROUGH; case JUMP_BACKWARD_NO_INTERRUPT: + { + if ((next_instr != tstate->interp->jit_state.close_loop_instr) && + (next_instr != tstate->interp->jit_state.insert_exec_instr) && + tstate->interp->jit_state.code_curr_size > 5) { + // We encountered a JUMP_BACKWARD but not to the top of our own loop. + // We don't want to continue tracing as we might get stuck in the + // inner loop. Instead, end the trace where the executor of the + // inner loop might start and let the traces rejoin. + OPT_STAT_INC(inner_loop); + ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); + DPRINTF(2, "JUMP_BACKWARD not to top ends trace %p %p %p\n", next_instr, tstate->interp->jit_state.close_loop_instr, tstate->interp->jit_state.insert_exec_instr); + goto done; + } break; + } case RESUME: case RESUME_CHECK: @@ -854,6 +861,12 @@ _PyJit_translate_single_bytecode_to_trace( if (needs_guard_ip) { ADD_TO_TRACE(_GUARD_IP, 0, (uintptr_t)next_instr, 0); } + // Loop back to the start + int is_first_instr = tstate->interp->jit_state.close_loop_instr == next_instr || tstate->interp->jit_state.insert_exec_instr == next_instr; + if (is_first_instr && tstate->interp->jit_state.code_curr_size > 5) { + ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0, 0); + goto done; + } tstate->interp->jit_state.code_curr_size = trace_length; tstate->interp->jit_state.code_max_size = max_length; return 1; From eb733782f1dbd8588ed9bf2062e4448830fff9b0 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 23 Oct 2025 02:02:55 +0100 Subject: [PATCH 086/190] fix test --- Lib/test/test_sys.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 98dd466849e555..c5e50dffb13938 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -2255,7 +2255,7 @@ def frame_3_jit() -> None: # 1 extra iteration for tracing. for i in range(_testinternalcapi.TIER2_THRESHOLD + 2): # Careful, doing this in the reverse order breaks tracing: - expected = {enabled} and i >= _testinternalcapi.TIER2_THRESHOLD + 1 + expected = {enabled} and i >= _testinternalcapi.TIER2_THRESHOLD assert sys._jit.is_active() is expected frame_2_jit(expected) assert sys._jit.is_active() is expected From 2b5fe3a12aa4c6cea2d6f3d0301f53983c0a4dd7 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 23 Oct 2025 19:03:20 +0100 Subject: [PATCH 087/190] debug changes --- Python/ceval.c | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index 3ca8378294998c..ada9a9fd736c96 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1222,27 +1222,20 @@ _PyTier2Interpreter( uopcode = next_uop->opcode; #ifdef Py_DEBUG if (frame->lltrace >= 4) { - if (next_uop->opcode != _YIELD_VALUE && - next_uop->opcode != _FOR_ITER_GEN_FRAME && - next_uop->opcode != _PUSH_FRAME && - next_uop->opcode != _PY_FRAME_KW && - next_uop->opcode != _SAVE_RETURN_OFFSET && - next_uop->opcode != _SAVE_RETURN_OFFSET) { - if (next_uop->opcode != _START_EXECUTOR) { - if (next_uop->format == UOP_FORMAT_TARGET) { - _Py_CODEUNIT *aim = _PyFrame_GetBytecode(frame) + next_uop->target; + if (next_uop->opcode != _START_EXECUTOR) { + if (next_uop->format == UOP_FORMAT_TARGET) { + _Py_CODEUNIT *aim = _PyFrame_GetBytecode(frame) + next_uop->target; + printf(" aim=[%s]\n", _PyOpcode_OpName[aim->op.code]); + } + else if (next_uop->format == UOP_FORMAT_JUMP) { + _PyUOpInstruction *aim_uop = current_executor->trace + next_uop->jump_target; + if (aim_uop->format == UOP_FORMAT_TARGET) { + _Py_CODEUNIT *aim = _PyFrame_GetBytecode(frame) + aim_uop->target; printf(" aim=[%s]\n", _PyOpcode_OpName[aim->op.code]); } - else if (next_uop->format == UOP_FORMAT_JUMP) { - _PyUOpInstruction *aim_uop = current_executor->trace + next_uop->jump_target; - if (aim_uop->format == UOP_FORMAT_TARGET) { - _Py_CODEUNIT *aim = _PyFrame_GetBytecode(frame) + aim_uop->target; - printf(" aim=[%s]\n", _PyOpcode_OpName[aim->op.code]); - } - } } - dump_stack(frame, stack_pointer); } + dump_stack(frame, stack_pointer); if (next_uop->opcode == _START_EXECUTOR) { printf("%4d uop: ", 0); } From 3385420364c8c6fe2e067e265a7252af76a6786c Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 23 Oct 2025 19:04:16 +0100 Subject: [PATCH 088/190] add comment to CI --- .github/workflows/jit.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 48b2e943e818c8..c91328cf01cc2d 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -57,6 +57,7 @@ jobs: fail-fast: false matrix: target: +# To re-enable later when we support thesee. # - i686-pc-windows-msvc/msvc # - x86_64-pc-windows-msvc/msvc # - aarch64-pc-windows-msvc/msvc @@ -70,6 +71,7 @@ jobs: llvm: - 19 include: +# To re-enable later when we support thesee. # - target: i686-pc-windows-msvc/msvc # architecture: Win32 # runner: windows-2022 From cedd7af91afc1b1f43d32b3419dfc6a70a75fd43 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 24 Oct 2025 00:13:48 +0100 Subject: [PATCH 089/190] Rewrite the tracing JIT to use a common opcode handler --- Include/internal/pycore_interp_structs.h | 11 +- Include/internal/pycore_magic_number.h | 3 +- Include/internal/pycore_opcode_metadata.h | 9 +- Include/internal/pycore_optimizer.h | 18 +- Include/opcode_ids.h | 33 +- Lib/_opcode_metadata.py | 33 +- Lib/test/test_sys.py | 2 +- Makefile.pre.in | 9 +- Python/bytecodes.c | 36 +- Python/ceval.c | 14 - Python/ceval_macros.h | 81 +- Python/executor_cases.c.h | 2 +- Python/generated_cases.c.h | 45 +- Python/generated_tracer_cases.c.h | 14556 ------------------- Python/opcode_targets.h | 1134 +- Python/optimizer.c | 128 +- Tools/cases_generator/generators_common.py | 14 - Tools/cases_generator/target_generator.py | 7 +- Tools/cases_generator/tier1_generator.py | 23 + Tools/cases_generator/tracer_generator.py | 222 - 20 files changed, 712 insertions(+), 15668 deletions(-) delete mode 100644 Python/generated_tracer_cases.c.h delete mode 100644 Tools/cases_generator/tracer_generator.py diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index ac1bcdf74ae417..c2cddfe47d3f73 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -759,18 +759,23 @@ typedef _Py_CODEUNIT *(*_PyJitEntryFuncPtr)(struct _PyExecutorObject *exec, _PyI typedef struct _PyJitTracerState { bool dependencies_still_valid; + bool do_not_specialize; + bool dynamic_jump_taken; int code_max_size; int code_curr_size; int initial_stack_depth; int initial_chain_depth; + int prev_instr_oparg; + int prev_instr_stacklevel; _PyUOpInstruction *code_buffer; _Py_CODEUNIT *insert_exec_instr; _Py_CODEUNIT *close_loop_instr; - _Py_CODEUNIT *last_specialized_instr; PyCodeObject *initial_code; // Strong PyFunctionObject *initial_func; // Strong - struct _PyExitData *previous_exit; - _PyInterpreterFrame *current_frame; + _Py_CODEUNIT *prev_instr; + PyCodeObject *prev_instr_code; // Strong + struct _PyExitData *prev_exit; + _PyInterpreterFrame *prev_instr_frame; _PyBloomFilter dependencies; } _PyJitTracerState; diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index 7ec7bd1c695516..09c6b1f5b39274 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -286,6 +286,7 @@ Known values: Python 3.15a1 3653 (Fix handling of opcodes that may leave operands on the stack when optimizing LOAD_FAST) Python 3.15a1 3654 (Fix missing exception handlers in logical expression) Python 3.15a1 3655 (Fix miscompilation of some module-level annotations) + Python 3.15a2 3656 (Add RECORD_PREVIOUS_INST for a tracing JIT) Python 3.16 will start with 3700 @@ -299,7 +300,7 @@ PC/launcher.c must also be updated. */ -#define PYC_MAGIC_NUMBER 3655 +#define PYC_MAGIC_NUMBER 3656 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index e8e0c59874a546..288396f34fc3d4 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -412,6 +412,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 0; case RAISE_VARARGS: return oparg; + case RECORD_PREVIOUS_INST: + return 0; case RERAISE: return 1 + oparg; case RESERVED: @@ -895,6 +897,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case RAISE_VARARGS: return 0; + case RECORD_PREVIOUS_INST: + return 0; case RERAISE: return oparg; case RESERVED: @@ -1249,6 +1253,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [PUSH_EXC_INFO] = { true, INSTR_FMT_IX, 0 }, [PUSH_NULL] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, + [RECORD_PREVIOUS_INST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG }, [RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [RESERVED] = { true, INSTR_FMT_IX, 0 }, [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, @@ -1695,6 +1700,7 @@ const char *_PyOpcode_OpName[267] = { [PUSH_EXC_INFO] = "PUSH_EXC_INFO", [PUSH_NULL] = "PUSH_NULL", [RAISE_VARARGS] = "RAISE_VARARGS", + [RECORD_PREVIOUS_INST] = "RECORD_PREVIOUS_INST", [RERAISE] = "RERAISE", [RESERVED] = "RESERVED", [RESUME] = "RESUME", @@ -1805,7 +1811,6 @@ const uint8_t _PyOpcode_NeedsGuardIp[256] = { extern const uint8_t _PyOpcode_Deopt[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Deopt[256] = { - [121] = 121, [122] = 122, [123] = 123, [124] = 124, @@ -2017,6 +2022,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [PUSH_EXC_INFO] = PUSH_EXC_INFO, [PUSH_NULL] = PUSH_NULL, [RAISE_VARARGS] = RAISE_VARARGS, + [RECORD_PREVIOUS_INST] = RECORD_PREVIOUS_INST, [RERAISE] = RERAISE, [RESERVED] = RESERVED, [RESUME] = RESUME, @@ -2066,7 +2072,6 @@ const uint8_t _PyOpcode_Deopt[256] = { #endif // NEED_OPCODE_METADATA #define EXTRA_CASES \ - case 121: \ case 122: \ case 123: \ case 124: \ diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 9c3aaf1edaf550..ca46249bde98df 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -358,22 +358,12 @@ PyAPI_FUNC(int) _PyDumpExecutors(FILE *out); extern void _Py_ClearExecutorDeletionList(PyInterpreterState *interp); #endif -int -_PyJit_translate_single_bytecode_to_trace( - PyThreadState *tstate, - _PyInterpreterFrame *frame, - _Py_CODEUNIT *this_instr, - _Py_CODEUNIT *next_instr, - PyCodeObject *code, - PyFunctionObject *func, - int old_stack_level, - int opcode, - int oparg, - int jump_taken); +int _PyJit_translate_single_bytecode_to_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr); void -_PyJit_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *insert_exec_instr, - _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit); +_PyJit_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, + _Py_CODEUNIT *curr_instr, _Py_CODEUNIT *insert_exec_instr, + _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit, int oparg); void _PyJit_FinalizeTracing(PyThreadState *tstate); diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 1d5c74adefcd35..1c7468456dbd02 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -115,22 +115,23 @@ extern "C" { #define POP_JUMP_IF_NOT_NONE 102 #define POP_JUMP_IF_TRUE 103 #define RAISE_VARARGS 104 -#define RERAISE 105 -#define SEND 106 -#define SET_ADD 107 -#define SET_FUNCTION_ATTRIBUTE 108 -#define SET_UPDATE 109 -#define STORE_ATTR 110 -#define STORE_DEREF 111 -#define STORE_FAST 112 -#define STORE_FAST_LOAD_FAST 113 -#define STORE_FAST_STORE_FAST 114 -#define STORE_GLOBAL 115 -#define STORE_NAME 116 -#define SWAP 117 -#define UNPACK_EX 118 -#define UNPACK_SEQUENCE 119 -#define YIELD_VALUE 120 +#define RECORD_PREVIOUS_INST 105 +#define RERAISE 106 +#define SEND 107 +#define SET_ADD 108 +#define SET_FUNCTION_ATTRIBUTE 109 +#define SET_UPDATE 110 +#define STORE_ATTR 111 +#define STORE_DEREF 112 +#define STORE_FAST 113 +#define STORE_FAST_LOAD_FAST 114 +#define STORE_FAST_STORE_FAST 115 +#define STORE_GLOBAL 116 +#define STORE_NAME 117 +#define SWAP 118 +#define UNPACK_EX 119 +#define UNPACK_SEQUENCE 120 +#define YIELD_VALUE 121 #define RESUME 128 #define BINARY_OP_ADD_FLOAT 129 #define BINARY_OP_ADD_INT 130 diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index f168d169a32948..b6bce074f8e579 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -312,22 +312,23 @@ 'POP_JUMP_IF_NOT_NONE': 102, 'POP_JUMP_IF_TRUE': 103, 'RAISE_VARARGS': 104, - 'RERAISE': 105, - 'SEND': 106, - 'SET_ADD': 107, - 'SET_FUNCTION_ATTRIBUTE': 108, - 'SET_UPDATE': 109, - 'STORE_ATTR': 110, - 'STORE_DEREF': 111, - 'STORE_FAST': 112, - 'STORE_FAST_LOAD_FAST': 113, - 'STORE_FAST_STORE_FAST': 114, - 'STORE_GLOBAL': 115, - 'STORE_NAME': 116, - 'SWAP': 117, - 'UNPACK_EX': 118, - 'UNPACK_SEQUENCE': 119, - 'YIELD_VALUE': 120, + 'RECORD_PREVIOUS_INST': 105, + 'RERAISE': 106, + 'SEND': 107, + 'SET_ADD': 108, + 'SET_FUNCTION_ATTRIBUTE': 109, + 'SET_UPDATE': 110, + 'STORE_ATTR': 111, + 'STORE_DEREF': 112, + 'STORE_FAST': 113, + 'STORE_FAST_LOAD_FAST': 114, + 'STORE_FAST_STORE_FAST': 115, + 'STORE_GLOBAL': 116, + 'STORE_NAME': 117, + 'SWAP': 118, + 'UNPACK_EX': 119, + 'UNPACK_SEQUENCE': 120, + 'YIELD_VALUE': 121, 'INSTRUMENTED_END_FOR': 234, 'INSTRUMENTED_POP_ITER': 235, 'INSTRUMENTED_END_SEND': 236, diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index c5e50dffb13938..98dd466849e555 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -2255,7 +2255,7 @@ def frame_3_jit() -> None: # 1 extra iteration for tracing. for i in range(_testinternalcapi.TIER2_THRESHOLD + 2): # Careful, doing this in the reverse order breaks tracing: - expected = {enabled} and i >= _testinternalcapi.TIER2_THRESHOLD + expected = {enabled} and i >= _testinternalcapi.TIER2_THRESHOLD + 1 assert sys._jit.is_active() is expected frame_2_jit(expected) assert sys._jit.is_active() is expected diff --git a/Makefile.pre.in b/Makefile.pre.in index 2db96b74475705..19423c11545c19 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2141,7 +2141,7 @@ Objects/mimalloc/page.o: $(srcdir)/Objects/mimalloc/page-queue.c regen-cases: \ regen-opcode-ids regen-opcode-targets regen-uop-ids regen-opcode-metadata-py \ regen-generated-cases regen-executor-cases regen-optimizer-cases \ - regen-opcode-metadata regen-uop-metadata regen-tracer-cases + regen-opcode-metadata regen-uop-metadata .PHONY: regen-opcode-ids regen-opcode-ids: @@ -2187,13 +2187,6 @@ regen-optimizer-cases: $(srcdir)/Python/bytecodes.c $(UPDATE_FILE) $(srcdir)/Python/optimizer_cases.c.h $(srcdir)/Python/optimizer_cases.c.h.new -.PHONY: regen-tracer-cases -regen-tracer-cases: - $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/tracer_generator.py \ - -o $(srcdir)/Python/generated_tracer_cases.c.h.new $(srcdir)/Python/bytecodes.c - $(UPDATE_FILE) $(srcdir)/Python/generated_tracer_cases.c.h $(srcdir)/Python/generated_tracer_cases.c.h.new - - .PHONY: regen-opcode-metadata regen-opcode-metadata: $(PYTHON_FOR_REGEN) $(srcdir)/Tools/cases_generator/opcode_metadata_generator.py \ diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 388da83dbc24bf..73ee9fdb8fcca3 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -45,6 +45,8 @@ #define USE_COMPUTED_GOTOS 0 #include "ceval_macros.h" +#include "ceval_macros.h" +#include "ceval_macros.h" #include "../Include/internal/pycore_code.h" #include "../Include/internal/pycore_stackref.h" @@ -2985,14 +2987,9 @@ dummy_func( oparg >>= 8; insert_exec_at--; } - _PyJit_InitializeTracing(tstate, frame, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL); + _PyJit_InitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, oparg); ENTER_TRACING(); } - int _jump_taken = false; - PyCodeObject *old_code = _PyFrame_GetCode(frame); - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - int _old_stack_level = 0; - TRACING_DISPATCH(); } else { ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); @@ -3057,9 +3054,6 @@ dummy_func( } assert(executor != tstate->interp->cold_executor); tstate->jit_exit = NULL; - #if TRACING_JIT - RECORD_TRACE_NO_DISPATCH(); - #endif TIER1_TO_TIER2(executor); #else Py_FatalError("ENTER_EXECUTOR is not supported in this build"); @@ -5224,6 +5218,25 @@ dummy_func( Py_FatalError("Executing RESERVED instruction."); } + tier1 no_save_ip inst(RECORD_PREVIOUS_INST, (--)) { + assert(IS_JIT_TRACING()); + int opcode = next_instr->op.code; + int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr); + if (full) { + LEAVE_TRACING(); + int err = bail_tracing_and_jit(tstate, frame); + ERROR_IF(err < 0); + DISPATCH_GOTO_NON_TRACING(); + } + tstate->interp->jit_state.do_not_specialize = false; + PyCodeObject *prev_code = (PyCodeObject *)Py_NewRef(_PyFrame_GetCode(frame)); + Py_SETREF(tstate->interp->jit_state.prev_instr_code, prev_code); + tstate->interp->jit_state.prev_instr = next_instr; + tstate->interp->jit_state.prev_instr_frame = frame; + tstate->interp->jit_state.prev_instr_oparg = oparg; + tstate->interp->jit_state.prev_instr_stacklevel = STACK_LEVEL(); + DISPATCH_GOTO_NON_TRACING(); + } ///////// Tier-2 only opcodes ///////// op (_GUARD_IS_TRUE_POP, (flag -- )) { @@ -5456,7 +5469,10 @@ dummy_func( _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); assert(tstate->current_executor == (PyObject *)previous_executor); int chain_depth = previous_executor->vm_data.chain_depth + 1; - _PyJit_InitializeTracing(tstate, frame, target, target, STACK_LEVEL(), chain_depth, exit); + // Note: it's safe to use target->op.arg here instead of the oparg given by EXTENDED_ARG. + // The invariant in the optimizer is the deopt target always points back to the first EXTENDED_ARG. + // So setting it to anything else is wrong. + _PyJit_InitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, target->op.arg); exit->temperature = initial_temperature_backoff_counter(); GOTO_TIER_ONE(target, 1); } diff --git a/Python/ceval.c b/Python/ceval.c index ada9a9fd736c96..f4bf7b244dc794 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -984,17 +984,6 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch) } } -#if _Py_TIER2 -// 1 for trace full, 0 for successful write. -static inline int -add_to_code_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, PyCodeObject *old_code, PyFunctionObject *old_func, int old_stack_level, _Py_CODEUNIT *this_instr, _Py_CODEUNIT *next_instr, int opcode, int oparg, int jump_taken) -{ - assert(frame != NULL); - assert(tstate->interp->jit_state.code_curr_size < UOP_MAX_TRACE_LENGTH); - return !_PyJit_translate_single_bytecode_to_trace(tstate, frame, this_instr, next_instr, old_code, old_func, old_stack_level, opcode, oparg, jump_taken); -} - - // 0 for success, -1 for error. static int bail_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame) @@ -1007,7 +996,6 @@ bail_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame) _PyJit_FinalizeTracing(tstate); return err; } -#endif /* _PyEval_EvalFrameDefault is too large to optimize for speed with PGO on MSVC. */ @@ -1025,7 +1013,6 @@ bail_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame) #if _Py_TAIL_CALL_INTERP #include "opcode_targets.h" #include "generated_cases.c.h" -#include "generated_tracer_cases.c.h" #endif #if (defined(__GNUC__) && __GNUC__ >= 10 && !defined(__clang__)) && defined(__x86_64__) @@ -1154,7 +1141,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #else goto start_frame; # include "generated_cases.c.h" - #include "generated_tracer_cases.c.h" #endif diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index e19a614f28dec6..bfc84e8c66230b 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -94,6 +94,10 @@ do { \ Py_MUSTTAIL return (((py_tail_call_funcptr *)instruction_funcptr_table)[opcode])(TAIL_CALL_ARGS); \ } while (0) +# define DISPATCH_GOTO_NON_TRACING() \ + do { \ + Py_MUSTTAIL return (((py_tail_call_funcptr *)DISPATCH_TABLE)[opcode])(TAIL_CALL_ARGS); \ + } while (0) # define JUMP_TO_LABEL(name) \ do { \ Py_MUSTTAIL return (_TAIL_CALL_##name)(TAIL_CALL_ARGS); \ @@ -117,7 +121,7 @@ # define TARGET(op) TARGET_##op: # define TRACING_TARGET(op) TARGET_TRACING_##op: # define DISPATCH_GOTO() goto *opcode_targets[opcode] - +# define DISPATCH_GOTO_NON_TRACING() goto *DISPATCH_TABLE[opcode]; # define JUMP_TO_LABEL(name) goto name; # define JUMP_TO_PREDICTED(name) goto PREDICTED_##name; # define LABEL(name) name: @@ -129,35 +133,14 @@ # define LABEL(name) name: #endif -#define TRACING_JUMP_TO_LABEL(label) \ - RECORD_DYNAMIC_JUMP_TAKEN() \ - RECORD_TRACE_NO_DISPATCH() \ - assert(!IS_JIT_TRACING()); \ - JUMP_TO_LABEL(label); - #if _Py_TAIL_CALL_INTERP || USE_COMPUTED_GOTOS # define IS_JIT_TRACING() (DISPATCH_TABLE_VAR == TRACING_DISPATCH_TABLE) -// tstate->interp->jit_state.last_specialized_instr != this_instr is required to not get stuck in infinite -// specialization loops due to specialization failure. -# define IS_JIT_TRACING_MAKING_PROGRESS() (IS_JIT_TRACING() && tstate->interp->jit_state.last_specialized_instr != this_instr) +// Required to not get stuck in infinite pecialization loops due to specialization failure. +# define IS_JIT_TRACING_MAKING_PROGRESS() (IS_JIT_TRACING() && !tstate->interp->jit_state.do_not_specialize) # define ENTER_TRACING() \ DISPATCH_TABLE_VAR = TRACING_DISPATCH_TABLE; # define LEAVE_TRACING() \ DISPATCH_TABLE_VAR = DISPATCH_TABLE; -# define RECORD_TRACE_NO_DISPATCH() do { \ - int err = 0; \ - _PyFrame_SetStackPointer(frame, stack_pointer); \ - /* We need to check once more here in case it swapped out halfway. */ \ - if (IS_JIT_TRACING()) { \ - int full = add_to_code_trace(tstate, frame, old_code, old_func, _old_stack_level, this_instr, next_instr, opcode, oparg, _jump_taken); \ - if (full) { \ - LEAVE_TRACING(); \ - err = bail_tracing_and_jit(tstate, frame); \ - } \ - } \ - stack_pointer = _PyFrame_GetStackPointer(frame); \ - if (err < 0) { JUMP_TO_LABEL(error); } \ - } while (0); #endif @@ -197,21 +180,22 @@ do { \ DISPATCH_GOTO(); \ } +#define DISPATCH_NON_TRACING() \ + { \ + assert(frame->stackpointer == NULL); \ + NEXTOPARG(); \ + PRE_DISPATCH_GOTO(); \ + DISPATCH_GOTO_NON_TRACING(); \ + } + +#if _Py_TIER2 #define DISPATCH_SAME_OPARG() \ { \ + tstate->interp->jit_state.do_not_specialize = true; \ opcode = next_instr->op.code; \ PRE_DISPATCH_GOTO(); \ - DISPATCH_GOTO(); \ + DISPATCH_GOTO_NON_TRACING(); \ } - -#define TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() \ -{ \ - tstate->interp->jit_state.last_specialized_instr = this_instr; \ - opcode = next_instr->op.code; \ - PRE_DISPATCH_GOTO(); \ - DISPATCH_GOTO(); \ -} - #define DISPATCH_INLINED(NEW_FRAME) \ do { \ assert(tstate->interp->eval_frame == NULL); \ @@ -221,21 +205,24 @@ do { \ CALL_STAT_INC(inlined_py_calls); \ JUMP_TO_LABEL(start_frame); \ } while (0) - -#define TRACING_DISPATCH_INLINED(NEW_FRAME) \ - tstate->interp->jit_state.last_specialized_instr = this_instr; \ - RECORD_TRACE_NO_DISPATCH(); \ - DISPATCH_INLINED(NEW_FRAME); - -#define TRACING_DISPATCH() \ +#else +#define DISPATCH_SAME_OPARG() \ { \ - tstate->interp->jit_state.last_specialized_instr = this_instr; \ - assert(frame->stackpointer == NULL); \ - RECORD_TRACE_NO_DISPATCH(); \ - NEXTOPARG(); \ + opcode = next_instr->op.code; \ PRE_DISPATCH_GOTO(); \ DISPATCH_GOTO(); \ } +#define DISPATCH_INLINED(NEW_FRAME) \ + do { \ + assert(tstate->interp->eval_frame == NULL); \ + _PyFrame_SetStackPointer(frame, stack_pointer); \ + assert((NEW_FRAME)->previous == frame); \ + frame = tstate->current_frame = (NEW_FRAME); \ + CALL_STAT_INC(inlined_py_calls); \ + JUMP_TO_LABEL(start_frame); \ + } while (0) +#endif + /* Tuple access macros */ @@ -368,7 +355,7 @@ GETITEM(PyObject *v, Py_ssize_t i) { #define RECORD_BRANCH_TAKEN(bitset, flag) #endif -#define RECORD_DYNAMIC_JUMP_TAKEN() _jump_taken = 1; +#define RECORD_DYNAMIC_JUMP_TAKEN() tstate->interp->jit_state.dynamic_jump_taken = true; #define UNBOUNDLOCAL_ERROR_MSG \ "cannot access local variable '%s' where it is not associated with a value" @@ -426,6 +413,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer) #define TIER1_TO_TIER2(EXECUTOR) \ do { \ + LEAVE_TRACING(); \ OPT_STAT_INC(traces_executed); \ next_instr = _Py_jit_entry((EXECUTOR), frame, stack_pointer, tstate); \ frame = tstate->current_frame; \ @@ -440,6 +428,7 @@ do { \ assert(next_instr->op.code != ENTER_EXECUTOR); \ assert(tstate->interp->jit_state.code_curr_size == 2); \ ENTER_TRACING(); \ + DISPATCH_NON_TRACING(); \ } \ DISPATCH(); \ } while (0) diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 72927d8378c3bd..f839b9579ef38c 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7501,7 +7501,7 @@ _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); assert(tstate->current_executor == (PyObject *)previous_executor); int chain_depth = previous_executor->vm_data.chain_depth + 1; - _PyJit_InitializeTracing(tstate, frame, target, target, STACK_LEVEL(), chain_depth, exit); + _PyJit_InitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, target->op.arg); exit->temperature = initial_temperature_backoff_counter(); GOTO_TIER_ONE(target, 1); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 4fb61a06cdfe04..516d42a3972d76 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5493,9 +5493,6 @@ } assert(executor != tstate->interp->cold_executor); tstate->jit_exit = NULL; - #if TRACING_JIT - RECORD_TRACE_NO_DISPATCH(); - #endif TIER1_TO_TIER2(executor); #else Py_FatalError("ENTER_EXECUTOR is not supported in this build"); @@ -5656,6 +5653,7 @@ JUMP_TO_LABEL(error); } JUMPBY(oparg + 1); + RECORD_DYNAMIC_JUMP_TAKEN(); stack_pointer[-1] = null_or_index; DISPATCH(); } @@ -5784,6 +5782,7 @@ if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { null_or_index = PyStackRef_TagInt(-1); JUMPBY(oparg + 1); + RECORD_DYNAMIC_JUMP_TAKEN(); stack_pointer[-1] = null_or_index; DISPATCH(); } @@ -5864,6 +5863,7 @@ STAT_INC(FOR_ITER, hit); if (r->len <= 0) { JUMPBY(oparg + 1); + RECORD_DYNAMIC_JUMP_TAKEN(); DISPATCH(); } } @@ -5926,6 +5926,7 @@ if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { null_or_index = PyStackRef_TagInt(-1); JUMPBY(oparg + 1); + RECORD_DYNAMIC_JUMP_TAKEN(); stack_pointer[-1] = null_or_index; DISPATCH(); } @@ -7676,14 +7677,9 @@ oparg >>= 8; insert_exec_at--; } - _PyJit_InitializeTracing(tstate, frame, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL); + _PyJit_InitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, oparg); ENTER_TRACING(); } - int _jump_taken = false; - PyCodeObject *old_code = _PyFrame_GetCode(frame); - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - int _old_stack_level = 0; - TRACING_DISPATCH(); } else { ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); @@ -10200,6 +10196,36 @@ JUMP_TO_LABEL(error); } + + TARGET(RECORD_PREVIOUS_INST) { + INSTRUCTION_STATS(RECORD_PREVIOUS_INST); + assert(IS_JIT_TRACING()); + int opcode = next_instr->op.code; + _PyFrame_SetStackPointer(frame, stack_pointer); + int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (full) { + LEAVE_TRACING(); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = bail_tracing_and_jit(tstate, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + DISPATCH_GOTO_NON_TRACING(); + } + tstate->interp->jit_state.do_not_specialize = false; + PyCodeObject *prev_code = (PyCodeObject *)Py_NewRef(_PyFrame_GetCode(frame)); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_SETREF(tstate->interp->jit_state.prev_instr_code, prev_code); + stack_pointer = _PyFrame_GetStackPointer(frame); + tstate->interp->jit_state.prev_instr = next_instr; + tstate->interp->jit_state.prev_instr_frame = frame; + tstate->interp->jit_state.prev_instr_oparg = oparg; + tstate->interp->jit_state.prev_instr_stacklevel = STACK_LEVEL(); + DISPATCH_GOTO_NON_TRACING(); + } + TARGET(RERAISE) { #if _Py_TAIL_CALL_INTERP int opcode = RERAISE; @@ -10517,6 +10543,7 @@ if (err == 0) { assert(retval_o != NULL); JUMPBY(oparg); + RECORD_DYNAMIC_JUMP_TAKEN(); } else { stack_pointer += -1; diff --git a/Python/generated_tracer_cases.c.h b/Python/generated_tracer_cases.c.h deleted file mode 100644 index 932a57a22e5110..00000000000000 --- a/Python/generated_tracer_cases.c.h +++ /dev/null @@ -1,14556 +0,0 @@ -// This file is generated by Tools/cases_generator/tracer_generator.py -// from: -// Python/bytecodes.c -// Do not edit! - #define TIER_ONE 1 - #define TRACING_JIT 1 - #ifdef _Py_TIER2 /* BEGIN TRACING INSTRUCTIONS */ - - - TRACING_TARGET(BINARY_OP) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP); - PREDICTED_TRACING_BINARY_OP:; - _Py_CODEUNIT* const this_instr = next_instr - 6; - (void)this_instr; - opcode = BINARY_OP; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef lhs; - _PyStackRef rhs; - _PyStackRef res; - // _SPECIALIZE_BINARY_OP - { - rhs = stack_pointer[-1]; - lhs = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() - } - OPCODE_DEFERRED_INC(BINARY_OP); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - assert(NB_ADD <= oparg); - assert(oparg <= NB_OPARG_LAST); - } - /* Skip 4 cache entries */ - // _BINARY_OP - { - PyObject *lhs_o = PyStackRef_AsPyObjectBorrow(lhs); - PyObject *rhs_o = PyStackRef_AsPyObjectBorrow(rhs); - assert(_PyEval_BinaryOps[oparg]); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyEval_BinaryOps[oparg](lhs_o, rhs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = lhs; - lhs = res; - stack_pointer[-2] = lhs; - PyStackRef_CLOSE(tmp); - tmp = rhs; - rhs = PyStackRef_NULL; - stack_pointer[-1] = rhs; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_ADD_FLOAT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_ADD_FLOAT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_ADD_FLOAT); - opcode = BINARY_OP_ADD_FLOAT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_TOS_FLOAT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyFloat_CheckExact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_FLOAT - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!PyFloat_CheckExact(left_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_ADD_FLOAT - { - right = value; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval + - ((PyFloatObject *)right_o)->ob_fval; - res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (PyStackRef_IsNull(res)) { - TRACING_JUMP_TO_LABEL(pop_2_error); - } - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_ADD_INT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_ADD_INT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_ADD_INT); - opcode = BINARY_OP_ADD_INT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_TOS_INT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_INT - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!_PyLong_CheckExactAndCompact(left_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_ADD_INT - { - right = value; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); - STAT_INC(BINARY_OP, hit); - res = _PyCompactLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o); - if (PyStackRef_IsNull(res)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_ADD_UNICODE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_ADD_UNICODE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_ADD_UNICODE); - opcode = BINARY_OP_ADD_UNICODE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef nos; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_TOS_UNICODE - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyUnicode_CheckExact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_UNICODE - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyUnicode_CheckExact(o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_ADD_UNICODE - { - right = value; - left = nos; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyUnicode_CheckExact(left_o)); - assert(PyUnicode_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - PyObject *res_o = PyUnicode_Concat(left_o, right_o); - PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(pop_2_error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_EXTEND) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_EXTEND; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_EXTEND); - opcode = BINARY_OP_EXTEND; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - /* Skip 1 cache entry */ - // _GUARD_BINARY_OP_EXTEND - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *descr = read_obj(&this_instr[2].cache); - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; - assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); - assert(d && d->guard); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = d->guard(left_o, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!res) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip -4 cache entry */ - // _BINARY_OP_EXTEND - { - PyObject *descr = read_obj(&this_instr[2].cache); - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5); - _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr; - STAT_INC(BINARY_OP, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = d->action(left_o, right_o); - _PyStackRef tmp = right; - right = PyStackRef_NULL; - stack_pointer[-1] = right; - PyStackRef_CLOSE(tmp); - tmp = left; - left = PyStackRef_NULL; - stack_pointer[-2] = left; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_INPLACE_ADD_UNICODE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_INPLACE_ADD_UNICODE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_INPLACE_ADD_UNICODE); - opcode = BINARY_OP_INPLACE_ADD_UNICODE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef nos; - _PyStackRef left; - _PyStackRef right; - // _GUARD_TOS_UNICODE - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyUnicode_CheckExact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_UNICODE - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyUnicode_CheckExact(o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_INPLACE_ADD_UNICODE - { - right = value; - left = nos; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - assert(PyUnicode_CheckExact(left_o)); - assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(right))); - int next_oparg; - #if TIER_ONE - assert(next_instr->op.code == STORE_FAST); - next_oparg = next_instr->op.arg; - #else - next_oparg = (int)CURRENT_OPERAND0(); - #endif - _PyStackRef *target_local = &GETLOCAL(next_oparg); - assert(PyUnicode_CheckExact(left_o)); - if (PyStackRef_AsPyObjectBorrow(*target_local) != left_o) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - STAT_INC(BINARY_OP, hit); - assert(Py_REFCNT(left_o) >= 2 || !PyStackRef_IsHeapSafe(left)); - PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); - PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); - PyObject *right_o = PyStackRef_AsPyObjectSteal(right); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyUnicode_Append(&temp, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - *target_local = PyStackRef_FromPyObjectSteal(temp); - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(*target_local)) { - TRACING_JUMP_TO_LABEL(error); - } - #if TIER_ONE - - assert(next_instr->op.code == STORE_FAST); - SKIP_OVER(1); - #endif - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_MULTIPLY_FLOAT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_MULTIPLY_FLOAT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_MULTIPLY_FLOAT); - opcode = BINARY_OP_MULTIPLY_FLOAT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_TOS_FLOAT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyFloat_CheckExact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_FLOAT - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!PyFloat_CheckExact(left_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_MULTIPLY_FLOAT - { - right = value; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval * - ((PyFloatObject *)right_o)->ob_fval; - res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (PyStackRef_IsNull(res)) { - TRACING_JUMP_TO_LABEL(pop_2_error); - } - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_MULTIPLY_INT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_MULTIPLY_INT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_MULTIPLY_INT); - opcode = BINARY_OP_MULTIPLY_INT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_TOS_INT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_INT - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!_PyLong_CheckExactAndCompact(left_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_MULTIPLY_INT - { - right = value; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); - STAT_INC(BINARY_OP, hit); - res = _PyCompactLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o); - if (PyStackRef_IsNull(res)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_SUBSCR_DICT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_SUBSCR_DICT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBSCR_DICT); - opcode = BINARY_OP_SUBSCR_DICT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef nos; - _PyStackRef dict_st; - _PyStackRef sub_st; - _PyStackRef res; - // _GUARD_NOS_DICT - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyDict_CheckExact(o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_SUBSCR_DICT - { - sub_st = stack_pointer[-1]; - dict_st = nos; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - assert(PyDict_CheckExact(dict)); - STAT_INC(BINARY_OP, hit); - PyObject *res_o; - _PyFrame_SetStackPointer(frame, stack_pointer); - int rc = PyDict_GetItemRef(dict, sub, &res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (rc == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetKeyError(sub); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = sub_st; - sub_st = PyStackRef_NULL; - stack_pointer[-1] = sub_st; - PyStackRef_CLOSE(tmp); - tmp = dict_st; - dict_st = PyStackRef_NULL; - stack_pointer[-2] = dict_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (rc <= 0) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_SUBSCR_GETITEM) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_SUBSCR_GETITEM; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBSCR_GETITEM); - opcode = BINARY_OP_SUBSCR_GETITEM; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef container; - _PyStackRef getitem; - _PyStackRef sub; - _PyStackRef new_frame; - /* Skip 5 cache entries */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _BINARY_OP_SUBSCR_CHECK_FUNC - { - container = stack_pointer[-2]; - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); - if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; - PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem); - if (getitem_o == NULL) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - assert(PyFunction_Check(getitem_o)); - uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version); - if (((PyFunctionObject *)getitem_o)->func_version != cached_version) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); - assert(code->co_argcount == 2); - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - getitem = PyStackRef_FromPyObjectNew(getitem_o); - STAT_INC(BINARY_OP, hit); - } - // _BINARY_OP_SUBSCR_INIT_CALL - { - sub = stack_pointer[-1]; - _PyInterpreterFrame* pushed_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); - pushed_frame->localsplus[0] = container; - pushed_frame->localsplus[1] = sub; - frame->return_offset = 6u ; - new_frame = PyStackRef_Wrap(pushed_frame); - } - // _PUSH_FRAME - { - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_SUBSCR_LIST_INT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_SUBSCR_LIST_INT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_INT); - opcode = BINARY_OP_SUBSCR_LIST_INT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef nos; - _PyStackRef list_st; - _PyStackRef sub_st; - _PyStackRef res; - // _GUARD_TOS_INT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_LIST - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyList_CheckExact(o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_SUBSCR_LIST_INT - { - sub_st = value; - list_st = nos; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - assert(PyLong_CheckExact(sub)); - assert(PyList_CheckExact(list)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - #ifdef Py_GIL_DISABLED - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - STAT_INC(BINARY_OP, hit); - res = PyStackRef_FromPyObjectSteal(res_o); - #else - if (index >= PyList_GET_SIZE(list)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - STAT_INC(BINARY_OP, hit); - PyObject *res_o = PyList_GET_ITEM(list, index); - assert(res_o != NULL); - res = PyStackRef_FromPyObjectNew(res_o); - #endif - STAT_INC(BINARY_OP, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = list_st; - list_st = res; - stack_pointer[-2] = list_st; - PyStackRef_CLOSE(tmp); - tmp = sub_st; - sub_st = PyStackRef_NULL; - stack_pointer[-1] = sub_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_SUBSCR_LIST_SLICE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_SUBSCR_LIST_SLICE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBSCR_LIST_SLICE); - opcode = BINARY_OP_SUBSCR_LIST_SLICE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef tos; - _PyStackRef nos; - _PyStackRef list_st; - _PyStackRef sub_st; - _PyStackRef res; - // _GUARD_TOS_SLICE - { - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PySlice_Check(o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_LIST - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyList_CheckExact(o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_SUBSCR_LIST_SLICE - { - sub_st = tos; - list_st = nos; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - assert(PySlice_Check(sub)); - assert(PyList_CheckExact(list)); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyList_SliceSubscript(list, sub); - stack_pointer = _PyFrame_GetStackPointer(frame); - STAT_INC(BINARY_OP, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = sub_st; - sub_st = PyStackRef_NULL; - stack_pointer[-1] = sub_st; - PyStackRef_CLOSE(tmp); - tmp = list_st; - list_st = PyStackRef_NULL; - stack_pointer[-2] = list_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_SUBSCR_STR_INT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_SUBSCR_STR_INT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBSCR_STR_INT); - opcode = BINARY_OP_SUBSCR_STR_INT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef nos; - _PyStackRef str_st; - _PyStackRef sub_st; - _PyStackRef res; - // _GUARD_TOS_INT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_UNICODE - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyUnicode_CheckExact(o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_SUBSCR_STR_INT - { - sub_st = value; - str_st = nos; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *str = PyStackRef_AsPyObjectBorrow(str_st); - assert(PyLong_CheckExact(sub)); - assert(PyUnicode_CheckExact(str)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (PyUnicode_GET_LENGTH(str) <= index) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - STAT_INC(BINARY_OP, hit); - PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(str_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_FromPyObjectBorrow(res_o); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_SUBSCR_TUPLE_INT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_SUBSCR_TUPLE_INT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBSCR_TUPLE_INT); - opcode = BINARY_OP_SUBSCR_TUPLE_INT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef nos; - _PyStackRef tuple_st; - _PyStackRef sub_st; - _PyStackRef res; - // _GUARD_TOS_INT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_TUPLE - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyTuple_CheckExact(o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_SUBSCR_TUPLE_INT - { - sub_st = value; - tuple_st = nos; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st); - assert(PyLong_CheckExact(sub)); - assert(PyTuple_CheckExact(tuple)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (index >= PyTuple_GET_SIZE(tuple)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - STAT_INC(BINARY_OP, hit); - PyObject *res_o = PyTuple_GET_ITEM(tuple, index); - assert(res_o != NULL); - PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - res = PyStackRef_FromPyObjectNew(res_o); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = tuple_st; - tuple_st = res; - stack_pointer[-1] = tuple_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_SUBTRACT_FLOAT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_SUBTRACT_FLOAT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBTRACT_FLOAT); - opcode = BINARY_OP_SUBTRACT_FLOAT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_TOS_FLOAT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyFloat_CheckExact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_FLOAT - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!PyFloat_CheckExact(left_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_SUBTRACT_FLOAT - { - right = value; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyFloat_CheckExact(left_o)); - assert(PyFloat_CheckExact(right_o)); - STAT_INC(BINARY_OP, hit); - double dres = - ((PyFloatObject *)left_o)->ob_fval - - ((PyFloatObject *)right_o)->ob_fval; - res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres); - if (PyStackRef_IsNull(res)) { - TRACING_JUMP_TO_LABEL(pop_2_error); - } - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_OP_SUBTRACT_INT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_OP_SUBTRACT_INT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 6; - INSTRUCTION_STATS(BINARY_OP_SUBTRACT_INT); - opcode = BINARY_OP_SUBTRACT_INT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5, "incorrect cache size"); - _PyStackRef value; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_TOS_INT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - // _GUARD_NOS_INT - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!_PyLong_CheckExactAndCompact(left_o)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - } - /* Skip 5 cache entries */ - // _BINARY_OP_SUBTRACT_INT - { - right = value; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyLong_CheckExact(left_o)); - assert(PyLong_CheckExact(right_o)); - assert(_PyLong_BothAreCompact((PyLongObject *)left_o, (PyLongObject *)right_o)); - STAT_INC(BINARY_OP, hit); - res = _PyCompactLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o); - if (PyStackRef_IsNull(res)) { - UPDATE_MISS_STATS(BINARY_OP); - assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - JUMP_TO_PREDICTED(TRACING_BINARY_OP); - } - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BINARY_SLICE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BINARY_SLICE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BINARY_SLICE); - opcode = BINARY_SLICE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef container; - _PyStackRef start; - _PyStackRef stop; - _PyStackRef res; - // _SPECIALIZE_BINARY_SLICE - { - #if ENABLE_SPECIALIZATION - OPCODE_DEFERRED_INC(BINARY_SLICE); - #endif /* ENABLE_SPECIALIZATION */ - } - // _BINARY_SLICE - { - stop = stack_pointer[-1]; - start = stack_pointer[-2]; - container = stack_pointer[-3]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), - PyStackRef_AsPyObjectSteal(stop)); - stack_pointer = _PyFrame_GetStackPointer(frame); - PyObject *res_o; - if (slice == NULL) { - res_o = NULL; - } - else { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice); - Py_DECREF(slice); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += 2; - } - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(container); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BUILD_INTERPOLATION) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BUILD_INTERPOLATION; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_INTERPOLATION); - opcode = BUILD_INTERPOLATION; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - _PyStackRef str; - _PyStackRef *format; - _PyStackRef interpolation; - format = &stack_pointer[-(oparg & 1)]; - str = stack_pointer[-1 - (oparg & 1)]; - value = stack_pointer[-2 - (oparg & 1)]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - PyObject *str_o = PyStackRef_AsPyObjectBorrow(str); - int conversion = oparg >> 2; - PyObject *format_o; - if (oparg & 1) { - format_o = PyStackRef_AsPyObjectBorrow(format[0]); - } - else { - format_o = &_Py_STR(empty); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *interpolation_o = _PyInterpolation_Build(value_o, str_o, conversion, format_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (oparg & 1) { - stack_pointer += -(oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(format[0]); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - stack_pointer += -(oparg & 1); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(str); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (interpolation_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - interpolation = PyStackRef_FromPyObjectSteal(interpolation_o); - stack_pointer[0] = interpolation; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BUILD_LIST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BUILD_LIST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_LIST); - opcode = BUILD_LIST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef *values; - _PyStackRef list; - values = &stack_pointer[-oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *list_o = _PyList_FromStackRefStealOnSuccess(values, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (list_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - list = PyStackRef_FromPyObjectStealMortal(list_o); - stack_pointer[-oparg] = list; - stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BUILD_MAP) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BUILD_MAP; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_MAP); - opcode = BUILD_MAP; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef *values; - _PyStackRef map; - values = &stack_pointer[-oparg*2]; - STACKREFS_TO_PYOBJECTS(values, oparg*2, values_o); - if (CONVERSION_FAILED(values_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg*2; --_i >= 0;) { - tmp = values[_i]; - values[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg*2; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *map_o = _PyDict_FromItems( - values_o, 2, - values_o+1, 2, - oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(values_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg*2; --_i >= 0;) { - tmp = values[_i]; - values[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg*2; - assert(WITHIN_STACK_BOUNDS()); - if (map_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - map = PyStackRef_FromPyObjectStealMortal(map_o); - stack_pointer[0] = map; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BUILD_SET) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BUILD_SET; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_SET); - opcode = BUILD_SET; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef *values; - _PyStackRef set; - values = &stack_pointer[-oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *set_o = PySet_New(NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (set_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = values[_i]; - values[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - int err = 0; - for (Py_ssize_t i = 0; i < oparg; i++) { - _PyStackRef value = values[i]; - values[i] = PyStackRef_NULL; - if (err == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - err = _PySet_AddTakeRef((PySetObject *)set_o, PyStackRef_AsPyObjectSteal(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - if (err) { - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(set_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - set = PyStackRef_FromPyObjectStealMortal(set_o); - stack_pointer[-oparg] = set; - stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BUILD_SLICE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BUILD_SLICE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_SLICE); - opcode = BUILD_SLICE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef *args; - _PyStackRef slice; - args = &stack_pointer[-oparg]; - PyObject *start_o = PyStackRef_AsPyObjectBorrow(args[0]); - PyObject *stop_o = PyStackRef_AsPyObjectBorrow(args[1]); - PyObject *step_o = oparg == 3 ? PyStackRef_AsPyObjectBorrow(args[2]) : NULL; - PyObject *slice_o = PySlice_New(start_o, stop_o, step_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - if (slice_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - slice = PyStackRef_FromPyObjectStealMortal(slice_o); - stack_pointer[0] = slice; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BUILD_STRING) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BUILD_STRING; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_STRING); - opcode = BUILD_STRING; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef *pieces; - _PyStackRef str; - pieces = &stack_pointer[-oparg]; - STACKREFS_TO_PYOBJECTS(pieces, oparg, pieces_o); - if (CONVERSION_FAILED(pieces_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = pieces[_i]; - pieces[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - PyObject *str_o = _PyUnicode_JoinArray(&_Py_STR(empty), pieces_o, oparg); - STACKREFS_TO_PYOBJECTS_CLEANUP(pieces_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = pieces[_i]; - pieces[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - if (str_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - str = PyStackRef_FromPyObjectSteal(str_o); - stack_pointer[0] = str; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BUILD_TEMPLATE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BUILD_TEMPLATE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_TEMPLATE); - opcode = BUILD_TEMPLATE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef strings; - _PyStackRef interpolations; - _PyStackRef template; - interpolations = stack_pointer[-1]; - strings = stack_pointer[-2]; - PyObject *strings_o = PyStackRef_AsPyObjectBorrow(strings); - PyObject *interpolations_o = PyStackRef_AsPyObjectBorrow(interpolations); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *template_o = _PyTemplate_Build(strings_o, interpolations_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(interpolations); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(strings); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (template_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - template = PyStackRef_FromPyObjectSteal(template_o); - stack_pointer[0] = template; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(BUILD_TUPLE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = BUILD_TUPLE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(BUILD_TUPLE); - opcode = BUILD_TUPLE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef *values; - _PyStackRef tup; - values = &stack_pointer[-oparg]; - PyObject *tup_o = _PyTuple_FromStackRefStealOnSuccess(values, oparg); - if (tup_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - tup = PyStackRef_FromPyObjectStealMortal(tup_o); - stack_pointer[-oparg] = tup; - stack_pointer += 1 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CACHE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CACHE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CACHE); - opcode = CACHE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - assert(0 && "Executing a cache."); - Py_FatalError("Executing a cache."); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL); - PREDICTED_TRACING_CALL:; - _Py_CODEUNIT* const this_instr = next_instr - 4; - (void)this_instr; - opcode = CALL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef res; - // _SPECIALIZE_CALL - { - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_Call(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() - } - OPCODE_DEFERRED_INC(CALL); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 2 cache entries */ - // _MAYBE_EXPAND_METHOD - { - if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - self_or_null = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - _PyStackRef temp = callable; - callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - // _DO_CALL - { - args = &stack_pointer[-oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - if (Py_TYPE(callable_o) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) - { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, callable, locals, - arguments, total_args, NULL, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (new_frame == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - frame->return_offset = 4u ; - TRACING_DISPATCH_INLINED(new_frame); - } - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - if (opcode == INSTRUMENTED_CALL) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); - if (res_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - } - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_ALLOC_AND_ENTER_INIT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_ALLOC_AND_ENTER_INIT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_ALLOC_AND_ENTER_INIT); - opcode = CALL_ALLOC_AND_ENTER_INIT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef init; - _PyStackRef self; - _PyStackRef *args; - _PyStackRef init_frame; - _PyStackRef new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_AND_ALLOCATE_OBJECT - { - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (!PyStackRef_IsNull(self_or_null)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (!PyType_Check(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyTypeObject *tp = (PyTypeObject *)callable_o; - if (FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - assert(tp->tp_new == PyBaseObject_Type.tp_new); - assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE); - assert(tp->tp_alloc == PyType_GenericAlloc); - PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; - PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); - PyCodeObject *code = (PyCodeObject *)init_func->func_code; - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - STAT_INC(CALL, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *self_o = PyType_GenericAlloc(tp, 0); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (self_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - self_or_null = PyStackRef_FromPyObjectSteal(self_o); - _PyStackRef temp = callable; - callable = PyStackRef_FromPyObjectNew(init_func); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - // _CREATE_INIT_FRAME - { - args = &stack_pointer[-oparg]; - self = self_or_null; - init = callable; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( - tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); - assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); - assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); - shim->localsplus[0] = PyStackRef_DUP(self); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, init, NULL, args-1, oparg+1, NULL, shim); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FrameClearAndPop(tstate, shim); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; - tstate->py_recursion_remaining--; - init_frame = PyStackRef_Wrap(temp); - } - // _PUSH_FRAME - { - new_frame = init_frame; - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_BOUND_METHOD_EXACT_ARGS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_BOUND_METHOD_EXACT_ARGS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BOUND_METHOD_EXACT_ARGS); - opcode = CALL_BOUND_METHOD_EXACT_ARGS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef null; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_CALL_BOUND_METHOD_EXACT_ARGS - { - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - if (!PyStackRef_IsNull(null)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _INIT_CALL_BOUND_METHOD_EXACT_ARGS - { - self_or_null = null; - assert(PyStackRef_IsNull(self_or_null)); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - STAT_INC(CALL, hit); - self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - _PyStackRef temp = callable; - callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - // flush - // _CHECK_FUNCTION_VERSION - { - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (!PyFunction_Check(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyFunctionObject *func = (PyFunctionObject *)callable_o; - if (func->func_version != func_version) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_FUNCTION_EXACT_ARGS - { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - assert(PyFunction_Check(callable_o)); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - PyCodeObject *code = (PyCodeObject *)func->func_code; - if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_STACK_SPACE - { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - PyCodeObject *code = (PyCodeObject *)func->func_code; - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_RECURSION_REMAINING - { - if (tstate->py_recursion_remaining <= 1) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _INIT_CALL_PY_EXACT_ARGS - { - args = &stack_pointer[-oparg]; - int has_self = !PyStackRef_IsNull(self_or_null); - STAT_INC(CALL, hit); - _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); - _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; - pushed_frame->localsplus[0] = self_or_null; - for (int i = 0; i < oparg; i++) { - first_non_self_local[i] = args[i]; - } - new_frame = PyStackRef_Wrap(pushed_frame); - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_BOUND_METHOD_GENERAL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_BOUND_METHOD_GENERAL; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); - opcode = CALL_BOUND_METHOD_GENERAL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef null; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_METHOD_VERSION - { - null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (Py_TYPE(callable_o) != &PyMethod_Type) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyObject *func = ((PyMethodObject *)callable_o)->im_func; - if (!PyFunction_Check(func)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (((PyFunctionObject *)func)->func_version != func_version) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (!PyStackRef_IsNull(null)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _EXPAND_METHOD - { - self_or_null = null; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - assert(PyStackRef_IsNull(self_or_null)); - assert(Py_TYPE(callable_o) == &PyMethod_Type); - self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - _PyStackRef temp = callable; - callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - assert(PyStackRef_FunctionCheck(callable)); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - // flush - // _CHECK_RECURSION_REMAINING - { - if (tstate->py_recursion_remaining <= 1) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _PY_FRAME_GENERAL - { - args = &stack_pointer[-oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { - args--; - total_args++; - } - assert(Py_TYPE(callable_o) == &PyFunction_Type); - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, callable, locals, - args, total_args, NULL, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - new_frame = PyStackRef_Wrap(temp); - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_BUILTIN_CLASS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_BUILTIN_CLASS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_CLASS); - opcode = CALL_BUILTIN_CLASS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_BUILTIN_CLASS - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (!PyType_Check(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyTypeObject *tp = (PyTypeObject *)callable_o; - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - if (tp->tp_vectorcall == NULL) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - STAT_INC(CALL, hit); - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = tp->tp_vectorcall((PyObject *)tp, args_o, total_args, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_BUILTIN_FAST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_BUILTIN_FAST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_FAST); - opcode = CALL_BUILTIN_FAST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_BUILTIN_FAST - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - if (!PyCFunction_CheckExact(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunctionFast_CAST(cfunc)( - PyCFunction_GET_SELF(callable_o), - args_o, - total_args); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_BUILTIN_FAST_WITH_KEYWORDS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_BUILTIN_FAST_WITH_KEYWORDS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_FAST_WITH_KEYWORDS); - opcode = CALL_BUILTIN_FAST_WITH_KEYWORDS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_BUILTIN_FAST_WITH_KEYWORDS - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - if (!PyCFunction_CheckExact(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - STAT_INC(CALL, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFastWithKeywords cfunc = - _PyCFunctionFastWithKeywords_CAST(PyCFunction_GET_FUNCTION(callable_o)); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = cfunc(PyCFunction_GET_SELF(callable_o), args_o, total_args, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_BUILTIN_O) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_BUILTIN_O; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_BUILTIN_O); - opcode = CALL_BUILTIN_O; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_BUILTIN_O - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { - args--; - total_args++; - } - if (total_args != 1) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (!PyCFunction_CheckExact(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (_Py_ReachedRecursionLimit(tstate)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable_o); - _PyStackRef arg = args[0]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable_o), PyStackRef_AsPyObjectBorrow(arg)); - stack_pointer = _PyFrame_GetStackPointer(frame); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_FUNCTION_EX) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_FUNCTION_EX; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_FUNCTION_EX); - opcode = CALL_FUNCTION_EX; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef func; - _PyStackRef callargs; - _PyStackRef func_st; - _PyStackRef null; - _PyStackRef callargs_st; - _PyStackRef kwargs_st; - _PyStackRef result; - // _MAKE_CALLARGS_A_TUPLE - { - callargs = stack_pointer[-2]; - func = stack_pointer[-4]; - PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); - if (!PyTuple_CheckExact(callargs_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *tuple_o = PySequence_Tuple(callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (tuple_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - _PyStackRef temp = callargs; - callargs = PyStackRef_FromPyObjectSteal(tuple_o); - stack_pointer[-2] = callargs; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - // _DO_CALL_FUNCTION_EX - { - kwargs_st = stack_pointer[-1]; - callargs_st = callargs; - null = stack_pointer[-3]; - func_st = func; - (void)null; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); - EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); - PyObject *result_o; - assert(!_PyErr_Occurred(tstate)); - if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - assert(PyTuple_CheckExact(callargs)); - PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? - PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; - stack_pointer[-2] = callargs_st; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!PyFunction_Check(func) && !PyMethod_Check(func)) { - if (result_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(result_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - } - } - else { - if (Py_TYPE(func) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { - PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); - assert(PyTuple_CheckExact(callargs)); - PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); - int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( - tstate, func_st, locals, - nargs, callargs, kwargs, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (new_frame == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - assert( 1u == 1); - frame->return_offset = 1; - TRACING_DISPATCH_INLINED(new_frame); - } - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - assert(PyTuple_CheckExact(callargs)); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - stack_pointer[-2] = callargs_st; - _PyFrame_SetStackPointer(frame, stack_pointer); - result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(kwargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(func_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (result_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - result = PyStackRef_FromPyObjectSteal(result_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = result; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_INTRINSIC_1) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_INTRINSIC_1; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_INTRINSIC_1); - opcode = CALL_INTRINSIC_1; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - assert(oparg <= MAX_INTRINSIC_1); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_INTRINSIC_2) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_INTRINSIC_2; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CALL_INTRINSIC_2); - opcode = CALL_INTRINSIC_2; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value2_st; - _PyStackRef value1_st; - _PyStackRef res; - value1_st = stack_pointer[-1]; - value2_st = stack_pointer[-2]; - assert(oparg <= MAX_INTRINSIC_2); - PyObject *value1 = PyStackRef_AsPyObjectBorrow(value1_st); - PyObject *value2 = PyStackRef_AsPyObjectBorrow(value2_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1); - _PyStackRef tmp = value1_st; - value1_st = PyStackRef_NULL; - stack_pointer[-1] = value1_st; - PyStackRef_CLOSE(tmp); - tmp = value2_st; - value2_st = PyStackRef_NULL; - stack_pointer[-2] = value2_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_ISINSTANCE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_ISINSTANCE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_ISINSTANCE); - opcode = CALL_ISINSTANCE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef null; - _PyStackRef callable; - _PyStackRef instance; - _PyStackRef cls; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _GUARD_THIRD_NULL - { - null = stack_pointer[-3]; - if (!PyStackRef_IsNull(null)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _GUARD_CALLABLE_ISINSTANCE - { - callable = stack_pointer[-4]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyInterpreterState *interp = tstate->interp; - if (callable_o != interp->callable_cache.isinstance) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CALL_ISINSTANCE - { - cls = stack_pointer[-1]; - instance = stack_pointer[-2]; - STAT_INC(CALL, hit); - PyObject *inst_o = PyStackRef_AsPyObjectBorrow(instance); - PyObject *cls_o = PyStackRef_AsPyObjectBorrow(cls); - _PyFrame_SetStackPointer(frame, stack_pointer); - int retval = PyObject_IsInstance(inst_o, cls_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (retval < 0) { - TRACING_JUMP_TO_LABEL(error); - } - (void)null; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(cls); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(instance); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callable); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = retval ? PyStackRef_True : PyStackRef_False; - assert((!PyStackRef_IsNull(res)) ^ (_PyErr_Occurred(tstate) != NULL)); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_KW) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_KW; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_KW); - PREDICTED_TRACING_CALL_KW:; - _Py_CODEUNIT* const this_instr = next_instr - 4; - (void)this_instr; - opcode = CALL_KW; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef kwnames; - _PyStackRef res; - // _SPECIALIZE_CALL_KW - { - self_or_null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() - } - OPCODE_DEFERRED_INC(CALL_KW); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 2 cache entries */ - // _MAYBE_EXPAND_METHOD_KW - { - if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - self_or_null = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - _PyStackRef temp = callable; - callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - // _DO_CALL_KW - { - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - if (Py_TYPE(callable_o) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) - { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, callable, locals, - arguments, positional_args, kwnames_o, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (new_frame == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); - frame->return_offset = 4u ; - TRACING_DISPATCH_INLINED(new_frame); - } - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = kwnames; - kwnames = PyStackRef_NULL; - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; - stack_pointer[-1] = kwnames; - PyStackRef_CLOSE(tmp); - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-2 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-3 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - if (opcode == INSTRUMENTED_CALL_KW) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); - if (res_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = kwnames; - kwnames = PyStackRef_NULL; - stack_pointer[-1] = kwnames; - PyStackRef_CLOSE(tmp); - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-2 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-3 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_KW_BOUND_METHOD) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_KW_BOUND_METHOD; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_KW_BOUND_METHOD); - opcode = CALL_KW_BOUND_METHOD; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef null; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef kwnames; - _PyStackRef new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - } - // _CHECK_METHOD_VERSION_KW - { - null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (Py_TYPE(callable_o) != &PyMethod_Type) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - PyObject *func = ((PyMethodObject *)callable_o)->im_func; - if (!PyFunction_Check(func)) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - if (((PyFunctionObject *)func)->func_version != func_version) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - if (!PyStackRef_IsNull(null)) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - } - // _EXPAND_METHOD_KW - { - self_or_null = null; - assert(PyStackRef_IsNull(self_or_null)); - _PyStackRef callable_s = callable; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - assert(Py_TYPE(callable_o) == &PyMethod_Type); - self_or_null = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_self); - callable = PyStackRef_FromPyObjectNew(((PyMethodObject *)callable_o)->im_func); - assert(PyStackRef_FunctionCheck(callable)); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callable_s); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - // flush - // _PY_FRAME_KW - { - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - assert(Py_TYPE(callable_o) == &PyFunction_Type); - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, callable, locals, - arguments, positional_args, kwnames_o, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - new_frame = PyStackRef_Wrap(temp); - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_KW_NON_PY) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_KW_NON_PY; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_KW_NON_PY); - opcode = CALL_KW_NON_PY; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef kwnames; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CHECK_IS_NOT_PY_CALLABLE_KW - { - callable = stack_pointer[-3 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (PyFunction_Check(callable_o)) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - if (Py_TYPE(callable_o) == &PyMethod_Type) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - } - // _CALL_KW_NON_PY - { - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = stack_pointer[-2 - oparg]; - #if TIER_ONE - assert(opcode != INSTRUMENTED_CALL); - #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = kwnames; - kwnames = PyStackRef_NULL; - stack_pointer[-1] = kwnames; - PyStackRef_CLOSE(tmp); - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-2 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-3 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_KW_PY) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_KW_PY; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_KW_PY); - opcode = CALL_KW_PY; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL_KW == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef kwnames; - _PyStackRef new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - } - // _CHECK_FUNCTION_VERSION_KW - { - callable = stack_pointer[-3 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (!PyFunction_Check(callable_o)) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - PyFunctionObject *func = (PyFunctionObject *)callable_o; - if (func->func_version != func_version) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - } - // _CHECK_RECURSION_REMAINING - { - if (tstate->py_recursion_remaining <= 1) { - UPDATE_MISS_STATS(CALL_KW); - assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - JUMP_TO_PREDICTED(TRACING_CALL_KW); - } - } - // _PY_FRAME_KW - { - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - self_or_null = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - assert(Py_TYPE(callable_o) == &PyFunction_Type); - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, callable, locals, - arguments, positional_args, kwnames_o, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - new_frame = PyStackRef_Wrap(temp); - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_LEN) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_LEN; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_LEN); - opcode = CALL_LEN; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef null; - _PyStackRef callable; - _PyStackRef arg; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _GUARD_NOS_NULL - { - null = stack_pointer[-2]; - if (!PyStackRef_IsNull(null)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _GUARD_CALLABLE_LEN - { - callable = stack_pointer[-3]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyInterpreterState *interp = tstate->interp; - if (callable_o != interp->callable_cache.len) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CALL_LEN - { - arg = stack_pointer[-1]; - (void)null; - STAT_INC(CALL, hit); - PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_ssize_t len_i = PyObject_Length(arg_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (len_i < 0) { - TRACING_JUMP_TO_LABEL(error); - } - PyObject *res_o = PyLong_FromSsize_t(len_i); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callable); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_LIST_APPEND) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_LIST_APPEND; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_LIST_APPEND); - opcode = CALL_LIST_APPEND; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef nos; - _PyStackRef self; - _PyStackRef arg; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _GUARD_CALLABLE_LIST_APPEND - { - callable = stack_pointer[-3]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyInterpreterState *interp = tstate->interp; - if (callable_o != interp->callable_cache.list_append) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _GUARD_NOS_NOT_NULL - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (o == NULL) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _GUARD_NOS_LIST - { - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyList_CheckExact(o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CALL_LIST_APPEND - { - arg = stack_pointer[-1]; - self = nos; - assert(oparg == 1); - PyObject *self_o = PyStackRef_AsPyObjectBorrow(self); - if (!PyList_CheckExact(self_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (!LOCK_OBJECT(self_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - STAT_INC(CALL, hit); - int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); - UNLOCK_OBJECT(self_o); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(self); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - #if TIER_ONE - - assert(next_instr->op.code == POP_TOP); - SKIP_OVER(1); - #endif - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_METHOD_DESCRIPTOR_FAST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_METHOD_DESCRIPTOR_FAST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST); - opcode = CALL_METHOD_DESCRIPTOR_FAST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_FAST - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - if (total_args == 0) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyMethodDef *meth = method->d_method; - if (meth->ml_flags != METH_FASTCALL) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); - assert(self != NULL); - if (!Py_IS_TYPE(self, method->d_common.d_type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - STAT_INC(CALL, hit); - int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFast cfunc = _PyCFunctionFast_CAST(meth->ml_meth); - PyObject *res_o = cfunc(self, (args_o + 1), nargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); - opcode = CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - if (total_args == 0) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyMethodDef *meth = method->d_method; - if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyTypeObject *d_type = method->d_common.d_type; - PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); - assert(self != NULL); - if (!Py_IS_TYPE(self, d_type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - STAT_INC(CALL, hit); - int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCFunctionFastWithKeywords cfunc = - _PyCFunctionFastWithKeywords_CAST(meth->ml_meth); - PyObject *res_o = cfunc(self, (args_o + 1), nargs, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_METHOD_DESCRIPTOR_NOARGS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_METHOD_DESCRIPTOR_NOARGS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_NOARGS); - opcode = CALL_METHOD_DESCRIPTOR_NOARGS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_NOARGS - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - assert(oparg == 0 || oparg == 1); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { - args--; - total_args++; - } - if (total_args != 1) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyMethodDef *meth = method->d_method; - _PyStackRef self_stackref = args[0]; - PyObject *self = PyStackRef_AsPyObjectBorrow(self_stackref); - if (!Py_IS_TYPE(self, method->d_common.d_type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (meth->ml_flags != METH_NOARGS) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (_Py_ReachedRecursionLimit(tstate)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, self, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(self_stackref); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_METHOD_DESCRIPTOR_O) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_METHOD_DESCRIPTOR_O; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_METHOD_DESCRIPTOR_O); - opcode = CALL_METHOD_DESCRIPTOR_O; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CALL_METHOD_DESCRIPTOR_O - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; - if (total_args != 2) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyMethodDef *meth = method->d_method; - if (meth->ml_flags != METH_O) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (_Py_ReachedRecursionLimit(tstate)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - _PyStackRef arg_stackref = arguments[1]; - _PyStackRef self_stackref = arguments[0]; - if (!Py_IS_TYPE(PyStackRef_AsPyObjectBorrow(self_stackref), - method->d_common.d_type)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyCFunction_TrampolineCall(cfunc, - PyStackRef_AsPyObjectBorrow(self_stackref), - PyStackRef_AsPyObjectBorrow(arg_stackref)); - stack_pointer = _PyFrame_GetStackPointer(frame); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_NON_PY_GENERAL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_NON_PY_GENERAL; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_NON_PY_GENERAL); - opcode = CALL_NON_PY_GENERAL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _CHECK_IS_NOT_PY_CALLABLE - { - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (PyFunction_Check(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - if (Py_TYPE(callable_o) == &PyMethod_Type) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CALL_NON_PY_GENERAL - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - #if TIER_ONE - assert(opcode != INSTRUMENTED_CALL); - #endif - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_PY_EXACT_ARGS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_PY_EXACT_ARGS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_PY_EXACT_ARGS); - opcode = CALL_PY_EXACT_ARGS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_FUNCTION_VERSION - { - callable = stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (!PyFunction_Check(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyFunctionObject *func = (PyFunctionObject *)callable_o; - if (func->func_version != func_version) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_FUNCTION_EXACT_ARGS - { - self_or_null = stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - assert(PyFunction_Check(callable_o)); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - PyCodeObject *code = (PyCodeObject *)func->func_code; - if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_STACK_SPACE - { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - PyCodeObject *code = (PyCodeObject *)func->func_code; - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_RECURSION_REMAINING - { - if (tstate->py_recursion_remaining <= 1) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _INIT_CALL_PY_EXACT_ARGS - { - args = &stack_pointer[-oparg]; - int has_self = !PyStackRef_IsNull(self_or_null); - STAT_INC(CALL, hit); - _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); - _PyStackRef *first_non_self_local = pushed_frame->localsplus + has_self; - pushed_frame->localsplus[0] = self_or_null; - for (int i = 0; i < oparg; i++) { - first_non_self_local[i] = args[i]; - } - new_frame = PyStackRef_Wrap(pushed_frame); - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_PY_GENERAL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_PY_GENERAL; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_PY_GENERAL); - opcode = CALL_PY_GENERAL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_FUNCTION_VERSION - { - callable = stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (!PyFunction_Check(callable_o)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - PyFunctionObject *func = (PyFunctionObject *)callable_o; - if (func->func_version != func_version) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CHECK_RECURSION_REMAINING - { - if (tstate->py_recursion_remaining <= 1) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _PY_FRAME_GENERAL - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - if (!PyStackRef_IsNull(self_or_null)) { - args--; - total_args++; - } - assert(Py_TYPE(callable_o) == &PyFunction_Type); - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( - tstate, callable, locals, - args, total_args, NULL, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (temp == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - new_frame = PyStackRef_Wrap(temp); - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_STR_1) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_STR_1; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_STR_1); - opcode = CALL_STR_1; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef null; - _PyStackRef callable; - _PyStackRef arg; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _GUARD_NOS_NULL - { - null = stack_pointer[-2]; - if (!PyStackRef_IsNull(null)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _GUARD_CALLABLE_STR_1 - { - callable = stack_pointer[-3]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (callable_o != (PyObject *)&PyUnicode_Type) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CALL_STR_1 - { - arg = stack_pointer[-1]; - PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); - STAT_INC(CALL, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Str(arg_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - (void)callable; - (void)null; - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_TUPLE_1) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_TUPLE_1; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_TUPLE_1); - opcode = CALL_TUPLE_1; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef null; - _PyStackRef callable; - _PyStackRef arg; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _GUARD_NOS_NULL - { - null = stack_pointer[-2]; - if (!PyStackRef_IsNull(null)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _GUARD_CALLABLE_TUPLE_1 - { - callable = stack_pointer[-3]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (callable_o != (PyObject *)&PyTuple_Type) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CALL_TUPLE_1 - { - arg = stack_pointer[-1]; - PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); - STAT_INC(CALL, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PySequence_Tuple(arg_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - (void)callable; - (void)null; - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CALL_TYPE_1) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CALL_TYPE_1; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(CALL_TYPE_1); - opcode = CALL_TYPE_1; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); - _PyStackRef null; - _PyStackRef callable; - _PyStackRef arg; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _GUARD_NOS_NULL - { - null = stack_pointer[-2]; - if (!PyStackRef_IsNull(null)) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _GUARD_CALLABLE_TYPE_1 - { - callable = stack_pointer[-3]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - if (callable_o != (PyObject *)&PyType_Type) { - UPDATE_MISS_STATS(CALL); - assert(_PyOpcode_Deopt[opcode] == (CALL)); - JUMP_TO_PREDICTED(TRACING_CALL); - } - } - // _CALL_TYPE_1 - { - arg = stack_pointer[-1]; - PyObject *arg_o = PyStackRef_AsPyObjectBorrow(arg); - assert(oparg == 1); - (void)callable; - (void)null; - STAT_INC(CALL, hit); - res = PyStackRef_FromPyObjectNew(Py_TYPE(arg_o)); - stack_pointer[-3] = res; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(CHECK_EG_MATCH) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CHECK_EG_MATCH; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CHECK_EG_MATCH); - opcode = CHECK_EG_MATCH; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef exc_value_st; - _PyStackRef match_type_st; - _PyStackRef rest; - _PyStackRef match; - match_type_st = stack_pointer[-1]; - exc_value_st = stack_pointer[-2]; - PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); - PyObject *match_type = PyStackRef_AsPyObjectBorrow(match_type_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyEval_CheckExceptStarTypeValid(tstate, match_type); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = match_type_st; - match_type_st = PyStackRef_NULL; - stack_pointer[-1] = match_type_st; - PyStackRef_CLOSE(tmp); - tmp = exc_value_st; - exc_value_st = PyStackRef_NULL; - stack_pointer[-2] = exc_value_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - PyObject *match_o = NULL; - PyObject *rest_o = NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_ExceptionGroupMatch(frame, exc_value, match_type, - &match_o, &rest_o); - _PyStackRef tmp = match_type_st; - match_type_st = PyStackRef_NULL; - stack_pointer[-1] = match_type_st; - PyStackRef_CLOSE(tmp); - tmp = exc_value_st; - exc_value_st = PyStackRef_NULL; - stack_pointer[-2] = exc_value_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res < 0) { - TRACING_JUMP_TO_LABEL(error); - } - assert((match_o == NULL) == (rest_o == NULL)); - if (match_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - if (!Py_IsNone(match_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyErr_SetHandledException(match_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - rest = PyStackRef_FromPyObjectSteal(rest_o); - match = PyStackRef_FromPyObjectSteal(match_o); - stack_pointer[0] = rest; - stack_pointer[1] = match; - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CHECK_EXC_MATCH) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CHECK_EXC_MATCH; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CHECK_EXC_MATCH); - opcode = CHECK_EXC_MATCH; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef left; - _PyStackRef right; - _PyStackRef b; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyExceptionInstance_Check(left_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyEval_CheckExceptTypeValid(tstate, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = PyErr_GivenExceptionMatches(left_o, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(right); - stack_pointer = _PyFrame_GetStackPointer(frame); - b = res ? PyStackRef_True : PyStackRef_False; - stack_pointer[0] = b; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CLEANUP_THROW) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CLEANUP_THROW; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CLEANUP_THROW); - opcode = CLEANUP_THROW; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef sub_iter; - _PyStackRef last_sent_val; - _PyStackRef exc_value_st; - _PyStackRef none; - _PyStackRef value; - exc_value_st = stack_pointer[-1]; - last_sent_val = stack_pointer[-2]; - sub_iter = stack_pointer[-3]; - PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st); - #if !_Py_TAIL_CALL_INTERP - assert(throwflag); - #endif - assert(exc_value && PyExceptionInstance_Check(exc_value)); - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = PyErr_GivenExceptionMatches(exc_value, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - value = PyStackRef_FromPyObjectNew(((PyStopIterationObject *)exc_value)->value); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = sub_iter; - sub_iter = value; - stack_pointer[-3] = sub_iter; - PyStackRef_CLOSE(tmp); - tmp = exc_value_st; - exc_value_st = PyStackRef_NULL; - stack_pointer[-1] = exc_value_st; - PyStackRef_CLOSE(tmp); - tmp = last_sent_val; - last_sent_val = PyStackRef_NULL; - stack_pointer[-2] = last_sent_val; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - none = PyStackRef_None; - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetRaisedException(tstate, Py_NewRef(exc_value)); - monitor_reraise(tstate, frame, this_instr); - TRACING_JUMP_TO_LABEL(exception_unwind); - } - stack_pointer[0] = none; - stack_pointer[1] = value; - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(COMPARE_OP) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = COMPARE_OP; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP); - PREDICTED_TRACING_COMPARE_OP:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - opcode = COMPARE_OP; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _SPECIALIZE_COMPARE_OP - { - right = stack_pointer[-1]; - left = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_CompareOp(left, right, next_instr, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() - } - OPCODE_DEFERRED_INC(COMPARE_OP); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _COMPARE_OP - { - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert((oparg >> 5) <= Py_GE); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_RichCompare(left_o, right_o, oparg >> 5); - _PyStackRef tmp = right; - right = PyStackRef_NULL; - stack_pointer[-1] = right; - PyStackRef_CLOSE(tmp); - tmp = left; - left = PyStackRef_NULL; - stack_pointer[-2] = left; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - if (oparg & 16) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int res_bool = PyObject_IsTrue(res_o); - Py_DECREF(res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_bool < 0) { - TRACING_JUMP_TO_LABEL(error); - } - res = res_bool ? PyStackRef_True : PyStackRef_False; - } - else { - res = PyStackRef_FromPyObjectSteal(res_o); - } - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(COMPARE_OP_FLOAT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = COMPARE_OP_FLOAT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_FLOAT); - opcode = COMPARE_OP_FLOAT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - _PyStackRef value; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_TOS_FLOAT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyFloat_CheckExact(value_o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - JUMP_TO_PREDICTED(TRACING_COMPARE_OP); - } - } - // _GUARD_NOS_FLOAT - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!PyFloat_CheckExact(left_o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - JUMP_TO_PREDICTED(TRACING_COMPARE_OP); - } - } - /* Skip 1 cache entry */ - // _COMPARE_OP_FLOAT - { - right = value; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - STAT_INC(COMPARE_OP, hit); - double dleft = PyFloat_AS_DOUBLE(left_o); - double dright = PyFloat_AS_DOUBLE(right_o); - int sign_ish = COMPARISON_BIT(dleft, dright); - PyStackRef_CLOSE_SPECIALIZED(left, _PyFloat_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(right, _PyFloat_ExactDealloc); - res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(COMPARE_OP_INT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = COMPARE_OP_INT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_INT); - opcode = COMPARE_OP_INT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - _PyStackRef value; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_TOS_INT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - JUMP_TO_PREDICTED(TRACING_COMPARE_OP); - } - } - // _GUARD_NOS_INT - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - if (!_PyLong_CheckExactAndCompact(left_o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - JUMP_TO_PREDICTED(TRACING_COMPARE_OP); - } - } - /* Skip 1 cache entry */ - // _COMPARE_OP_INT - { - right = value; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(_PyLong_IsCompact((PyLongObject *)left_o)); - assert(_PyLong_IsCompact((PyLongObject *)right_o)); - STAT_INC(COMPARE_OP, hit); - assert(_PyLong_DigitCount((PyLongObject *)left_o) <= 1 && - _PyLong_DigitCount((PyLongObject *)right_o) <= 1); - Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left_o); - Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right_o); - int sign_ish = COMPARISON_BIT(ileft, iright); - PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); - res = (sign_ish & oparg) ? PyStackRef_True : PyStackRef_False; - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(COMPARE_OP_STR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = COMPARE_OP_STR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(COMPARE_OP_STR); - opcode = COMPARE_OP_STR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_COMPARE_OP == 1, "incorrect cache size"); - _PyStackRef value; - _PyStackRef nos; - _PyStackRef left; - _PyStackRef right; - _PyStackRef res; - // _GUARD_TOS_UNICODE - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyUnicode_CheckExact(value_o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - JUMP_TO_PREDICTED(TRACING_COMPARE_OP); - } - } - // _GUARD_NOS_UNICODE - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyUnicode_CheckExact(o)) { - UPDATE_MISS_STATS(COMPARE_OP); - assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - JUMP_TO_PREDICTED(TRACING_COMPARE_OP); - } - } - /* Skip 1 cache entry */ - // _COMPARE_OP_STR - { - right = value; - left = nos; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - STAT_INC(COMPARE_OP, hit); - int eq = _PyUnicode_Equal(left_o, right_o); - assert((oparg >> 5) == Py_EQ || (oparg >> 5) == Py_NE); - PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc); - PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc); - assert(eq == 0 || eq == 1); - assert((oparg & 0xf) == COMPARISON_NOT_EQUALS || (oparg & 0xf) == COMPARISON_EQUALS); - assert(COMPARISON_NOT_EQUALS + 1 == COMPARISON_EQUALS); - res = ((COMPARISON_NOT_EQUALS + eq) & oparg) ? PyStackRef_True : PyStackRef_False; - } - stack_pointer[-2] = res; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CONTAINS_OP) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CONTAINS_OP; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(CONTAINS_OP); - PREDICTED_TRACING_CONTAINS_OP:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - opcode = CONTAINS_OP; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef right; - _PyStackRef left; - _PyStackRef b; - // _SPECIALIZE_CONTAINS_OP - { - right = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_ContainsOp(right, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() - } - OPCODE_DEFERRED_INC(CONTAINS_OP); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _CONTAINS_OP - { - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = PySequence_Contains(right_o, left_o); - _PyStackRef tmp = right; - right = PyStackRef_NULL; - stack_pointer[-1] = right; - PyStackRef_CLOSE(tmp); - tmp = left; - left = PyStackRef_NULL; - stack_pointer[-2] = left; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res < 0) { - TRACING_JUMP_TO_LABEL(error); - } - b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; - } - stack_pointer[0] = b; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CONTAINS_OP_DICT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CONTAINS_OP_DICT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(CONTAINS_OP_DICT); - opcode = CONTAINS_OP_DICT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); - _PyStackRef tos; - _PyStackRef left; - _PyStackRef right; - _PyStackRef b; - // _GUARD_TOS_DICT - { - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyDict_CheckExact(o)) { - UPDATE_MISS_STATS(CONTAINS_OP); - assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); - JUMP_TO_PREDICTED(TRACING_CONTAINS_OP); - } - } - /* Skip 1 cache entry */ - // _CONTAINS_OP_DICT - { - right = tos; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyDict_CheckExact(right_o)); - STAT_INC(CONTAINS_OP, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = PyDict_Contains(right_o, left_o); - _PyStackRef tmp = right; - right = PyStackRef_NULL; - stack_pointer[-1] = right; - PyStackRef_CLOSE(tmp); - tmp = left; - left = PyStackRef_NULL; - stack_pointer[-2] = left; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res < 0) { - TRACING_JUMP_TO_LABEL(error); - } - b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; - } - stack_pointer[0] = b; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CONTAINS_OP_SET) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CONTAINS_OP_SET; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(CONTAINS_OP_SET); - opcode = CONTAINS_OP_SET; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_CONTAINS_OP == 1, "incorrect cache size"); - _PyStackRef tos; - _PyStackRef left; - _PyStackRef right; - _PyStackRef b; - // _GUARD_TOS_ANY_SET - { - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyAnySet_CheckExact(o)) { - UPDATE_MISS_STATS(CONTAINS_OP); - assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); - JUMP_TO_PREDICTED(TRACING_CONTAINS_OP); - } - } - /* Skip 1 cache entry */ - // _CONTAINS_OP_SET - { - right = tos; - left = stack_pointer[-2]; - PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); - PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyAnySet_CheckExact(right_o)); - STAT_INC(CONTAINS_OP, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PySet_Contains((PySetObject *)right_o, left_o); - _PyStackRef tmp = right; - right = PyStackRef_NULL; - stack_pointer[-1] = right; - PyStackRef_CLOSE(tmp); - tmp = left; - left = PyStackRef_NULL; - stack_pointer[-2] = left; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res < 0) { - TRACING_JUMP_TO_LABEL(error); - } - b = (res ^ oparg) ? PyStackRef_True : PyStackRef_False; - } - stack_pointer[0] = b; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(CONVERT_VALUE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = CONVERT_VALUE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(CONVERT_VALUE); - opcode = CONVERT_VALUE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - _PyStackRef result; - value = stack_pointer[-1]; - conversion_func conv_fn; - assert(oparg >= FVC_STR && oparg <= FVC_ASCII); - conv_fn = _PyEval_ConversionFuncs[oparg]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *result_o = conv_fn(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (result_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - result = PyStackRef_FromPyObjectSteal(result_o); - stack_pointer[0] = result; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(COPY) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = COPY; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(COPY); - opcode = COPY; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef bottom; - _PyStackRef top; - bottom = stack_pointer[-1 - (oparg-1)]; - top = PyStackRef_DUP(bottom); - stack_pointer[0] = top; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(COPY_FREE_VARS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = COPY_FREE_VARS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(COPY_FREE_VARS); - opcode = COPY_FREE_VARS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - PyCodeObject *co = _PyFrame_GetCode(frame); - assert(PyStackRef_FunctionCheck(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - PyObject *closure = func->func_closure; - assert(oparg == co->co_nfreevars); - int offset = co->co_nlocalsplus - oparg; - for (int i = 0; i < oparg; ++i) { - PyObject *o = PyTuple_GET_ITEM(closure, i); - frame->localsplus[offset + i] = PyStackRef_FromPyObjectNew(o); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(DELETE_ATTR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = DELETE_ATTR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_ATTR); - opcode = DELETE_ATTR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef owner; - owner = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_DelAttr(PyStackRef_AsPyObjectBorrow(owner), name); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(DELETE_DEREF) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = DELETE_DEREF; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_DEREF); - opcode = DELETE_DEREF; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - PyObject *cell = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - PyObject *oldobj = PyCell_SwapTakeRef((PyCellObject *)cell, NULL); - if (oldobj == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(oldobj); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(DELETE_FAST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = DELETE_FAST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_FAST); - opcode = DELETE_FAST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef v = GETLOCAL(oparg); - if (PyStackRef_IsNull(v)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, - UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = PyStackRef_NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(DELETE_GLOBAL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = DELETE_GLOBAL; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_GLOBAL); - opcode = DELETE_GLOBAL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyDict_Pop(GLOBALS(), name, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - if (err == 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(DELETE_NAME) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = DELETE_NAME; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_NAME); - opcode = DELETE_NAME; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *ns = LOCALS(); - int err; - if (ns == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_SystemError, - "no locals when deleting %R", name); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_DelItem(ns, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, - name); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(DELETE_SUBSCR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = DELETE_SUBSCR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DELETE_SUBSCR); - opcode = DELETE_SUBSCR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef container; - _PyStackRef sub; - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_DelItem(PyStackRef_AsPyObjectBorrow(container), - PyStackRef_AsPyObjectBorrow(sub)); - _PyStackRef tmp = sub; - sub = PyStackRef_NULL; - stack_pointer[-1] = sub; - PyStackRef_CLOSE(tmp); - tmp = container; - container = PyStackRef_NULL; - stack_pointer[-2] = container; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(DICT_MERGE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = DICT_MERGE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DICT_MERGE); - opcode = DICT_MERGE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef callable; - _PyStackRef dict; - _PyStackRef update; - update = stack_pointer[-1]; - dict = stack_pointer[-2 - (oparg - 1)]; - callable = stack_pointer[-5 - (oparg - 1)]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); - PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyDict_MergeEx(dict_o, update_o, 2); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatKwargsError(tstate, callable_o, update_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(update); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(update); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(DICT_UPDATE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = DICT_UPDATE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(DICT_UPDATE); - opcode = DICT_UPDATE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef dict; - _PyStackRef update; - update = stack_pointer[-1]; - dict = stack_pointer[-2 - (oparg - 1)]; - PyObject *dict_o = PyStackRef_AsPyObjectBorrow(dict); - PyObject *update_o = PyStackRef_AsPyObjectBorrow(update); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyDict_Update(dict_o, update_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_AttributeError); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object is not a mapping", - Py_TYPE(update_o)->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(update); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(update); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(END_ASYNC_FOR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = END_ASYNC_FOR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(END_ASYNC_FOR); - opcode = END_ASYNC_FOR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef awaitable_st; - _PyStackRef exc_st; - exc_st = stack_pointer[-1]; - awaitable_st = stack_pointer[-2]; - JUMPBY(0); - (void)oparg; - PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); - assert(exc && PyExceptionInstance_Check(exc)); - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = exc_st; - exc_st = PyStackRef_NULL; - stack_pointer[-1] = exc_st; - PyStackRef_CLOSE(tmp); - tmp = awaitable_st; - awaitable_st = PyStackRef_NULL; - stack_pointer[-2] = awaitable_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - } - else { - Py_INCREF(exc); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, this_instr); - TRACING_JUMP_TO_LABEL(exception_unwind); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(END_FOR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = END_FOR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(END_FOR); - opcode = END_FOR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - value = stack_pointer[-1]; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(END_SEND) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = END_SEND; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(END_SEND); - opcode = END_SEND; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef receiver; - _PyStackRef value; - _PyStackRef val; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; - val = value; - stack_pointer[-2] = val; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(receiver); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(ENTER_EXECUTOR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = ENTER_EXECUTOR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(ENTER_EXECUTOR); - opcode = ENTER_EXECUTOR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - #ifdef _Py_TIER2 - PyCodeObject *code = _PyFrame_GetCode(frame); - _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; - assert(executor->vm_data.index == INSTR_OFFSET() - 1); - assert(executor->vm_data.code == code); - assert(executor->vm_data.valid); - assert(tstate->current_executor == NULL); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { - opcode = executor->vm_data.opcode; - oparg = (oparg & ~255) | executor->vm_data.oparg; - next_instr = this_instr; - if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { - PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); - } - DISPATCH_GOTO(); - } - assert(executor != tstate->interp->cold_executor); - tstate->jit_exit = NULL; - #if TRACING_JIT - RECORD_TRACE_NO_DISPATCH(); - #endif - TIER1_TO_TIER2(executor); - #else - Py_FatalError("ENTER_EXECUTOR is not supported in this build"); - #endif /* _Py_TIER2 */ - TRACING_DISPATCH(); - } - - TRACING_TARGET(EXIT_INIT_CHECK) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = EXIT_INIT_CHECK; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(EXIT_INIT_CHECK); - opcode = EXIT_INIT_CHECK; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef should_be_none; - should_be_none = stack_pointer[-1]; - if (!PyStackRef_IsNone(should_be_none)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyErr_Format(PyExc_TypeError, - "__init__() should return None, not '%.200s'", - Py_TYPE(PyStackRef_AsPyObjectBorrow(should_be_none))->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(EXTENDED_ARG) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = EXTENDED_ARG; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(EXTENDED_ARG); - opcode = EXTENDED_ARG; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - assert(oparg); - opcode = next_instr->op.code; - oparg = oparg << 8 | next_instr->op.arg; - PRE_DISPATCH_GOTO(); - DISPATCH_GOTO(); - } - - TRACING_TARGET(FORMAT_SIMPLE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = FORMAT_SIMPLE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(FORMAT_SIMPLE); - opcode = FORMAT_SIMPLE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyUnicode_CheckExact(value_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Format(value_o, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - else { - res = value; - stack_pointer += -1; - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(FORMAT_WITH_SPEC) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = FORMAT_WITH_SPEC; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(FORMAT_WITH_SPEC); - opcode = FORMAT_WITH_SPEC; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - _PyStackRef fmt_spec; - _PyStackRef res; - fmt_spec = stack_pointer[-1]; - value = stack_pointer[-2]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Format(PyStackRef_AsPyObjectBorrow(value), PyStackRef_AsPyObjectBorrow(fmt_spec)); - _PyStackRef tmp = fmt_spec; - fmt_spec = PyStackRef_NULL; - stack_pointer[-1] = fmt_spec; - PyStackRef_CLOSE(tmp); - tmp = value; - value = PyStackRef_NULL; - stack_pointer[-2] = value; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(FOR_ITER) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = FOR_ITER; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER); - PREDICTED_TRACING_FOR_ITER:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - opcode = FOR_ITER; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef iter; - _PyStackRef null_or_index; - _PyStackRef next; - // _SPECIALIZE_FOR_ITER - { - null_or_index = stack_pointer[-1]; - iter = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_ForIter(iter, null_or_index, next_instr, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() - } - OPCODE_DEFERRED_INC(FOR_ITER); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _FOR_ITER - { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!PyStackRef_IsValid(item)) { - if (PyStackRef_IsError(item)) { - TRACING_JUMP_TO_LABEL(error); - } - JUMPBY(oparg + 1); - RECORD_DYNAMIC_JUMP_TAKEN(); - stack_pointer[-1] = null_or_index; - TRACING_DISPATCH(); - } - next = item; - } - stack_pointer[-1] = null_or_index; - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(FOR_ITER_GEN) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = FOR_ITER_GEN; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_GEN); - opcode = FOR_ITER_GEN; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - _PyStackRef iter; - _PyStackRef gen_frame; - _PyStackRef new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(TRACING_FOR_ITER); - } - } - // _FOR_ITER_GEN_FRAME - { - iter = stack_pointer[-2]; - PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(gen) != &PyGen_Type) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(TRACING_FOR_ITER); - } - #ifdef Py_GIL_DISABLED - if (!_PyObject_IsUniquelyReferenced((PyObject *)gen)) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(TRACING_FOR_ITER); - } - #endif - if (gen->gi_frame_state >= FRAME_EXECUTING) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(TRACING_FOR_ITER); - } - STAT_INC(FOR_ITER, hit); - _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; - _PyFrame_StackPush(pushed_frame, PyStackRef_None); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - pushed_frame->previous = frame; - frame->return_offset = (uint16_t)( 2u + oparg); - gen_frame = PyStackRef_Wrap(pushed_frame); - } - // _PUSH_FRAME - { - new_frame = gen_frame; - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(FOR_ITER_LIST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = FOR_ITER_LIST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_LIST); - opcode = FOR_ITER_LIST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - _PyStackRef iter; - _PyStackRef null_or_index; - _PyStackRef next; - /* Skip 1 cache entry */ - // _ITER_CHECK_LIST - { - null_or_index = stack_pointer[-1]; - iter = stack_pointer[-2]; - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(iter_o) != &PyList_Type) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(TRACING_FOR_ITER); - } - assert(PyStackRef_IsTaggedInt(null_or_index)); - #ifdef Py_GIL_DISABLED - if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(TRACING_FOR_ITER); - } - #endif - } - // _ITER_JUMP_LIST - { - #ifdef Py_GIL_DISABLED - - #else - PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(list_o) == &PyList_Type); - STAT_INC(FOR_ITER, hit); - if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { - null_or_index = PyStackRef_TagInt(-1); - JUMPBY(oparg + 1); - RECORD_DYNAMIC_JUMP_TAKEN(); - stack_pointer[-1] = null_or_index; - TRACING_DISPATCH(); - } - #endif - } - // _ITER_NEXT_LIST - { - PyObject *list_o = PyStackRef_AsPyObjectBorrow(iter); - assert(PyList_CheckExact(list_o)); - #ifdef Py_GIL_DISABLED - assert(_Py_IsOwnedByCurrentThread(list_o) || - _PyObject_GC_IS_SHARED(list_o)); - STAT_INC(FOR_ITER, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - int result = _PyList_GetItemRefNoLock((PyListObject *)list_o, PyStackRef_UntagInt(null_or_index), &next); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (result < 0) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(TRACING_FOR_ITER); - } - if (result == 0) { - null_or_index = PyStackRef_TagInt(-1); - JUMPBY(oparg + 1); - stack_pointer[-1] = null_or_index; - TRACING_DISPATCH(); - } - #else - next = PyStackRef_FromPyObjectNew(PyList_GET_ITEM(list_o, PyStackRef_UntagInt(null_or_index))); - #endif - null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); - } - stack_pointer[-1] = null_or_index; - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(FOR_ITER_RANGE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = FOR_ITER_RANGE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_RANGE); - opcode = FOR_ITER_RANGE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - _PyStackRef iter; - _PyStackRef next; - /* Skip 1 cache entry */ - // _ITER_CHECK_RANGE - { - iter = stack_pointer[-2]; - _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(r) != &PyRangeIter_Type) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(TRACING_FOR_ITER); - } - #ifdef Py_GIL_DISABLED - if (!_PyObject_IsUniquelyReferenced((PyObject *)r)) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(TRACING_FOR_ITER); - } - #endif - } - // _ITER_JUMP_RANGE - { - _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(r) == &PyRangeIter_Type); - #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced((PyObject *)r)); - #endif - STAT_INC(FOR_ITER, hit); - if (r->len <= 0) { - JUMPBY(oparg + 1); - RECORD_DYNAMIC_JUMP_TAKEN(); - TRACING_DISPATCH(); - } - } - // _ITER_NEXT_RANGE - { - _PyRangeIterObject *r = (_PyRangeIterObject *)PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(r) == &PyRangeIter_Type); - #ifdef Py_GIL_DISABLED - assert(_PyObject_IsUniquelyReferenced((PyObject *)r)); - #endif - assert(r->len > 0); - long value = r->start; - r->start = value + r->step; - r->len--; - PyObject *res = PyLong_FromLong(value); - if (res == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - next = PyStackRef_FromPyObjectSteal(res); - } - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(FOR_ITER_TUPLE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = FOR_ITER_TUPLE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(FOR_ITER_TUPLE); - opcode = FOR_ITER_TUPLE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); - _PyStackRef iter; - _PyStackRef null_or_index; - _PyStackRef next; - /* Skip 1 cache entry */ - // _ITER_CHECK_TUPLE - { - null_or_index = stack_pointer[-1]; - iter = stack_pointer[-2]; - PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - if (Py_TYPE(iter_o) != &PyTuple_Type) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - JUMP_TO_PREDICTED(TRACING_FOR_ITER); - } - assert(PyStackRef_IsTaggedInt(null_or_index)); - } - // _ITER_JUMP_TUPLE - { - PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); - (void)tuple_o; - assert(Py_TYPE(tuple_o) == &PyTuple_Type); - STAT_INC(FOR_ITER, hit); - if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { - null_or_index = PyStackRef_TagInt(-1); - JUMPBY(oparg + 1); - RECORD_DYNAMIC_JUMP_TAKEN(); - stack_pointer[-1] = null_or_index; - TRACING_DISPATCH(); - } - } - // _ITER_NEXT_TUPLE - { - PyObject *tuple_o = PyStackRef_AsPyObjectBorrow(iter); - assert(Py_TYPE(tuple_o) == &PyTuple_Type); - uintptr_t i = PyStackRef_UntagInt(null_or_index); - assert((size_t)i < (size_t)PyTuple_GET_SIZE(tuple_o)); - next = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(tuple_o, i)); - null_or_index = PyStackRef_IncrementTaggedIntNoOverflow(null_or_index); - } - stack_pointer[-1] = null_or_index; - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(GET_AITER) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = GET_AITER; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_AITER); - opcode = GET_AITER; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef obj; - _PyStackRef iter; - obj = stack_pointer[-1]; - unaryfunc getter = NULL; - PyObject *obj_o = PyStackRef_AsPyObjectBorrow(obj); - PyObject *iter_o; - PyTypeObject *type = Py_TYPE(obj_o); - if (type->tp_as_async != NULL) { - getter = type->tp_as_async->am_aiter; - } - if (getter == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, - "'async for' requires an object with " - "__aiter__ method, got %.100s", - type->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(obj); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - iter_o = (*getter)(obj_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(obj); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (iter_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - if (Py_TYPE(iter_o)->tp_as_async == NULL || - Py_TYPE(iter_o)->tp_as_async->am_anext == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, - "'async for' received an object from __aiter__ " - "that does not implement __anext__: %.100s", - Py_TYPE(iter_o)->tp_name); - Py_DECREF(iter_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[0] = iter; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(GET_ANEXT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = GET_ANEXT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_ANEXT); - opcode = GET_ANEXT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef aiter; - _PyStackRef awaitable; - aiter = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *awaitable_o = _PyEval_GetANext(PyStackRef_AsPyObjectBorrow(aiter)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (awaitable_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - awaitable = PyStackRef_FromPyObjectSteal(awaitable_o); - stack_pointer[0] = awaitable; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(GET_AWAITABLE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = GET_AWAITABLE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_AWAITABLE); - opcode = GET_AWAITABLE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef iterable; - _PyStackRef iter; - iterable = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = _PyEval_GetAwaitable(PyStackRef_AsPyObjectBorrow(iterable), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (iter_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - iter = PyStackRef_FromPyObjectSteal(iter_o); - stack_pointer[0] = iter; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(GET_ITER) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = GET_ITER; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_ITER); - opcode = GET_ITER; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef iterable; - _PyStackRef iter; - _PyStackRef index_or_null; - iterable = stack_pointer[-1]; - #ifdef Py_STATS - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_GatherStats_GetIter(iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - #endif - - PyTypeObject *tp = PyStackRef_TYPE(iterable); - if (tp == &PyTuple_Type || tp == &PyList_Type) { - iter = iterable; - index_or_null = PyStackRef_TagInt(0); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = PyObject_GetIter(PyStackRef_AsPyObjectBorrow(iterable)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (iter_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - iter = PyStackRef_FromPyObjectSteal(iter_o); - index_or_null = PyStackRef_NULL; - stack_pointer += 1; - } - stack_pointer[-1] = iter; - stack_pointer[0] = index_or_null; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(GET_LEN) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = GET_LEN; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_LEN); - opcode = GET_LEN; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef obj; - _PyStackRef len; - obj = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_ssize_t len_i = PyObject_Length(PyStackRef_AsPyObjectBorrow(obj)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (len_i < 0) { - TRACING_JUMP_TO_LABEL(error); - } - PyObject *len_o = PyLong_FromSsize_t(len_i); - if (len_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - len = PyStackRef_FromPyObjectSteal(len_o); - stack_pointer[0] = len; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(GET_YIELD_FROM_ITER) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = GET_YIELD_FROM_ITER; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(GET_YIELD_FROM_ITER); - opcode = GET_YIELD_FROM_ITER; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef iterable; - _PyStackRef iter; - iterable = stack_pointer[-1]; - PyObject *iterable_o = PyStackRef_AsPyObjectBorrow(iterable); - if (PyCoro_CheckExact(iterable_o)) { - if (!(_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_TypeError, - "cannot 'yield from' a coroutine object " - "in a non-coroutine generator"); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - iter = iterable; - } - else if (PyGen_CheckExact(iterable_o)) { - iter = iterable; - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *iter_o = PyObject_GetIter(iterable_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (iter_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - iter = PyStackRef_FromPyObjectSteal(iter_o); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = iterable; - iterable = iter; - stack_pointer[-1] = iterable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - stack_pointer[-1] = iter; - TRACING_DISPATCH(); - } - - TRACING_TARGET(IMPORT_FROM) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = IMPORT_FROM; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IMPORT_FROM); - opcode = IMPORT_FROM; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef from; - _PyStackRef res; - from = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyEval_ImportFrom(tstate, PyStackRef_AsPyObjectBorrow(from), name); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(IMPORT_NAME) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = IMPORT_NAME; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IMPORT_NAME); - opcode = IMPORT_NAME; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef level; - _PyStackRef fromlist; - _PyStackRef res; - fromlist = stack_pointer[-1]; - level = stack_pointer[-2]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = _PyEval_ImportName(tstate, frame, name, - PyStackRef_AsPyObjectBorrow(fromlist), - PyStackRef_AsPyObjectBorrow(level)); - _PyStackRef tmp = fromlist; - fromlist = PyStackRef_NULL; - stack_pointer[-1] = fromlist; - PyStackRef_CLOSE(tmp); - tmp = level; - level = PyStackRef_NULL; - stack_pointer[-2] = level; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_CALL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_CALL; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(INSTRUMENTED_CALL); - opcode = INSTRUMENTED_CALL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef func; - _PyStackRef maybe_self; - _PyStackRef *args; - _PyStackRef res; - /* Skip 3 cache entries */ - // _MAYBE_EXPAND_METHOD - { - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - self_or_null = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - _PyStackRef temp = callable; - callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-2 - oparg] = callable; - stack_pointer[-1 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - // _MONITOR_CALL - { - args = &stack_pointer[-oparg]; - maybe_self = self_or_null; - func = callable; - int is_meth = !PyStackRef_IsNull(maybe_self); - PyObject *function = PyStackRef_AsPyObjectBorrow(func); - PyObject *arg0; - if (is_meth) { - arg0 = PyStackRef_AsPyObjectBorrow(maybe_self); - } - else if (oparg) { - arg0 = PyStackRef_AsPyObjectBorrow(args[0]); - } - else { - arg0 = &_PyInstrumentation_MISSING; - } - stack_pointer[-2 - oparg] = func; - stack_pointer[-1 - oparg] = maybe_self; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg0 - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - } - // _DO_CALL - { - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - if (Py_TYPE(callable_o) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) - { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, callable, locals, - arguments, total_args, NULL, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (new_frame == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - frame->return_offset = 4u ; - TRACING_DISPATCH_INLINED(new_frame); - } - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - if (opcode == INSTRUMENTED_CALL) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); - if (res_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - } - assert((res_o != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp; - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-1 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-2 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_CALL_FUNCTION_EX) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_CALL_FUNCTION_EX; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_CALL_FUNCTION_EX); - opcode = INSTRUMENTED_CALL_FUNCTION_EX; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef func; - _PyStackRef callargs; - _PyStackRef func_st; - _PyStackRef null; - _PyStackRef callargs_st; - _PyStackRef kwargs_st; - _PyStackRef result; - // _MAKE_CALLARGS_A_TUPLE - { - callargs = stack_pointer[-2]; - func = stack_pointer[-4]; - PyObject *callargs_o = PyStackRef_AsPyObjectBorrow(callargs); - if (!PyTuple_CheckExact(callargs_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_Check_ArgsIterable(tstate, PyStackRef_AsPyObjectBorrow(func), callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *tuple_o = PySequence_Tuple(callargs_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (tuple_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - _PyStackRef temp = callargs; - callargs = PyStackRef_FromPyObjectSteal(tuple_o); - stack_pointer[-2] = callargs; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - // _DO_CALL_FUNCTION_EX - { - kwargs_st = stack_pointer[-1]; - callargs_st = callargs; - null = stack_pointer[-3]; - func_st = func; - (void)null; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_st); - EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_FUNCTION_EX, func); - PyObject *result_o; - assert(!_PyErr_Occurred(tstate)); - if (opcode == INSTRUMENTED_CALL_FUNCTION_EX) { - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - assert(PyTuple_CheckExact(callargs)); - PyObject *arg = PyTuple_GET_SIZE(callargs) > 0 ? - PyTuple_GET_ITEM(callargs, 0) : &_PyInstrumentation_MISSING; - stack_pointer[-2] = callargs_st; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!PyFunction_Check(func) && !PyMethod_Check(func)) { - if (result_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, func, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(result_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - } - } - else { - if (Py_TYPE(func) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)func)->vectorcall == _PyFunction_Vectorcall) { - PyObject *callargs = PyStackRef_AsPyObjectSteal(callargs_st); - assert(PyTuple_CheckExact(callargs)); - PyObject *kwargs = PyStackRef_IsNull(kwargs_st) ? NULL : PyStackRef_AsPyObjectSteal(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); - int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( - tstate, func_st, locals, - nargs, callargs, kwargs, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (new_frame == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - assert( 1u == 1); - frame->return_offset = 1; - TRACING_DISPATCH_INLINED(new_frame); - } - PyObject *callargs = PyStackRef_AsPyObjectBorrow(callargs_st); - assert(PyTuple_CheckExact(callargs)); - PyObject *kwargs = PyStackRef_AsPyObjectBorrow(kwargs_st); - assert(kwargs == NULL || PyDict_CheckExact(kwargs)); - stack_pointer[-2] = callargs_st; - _PyFrame_SetStackPointer(frame, stack_pointer); - result_o = PyObject_Call(func, callargs, kwargs); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(kwargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(callargs_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(func_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (result_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - result = PyStackRef_FromPyObjectSteal(result_o); - } - // _CHECK_PERIODIC_AT_END - { - stack_pointer[0] = result; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_CALL_KW) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_CALL_KW; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(INSTRUMENTED_CALL_KW); - opcode = INSTRUMENTED_CALL_KW; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef callable; - _PyStackRef self_or_null; - _PyStackRef *args; - _PyStackRef kwnames; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _MAYBE_EXPAND_METHOD_KW - { - self_or_null = stack_pointer[-2 - oparg]; - callable = stack_pointer[-3 - oparg]; - if (PyStackRef_TYPE(callable) == &PyMethod_Type && PyStackRef_IsNull(self_or_null)) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *self = ((PyMethodObject *)callable_o)->im_self; - self_or_null = PyStackRef_FromPyObjectNew(self); - PyObject *method = ((PyMethodObject *)callable_o)->im_func; - _PyStackRef temp = callable; - callable = PyStackRef_FromPyObjectNew(method); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(temp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - // _MONITOR_CALL_KW - { - args = &stack_pointer[-1 - oparg]; - int is_meth = !PyStackRef_IsNull(self_or_null); - PyObject *arg; - if (is_meth) { - arg = PyStackRef_AsPyObjectBorrow(self_or_null); - } - else if (args) { - arg = PyStackRef_AsPyObjectBorrow(args[0]); - } - else { - arg = &_PyInstrumentation_MISSING; - } - PyObject *function = PyStackRef_AsPyObjectBorrow(callable); - stack_pointer[-3 - oparg] = callable; - stack_pointer[-2 - oparg] = self_or_null; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, function, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - } - // _DO_CALL_KW - { - kwnames = stack_pointer[-1]; - args = &stack_pointer[-1 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); - PyObject *kwnames_o = PyStackRef_AsPyObjectBorrow(kwnames); - int total_args = oparg; - _PyStackRef *arguments = args; - if (!PyStackRef_IsNull(self_or_null)) { - arguments--; - total_args++; - } - int positional_args = total_args - (int)PyTuple_GET_SIZE(kwnames_o); - if (Py_TYPE(callable_o) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable_o)->vectorcall == _PyFunction_Vectorcall) - { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, callable, locals, - arguments, positional_args, kwnames_o, frame - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(kwnames); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (new_frame == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); - frame->return_offset = 4u ; - TRACING_DISPATCH_INLINED(new_frame); - } - STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); - if (CONVERSION_FAILED(args_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = kwnames; - kwnames = PyStackRef_NULL; - stack_pointer[-1] = kwnames; - PyStackRef_CLOSE(tmp); - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-2 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-3 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall( - callable_o, args_o, - positional_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - kwnames_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - STACKREFS_TO_PYOBJECTS_CLEANUP(args_o); - if (opcode == INSTRUMENTED_CALL_KW) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : PyStackRef_AsPyObjectBorrow(arguments[0]); - if (res_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable_o, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(res_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = kwnames; - kwnames = PyStackRef_NULL; - stack_pointer[-1] = kwnames; - PyStackRef_CLOSE(tmp); - for (int _i = oparg; --_i >= 0;) { - tmp = args[_i]; - args[_i] = PyStackRef_NULL; - PyStackRef_CLOSE(tmp); - } - tmp = self_or_null; - self_or_null = PyStackRef_NULL; - stack_pointer[-2 - oparg] = self_or_null; - PyStackRef_XCLOSE(tmp); - tmp = callable; - callable = PyStackRef_NULL; - stack_pointer[-3 - oparg] = callable; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3 - oparg; - assert(WITHIN_STACK_BOUNDS()); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_END_ASYNC_FOR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_END_ASYNC_FOR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_END_ASYNC_FOR); - opcode = INSTRUMENTED_END_ASYNC_FOR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef awaitable_st; - _PyStackRef exc_st; - // _MONITOR_END_ASYNC_FOR - { - assert((next_instr-oparg)->op.code == END_SEND || (next_instr-oparg)->op.code >= MIN_INSTRUMENTED_OPCODE); - INSTRUMENTED_JUMP(next_instr-oparg, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT); - } - // _END_ASYNC_FOR - { - exc_st = stack_pointer[-1]; - awaitable_st = stack_pointer[-2]; - JUMPBY(0); - (void)oparg; - PyObject *exc = PyStackRef_AsPyObjectBorrow(exc_st); - assert(exc && PyExceptionInstance_Check(exc)); - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = PyErr_GivenExceptionMatches(exc, PyExc_StopAsyncIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = exc_st; - exc_st = PyStackRef_NULL; - stack_pointer[-1] = exc_st; - PyStackRef_CLOSE(tmp); - tmp = awaitable_st; - awaitable_st = PyStackRef_NULL; - stack_pointer[-2] = awaitable_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - } - else { - Py_INCREF(exc); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, this_instr); - TRACING_JUMP_TO_LABEL(exception_unwind); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_END_FOR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_END_FOR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_END_FOR); - opcode = INSTRUMENTED_END_FOR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef receiver; - _PyStackRef value; - value = stack_pointer[-1]; - receiver = stack_pointer[-3]; - if (PyStackRef_GenCheck(receiver)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_END_SEND) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_END_SEND; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_END_SEND); - opcode = INSTRUMENTED_END_SEND; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef receiver; - _PyStackRef value; - _PyStackRef val; - value = stack_pointer[-1]; - receiver = stack_pointer[-2]; - PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); - if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - } - val = value; - stack_pointer[-2] = val; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(receiver); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_FOR_ITER) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_FOR_ITER; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_FOR_ITER); - opcode = INSTRUMENTED_FOR_ITER; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef iter; - _PyStackRef null_or_index; - _PyStackRef next; - /* Skip 1 cache entry */ - null_or_index = stack_pointer[-1]; - iter = stack_pointer[-2]; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (!PyStackRef_IsValid(item)) { - if (PyStackRef_IsError(item)) { - TRACING_JUMP_TO_LABEL(error); - } - JUMPBY(oparg + 1); - stack_pointer[-1] = null_or_index; - TRACING_DISPATCH(); - } - next = item; - INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); - stack_pointer[-1] = null_or_index; - stack_pointer[0] = next; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_INSTRUCTION) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_INSTRUCTION; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_INSTRUCTION); - opcode = INSTRUMENTED_INSTRUCTION; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyFrame_SetStackPointer(frame, stack_pointer); - int next_opcode = _Py_call_instrumentation_instruction( - tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (next_opcode < 0) { - TRACING_JUMP_TO_LABEL(error); - } - next_instr = this_instr; - if (_PyOpcode_Caches[next_opcode]) { - PAUSE_ADAPTIVE_COUNTER(next_instr[1].counter); - } - assert(next_opcode > 0 && next_opcode < 256); - opcode = next_opcode; - DISPATCH_GOTO(); - } - - TRACING_TARGET(INSTRUMENTED_JUMP_BACKWARD) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_JUMP_BACKWARD; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_JUMP_BACKWARD); - opcode = INSTRUMENTED_JUMP_BACKWARD; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - /* Skip 1 cache entry */ - // _CHECK_PERIODIC - { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - // _MONITOR_JUMP_BACKWARD - { - INSTRUMENTED_JUMP(this_instr, next_instr - oparg, PY_MONITORING_EVENT_JUMP); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_JUMP_FORWARD) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_JUMP_FORWARD; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_JUMP_FORWARD); - opcode = INSTRUMENTED_JUMP_FORWARD; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_JUMP); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_LINE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_LINE; - (void)(opcode); - #endif - _Py_CODEUNIT* const prev_instr = frame->instr_ptr; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_LINE); - opcode = INSTRUMENTED_LINE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - int original_opcode = 0; - if (tstate->tracing) { - PyCodeObject *code = _PyFrame_GetCode(frame); - int index = (int)(this_instr - _PyFrame_GetBytecode(frame)); - original_opcode = code->_co_monitoring->lines->data[index*code->_co_monitoring->lines->bytes_per_entry]; - next_instr = this_instr; - } else { - _PyFrame_SetStackPointer(frame, stack_pointer); - original_opcode = _Py_call_instrumentation_line( - tstate, frame, this_instr, prev_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (original_opcode < 0) { - next_instr = this_instr+1; - TRACING_JUMP_TO_LABEL(error); - } - next_instr = frame->instr_ptr; - if (next_instr != this_instr) { - TRACING_DISPATCH(); - } - } - if (_PyOpcode_Caches[original_opcode]) { - _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); - PAUSE_ADAPTIVE_COUNTER(cache->counter); - } - opcode = original_opcode; - DISPATCH_GOTO(); - } - - TRACING_TARGET(INSTRUMENTED_LOAD_SUPER_ATTR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_LOAD_SUPER_ATTR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_LOAD_SUPER_ATTR); - opcode = INSTRUMENTED_LOAD_SUPER_ATTR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef global_super_st; - _PyStackRef class_st; - _PyStackRef self_st; - _PyStackRef attr; - _PyStackRef *null; - /* Skip 1 cache entry */ - // _LOAD_SUPER_ATTR - { - self_st = stack_pointer[-1]; - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = self_st; - self_st = PyStackRef_NULL; - stack_pointer[-1] = self_st; - PyStackRef_CLOSE(tmp); - tmp = class_st; - class_st = PyStackRef_NULL; - stack_pointer[-2] = class_st; - PyStackRef_CLOSE(tmp); - tmp = global_super_st; - global_super_st = PyStackRef_NULL; - stack_pointer[-3] = global_super_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - } - PyObject *stack[] = {class, self}; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - if (super == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(super); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = self_st; - self_st = PyStackRef_NULL; - stack_pointer[-1] = self_st; - PyStackRef_CLOSE(tmp); - tmp = class_st; - class_st = PyStackRef_NULL; - stack_pointer[-2] = class_st; - PyStackRef_CLOSE(tmp); - tmp = global_super_st; - global_super_st = PyStackRef_NULL; - stack_pointer[-3] = global_super_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - if (super == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = PyObject_GetAttr(super, name); - Py_DECREF(super); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - attr = PyStackRef_FromPyObjectSteal(attr_o); - } - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[1]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer[0] = attr; - stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_NOT_TAKEN) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_NOT_TAKEN; - (void)(opcode); - #endif - _Py_CODEUNIT* const prev_instr = frame->instr_ptr; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_NOT_TAKEN); - opcode = INSTRUMENTED_NOT_TAKEN; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - (void)this_instr; - INSTRUMENTED_JUMP(prev_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_POP_ITER) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_POP_ITER; - (void)(opcode); - #endif - _Py_CODEUNIT* const prev_instr = frame->instr_ptr; - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_POP_ITER); - opcode = INSTRUMENTED_POP_ITER; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef iter; - _PyStackRef index_or_null; - index_or_null = stack_pointer[-1]; - iter = stack_pointer[-2]; - (void)index_or_null; - INSTRUMENTED_JUMP(prev_instr, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iter); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_POP_JUMP_IF_FALSE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_FALSE); - opcode = INSTRUMENTED_POP_JUMP_IF_FALSE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef cond; - /* Skip 1 cache entry */ - cond = stack_pointer[-1]; - assert(PyStackRef_BoolCheck(cond)); - int jump = PyStackRef_IsFalse(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); - if (jump) { - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_POP_JUMP_IF_NONE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NONE); - opcode = INSTRUMENTED_POP_JUMP_IF_NONE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - /* Skip 1 cache entry */ - value = stack_pointer[-1]; - int jump = PyStackRef_IsNone(value); - RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); - if (jump) { - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); - } - else { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += 1; - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_POP_JUMP_IF_NOT_NONE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_NOT_NONE); - opcode = INSTRUMENTED_POP_JUMP_IF_NOT_NONE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - /* Skip 1 cache entry */ - value = stack_pointer[-1]; - int jump = !PyStackRef_IsNone(value); - RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); - if (jump) { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); - } - else { - stack_pointer += -1; - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_POP_JUMP_IF_TRUE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(INSTRUMENTED_POP_JUMP_IF_TRUE); - opcode = INSTRUMENTED_POP_JUMP_IF_TRUE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef cond; - /* Skip 1 cache entry */ - cond = stack_pointer[-1]; - assert(PyStackRef_BoolCheck(cond)); - int jump = PyStackRef_IsTrue(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, jump); - if (jump) { - INSTRUMENTED_JUMP(this_instr, next_instr + oparg, PY_MONITORING_EVENT_BRANCH_RIGHT); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_RESUME) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_RESUME; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_RESUME); - opcode = INSTRUMENTED_RESUME; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - // _LOAD_BYTECODE - { - #ifdef Py_GIL_DISABLED - if (frame->tlbc_index != - ((_PyThreadStateImpl *)tstate)->tlbc_index) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_CODEUNIT *bytecode = - _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (bytecode == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); - frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; - frame->instr_ptr = bytecode + off; - next_instr = frame->instr_ptr; - TRACING_DISPATCH(); - } - #endif - } - // _MAYBE_INSTRUMENT - { - #ifdef Py_GIL_DISABLED - - int check_instrumentation = 1; - #else - int check_instrumentation = (tstate->tracing == 0); - #endif - if (check_instrumentation) { - uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - if (code_version != global_version) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - next_instr = this_instr; - TRACING_DISPATCH(); - } - } - } - // _CHECK_PERIODIC_IF_NOT_YIELD_FROM - { - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - } - // _MONITOR_RESUME - { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation( - tstate, oparg > 0, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - if (frame->instr_ptr != this_instr) { - next_instr = frame->instr_ptr; - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_RETURN_VALUE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_RETURN_VALUE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_RETURN_VALUE); - opcode = INSTRUMENTED_RETURN_VALUE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef val; - _PyStackRef retval; - _PyStackRef res; - // _RETURN_VALUE_EVENT - { - val = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_RETURN, - frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - } - // _RETURN_VALUE - { - retval = val; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - _PyStackRef temp = PyStackRef_MakeHeapSafe(retval); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(STACK_LEVEL() == 0); - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - stack_pointer = _PyFrame_GetStackPointer(frame); - #if TIER_ONE - LOAD_IP(frame->return_offset); - #endif - #if TIER_TWO - TIER2_STORE_IP(frame->return_offset); - #endif - res = temp; - LLTRACE_RESUME_FRAME(); - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INSTRUMENTED_YIELD_VALUE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INSTRUMENTED_YIELD_VALUE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); - opcode = INSTRUMENTED_YIELD_VALUE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef val; - _PyStackRef retval; - _PyStackRef value; - // _YIELD_VALUE_EVENT - { - val = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_arg( - tstate, PY_MONITORING_EVENT_PY_YIELD, - frame, this_instr, PyStackRef_AsPyObjectBorrow(val)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - if (frame->instr_ptr != this_instr) { - next_instr = frame->instr_ptr; - TRACING_DISPATCH(); - } - } - // _YIELD_VALUE - { - retval = val; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); - assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); - assert(oparg == 0 || oparg == 1); - gen->gi_frame_state = FRAME_SUSPENDED + oparg; - _PyStackRef temp = retval; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; - assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - #if TIER_ONE - assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || - frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); - #endif - stack_pointer = _PyFrame_GetStackPointer(frame); - #if TIER_ONE - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - #endif - #if TIER_TWO - TIER2_STORE_IP(1 + INLINE_CACHE_ENTRIES_SEND); - #endif - value = PyStackRef_MakeHeapSafe(temp); - LLTRACE_RESUME_FRAME(); - } - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(INTERPRETER_EXIT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = INTERPRETER_EXIT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(INTERPRETER_EXIT); - opcode = INTERPRETER_EXIT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef retval; - retval = stack_pointer[-1]; - assert(frame->owner == FRAME_OWNED_BY_INTERPRETER); - assert(_PyFrame_IsIncomplete(frame)); - tstate->current_frame = frame->previous; - assert(!_PyErr_Occurred(tstate)); - PyObject *result = PyStackRef_AsPyObjectSteal(retval); - #if !_Py_TAIL_CALL_INTERP - assert(frame == &entry.frame); - #endif - #ifdef _Py_TIER2 - _PyStackRef executor = frame->localsplus[0]; - assert(tstate->current_executor == NULL); - if (!PyStackRef_IsNull(executor)) { - tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(executor); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += 1; - } - #endif - LLTRACE_RESUME_FRAME(); - return result; - } - - TRACING_TARGET(IS_OP) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = IS_OP; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(IS_OP); - opcode = IS_OP; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef left; - _PyStackRef right; - _PyStackRef b; - right = stack_pointer[-1]; - left = stack_pointer[-2]; - int res = Py_Is(PyStackRef_AsPyObjectBorrow(left), PyStackRef_AsPyObjectBorrow(right)) ^ oparg; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = right; - right = PyStackRef_NULL; - stack_pointer[-1] = right; - PyStackRef_CLOSE(tmp); - tmp = left; - left = PyStackRef_NULL; - stack_pointer[-2] = left; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - b = res ? PyStackRef_True : PyStackRef_False; - stack_pointer[0] = b; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(JUMP_BACKWARD) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = JUMP_BACKWARD; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(JUMP_BACKWARD); - PREDICTED_TRACING_JUMP_BACKWARD:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - opcode = JUMP_BACKWARD; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - /* Skip 1 cache entry */ - // _SPECIALIZE_JUMP_BACKWARD - { - #if ENABLE_SPECIALIZATION_FT - if (this_instr->op.code == JUMP_BACKWARD) { - uint8_t desired = tstate->interp->jit ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; - FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, desired); - next_instr = this_instr; - DISPATCH_SAME_OPARG() - } - #endif - } - // _CHECK_PERIODIC - { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - // _JUMP_BACKWARD_NO_INTERRUPT - { - assert(oparg <= INSTR_OFFSET()); - JUMPBY(-oparg); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(JUMP_BACKWARD_JIT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = JUMP_BACKWARD_JIT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(JUMP_BACKWARD_JIT); - opcode = JUMP_BACKWARD_JIT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(1 == 1, "incorrect cache size"); - /* Skip 1 cache entry */ - // _CHECK_PERIODIC - { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - // _JUMP_BACKWARD_NO_INTERRUPT - { - assert(oparg <= INSTR_OFFSET()); - JUMPBY(-oparg); - } - // _JIT - { - #ifdef _Py_TIER2 - _Py_BackoffCounter counter = this_instr[1].counter; - if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && - this_instr->op.code == JUMP_BACKWARD_JIT && - next_instr->op.code != ENTER_EXECUTOR) { - if (tstate->interp->jit_state.code_buffer == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - tstate->interp->jit_state.code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (tstate->interp->jit_state.code_buffer == NULL) { - TRACING_DISPATCH(); - } - } - int _is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL); - if (!_is_sys_tracing) { - _Py_CODEUNIT *insert_exec_at = this_instr; - while (oparg > 255) { - oparg >>= 8; - insert_exec_at--; - } - _PyJit_InitializeTracing(tstate, frame, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL); - ENTER_TRACING(); - } - int _jump_taken = false; - PyCodeObject *old_code = _PyFrame_GetCode(frame); - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - int _old_stack_level = 0; - TRACING_DISPATCH(); - } - else { - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - } - #endif - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(JUMP_BACKWARD_NO_INTERRUPT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = JUMP_BACKWARD_NO_INTERRUPT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(JUMP_BACKWARD_NO_INTERRUPT); - opcode = JUMP_BACKWARD_NO_INTERRUPT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - assert(oparg <= INSTR_OFFSET()); - JUMPBY(-oparg); - TRACING_DISPATCH(); - } - - TRACING_TARGET(JUMP_BACKWARD_NO_JIT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = JUMP_BACKWARD_NO_JIT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(JUMP_BACKWARD_NO_JIT); - opcode = JUMP_BACKWARD_NO_JIT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(1 == 1, "incorrect cache size"); - /* Skip 1 cache entry */ - // _CHECK_PERIODIC - { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - // _JUMP_BACKWARD_NO_INTERRUPT - { - assert(oparg <= INSTR_OFFSET()); - JUMPBY(-oparg); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(JUMP_FORWARD) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = JUMP_FORWARD; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(JUMP_FORWARD); - opcode = JUMP_FORWARD; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - JUMPBY(oparg); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LIST_APPEND) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LIST_APPEND; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LIST_APPEND); - opcode = LIST_APPEND; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef list; - _PyStackRef v; - v = stack_pointer[-1]; - list = stack_pointer[-2 - (oparg-1)]; - int err = _PyList_AppendTakeRef((PyListObject *)PyStackRef_AsPyObjectBorrow(list), - PyStackRef_AsPyObjectSteal(v)); - if (err < 0) { - TRACING_JUMP_TO_LABEL(pop_1_error); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LIST_EXTEND) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LIST_EXTEND; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LIST_EXTEND); - opcode = LIST_EXTEND; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef list_st; - _PyStackRef iterable_st; - iterable_st = stack_pointer[-1]; - list_st = stack_pointer[-2 - (oparg-1)]; - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - PyObject *iterable = PyStackRef_AsPyObjectBorrow(iterable_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (none_val == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_TypeError); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches && - (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) - { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Clear(tstate); - _PyErr_Format(tstate, PyExc_TypeError, - "Value after * must be an iterable, not %.200s", - Py_TYPE(iterable)->tp_name); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - assert(Py_IsNone(none_val)); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR); - PREDICTED_TRACING_LOAD_ATTR:; - _Py_CODEUNIT* const this_instr = next_instr - 10; - (void)this_instr; - opcode = LOAD_ATTR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef owner; - _PyStackRef *attr; - _PyStackRef *self_or_null; - // _SPECIALIZE_LOAD_ATTR - { - owner = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadAttr(owner, next_instr, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() - } - OPCODE_DEFERRED_INC(LOAD_ATTR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 8 cache entries */ - // _LOAD_ATTR - { - attr = &stack_pointer[-1]; - self_or_null = &stack_pointer[0]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - if (oparg & 1) { - *attr = PyStackRef_NULL; - _PyFrame_SetStackPointer(frame, stack_pointer); - int is_meth = _PyObject_GetMethodStackRef(tstate, PyStackRef_AsPyObjectBorrow(owner), name, attr); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (is_meth) { - assert(!PyStackRef_IsNull(*attr)); - self_or_null[0] = owner; - } - else { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(*attr)) { - TRACING_JUMP_TO_LABEL(error); - } - self_or_null[0] = PyStackRef_NULL; - stack_pointer += 1; - } - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - *attr = PyStackRef_FromPyObjectSteal(attr_o); - stack_pointer += 1; - } - } - stack_pointer += (oparg&1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_CLASS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_CLASS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_CLASS); - opcode = LOAD_ATTR_CLASS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef *null; - /* Skip 1 cache entry */ - // _CHECK_ATTR_CLASS - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - if (!PyType_Check(owner_o)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_CLASS - { - PyObject *descr = read_obj(&this_instr[6].cache); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - attr = PyStackRef_FromPyObjectNew(descr); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = owner; - owner = attr; - stack_pointer[-1] = owner; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[0]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_CLASS_WITH_METACLASS_CHECK; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_CLASS_WITH_METACLASS_CHECK); - opcode = LOAD_ATTR_CLASS_WITH_METACLASS_CHECK; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef *null; - /* Skip 1 cache entry */ - // _CHECK_ATTR_CLASS - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - if (!PyType_Check(owner_o)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _GUARD_TYPE_VERSION - { - uint32_t type_version = read_u32(&this_instr[4].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _LOAD_ATTR_CLASS - { - PyObject *descr = read_obj(&this_instr[6].cache); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - attr = PyStackRef_FromPyObjectNew(descr); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = owner; - owner = attr; - stack_pointer[-1] = owner; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[0]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN); - opcode = LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - /* Skip 1 cache entry */ - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - uint32_t func_version = read_u32(&this_instr[4].cache); - PyObject *getattribute = read_obj(&this_instr[6].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert((oparg & 1) == 0); - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - PyTypeObject *cls = Py_TYPE(owner_o); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); - PyFunctionObject *f = (PyFunctionObject *)getattribute; - assert(func_version != 0); - if (f->func_version != func_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - PyCodeObject *code = (PyCodeObject *)f->func_code; - assert(code->co_argcount == 2); - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - STAT_INC(LOAD_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( - tstate, PyStackRef_FromPyObjectNew(f), 2, frame); - new_frame->localsplus[0] = owner; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - new_frame->localsplus[1] = PyStackRef_FromPyObjectNew(name); - frame->return_offset = 10u ; - TRACING_DISPATCH_INLINED(new_frame); - } - - TRACING_TARGET(LOAD_ATTR_INSTANCE_VALUE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_INSTANCE_VALUE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_INSTANCE_VALUE); - opcode = LOAD_ATTR_INSTANCE_VALUE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef *null; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _CHECK_MANAGED_OBJECT_HAS_VALUES - { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_dictoffset < 0); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _LOAD_ATTR_INSTANCE_VALUE - { - uint16_t offset = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); - PyObject *attr_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(*value_ptr); - if (attr_o == NULL) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - #ifdef Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(value_ptr, attr_o, &attr); - if (!increfed) { - if (true) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - #else - attr = PyStackRef_FromPyObjectNew(attr_o); - #endif - STAT_INC(LOAD_ATTR, hit); - stack_pointer[-1] = attr; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - /* Skip 5 cache entries */ - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[0]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_METHOD_LAZY_DICT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_METHOD_LAZY_DICT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_LAZY_DICT); - opcode = LOAD_ATTR_METHOD_LAZY_DICT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _CHECK_ATTR_METHOD_LAZY_DICT - { - uint16_t dictoffset = read_u16(&this_instr[4].cache); - char *ptr = ((char *)PyStackRef_AsPyObjectBorrow(owner)) + MANAGED_DICT_OFFSET + dictoffset; - PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*(PyObject **)ptr); - if (dict != NULL) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - /* Skip 1 cache entry */ - // _LOAD_ATTR_METHOD_LAZY_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - } - stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_METHOD_NO_DICT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_METHOD_NO_DICT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_NO_DICT); - opcode = LOAD_ATTR_METHOD_NO_DICT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_METHOD_NO_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - } - stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_METHOD_WITH_VALUES) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_METHOD_WITH_VALUES; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_METHOD_WITH_VALUES); - opcode = LOAD_ATTR_METHOD_WITH_VALUES; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef self; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT - { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - PyDictValues *ivs = _PyObject_InlineValues(owner_o); - if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _GUARD_KEYS_VERSION - { - uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _LOAD_ATTR_METHOD_WITH_VALUES - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert(oparg & 1); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - assert(_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)); - attr = PyStackRef_FromPyObjectNew(descr); - self = owner; - } - stack_pointer[-1] = attr; - stack_pointer[0] = self; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_MODULE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_MODULE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_MODULE); - opcode = LOAD_ATTR_MODULE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef *null; - /* Skip 1 cache entry */ - // _LOAD_ATTR_MODULE - { - owner = stack_pointer[-1]; - uint32_t dict_version = read_u32(&this_instr[2].cache); - uint16_t index = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; - assert(dict != NULL); - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - assert(keys->dk_kind == DICT_KEYS_UNICODE); - assert(index < FT_ATOMIC_LOAD_SSIZE_RELAXED(keys->dk_nentries)); - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(keys) + index; - PyObject *attr_o = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_value); - if (attr_o == NULL) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - #ifdef Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); - if (!increfed) { - if (true) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - #else - attr = PyStackRef_FromPyObjectNew(attr_o); - #endif - STAT_INC(LOAD_ATTR, hit); - stack_pointer[-1] = attr; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - /* Skip 5 cache entries */ - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[0]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_NONDESCRIPTOR_NO_DICT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_NONDESCRIPTOR_NO_DICT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_NO_DICT); - opcode = LOAD_ATTR_NONDESCRIPTOR_NO_DICT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_NONDESCRIPTOR_NO_DICT - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); - assert(Py_TYPE(PyStackRef_AsPyObjectBorrow(owner))->tp_dictoffset == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - attr = PyStackRef_FromPyObjectNew(descr); - } - stack_pointer[0] = attr; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES); - opcode = LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT - { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - PyDictValues *ivs = _PyObject_InlineValues(owner_o); - if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _GUARD_KEYS_VERSION - { - uint32_t keys_version = read_u32(&this_instr[4].cache); - PyTypeObject *owner_cls = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - PyHeapTypeObject *owner_heap_type = (PyHeapTypeObject *)owner_cls; - PyDictKeysObject *keys = owner_heap_type->ht_cached_keys; - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES - { - PyObject *descr = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); - STAT_INC(LOAD_ATTR, hit); - assert(descr != NULL); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - attr = PyStackRef_FromPyObjectNew(descr); - } - stack_pointer[0] = attr; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_PROPERTY) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_PROPERTY; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_PROPERTY); - opcode = LOAD_ATTR_PROPERTY; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - /* Skip 2 cache entries */ - // _LOAD_ATTR_PROPERTY_FRAME - { - PyObject *fget = read_obj(&this_instr[6].cache); - assert((oparg & 1) == 0); - assert(Py_IS_TYPE(fget, &PyFunction_Type)); - PyFunctionObject *f = (PyFunctionObject *)fget; - PyCodeObject *code = (PyCodeObject *)f->func_code; - if ((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - if (code->co_kwonlyargcount) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - if (code->co_argcount != 1) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - STAT_INC(LOAD_ATTR, hit); - _PyInterpreterFrame *pushed_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); - pushed_frame->localsplus[0] = owner; - new_frame = PyStackRef_Wrap(pushed_frame); - } - // _SAVE_RETURN_OFFSET - { - #if TIER_ONE - frame->return_offset = (uint16_t)(next_instr - this_instr); - #endif - #if TIER_TWO - frame->return_offset = oparg; - #endif - } - // _PUSH_FRAME - { - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_SLOT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_SLOT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_SLOT); - opcode = LOAD_ATTR_SLOT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef *null; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _LOAD_ATTR_SLOT - { - uint16_t index = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - PyObject **addr = (PyObject **)((char *)owner_o + index); - PyObject *attr_o = FT_ATOMIC_LOAD_PTR(*addr); - if (attr_o == NULL) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - #ifdef Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(addr, attr_o, &attr); - if (!increfed) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - #else - attr = PyStackRef_FromPyObjectNew(attr_o); - #endif - STAT_INC(LOAD_ATTR, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = owner; - owner = attr; - stack_pointer[-1] = owner; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - /* Skip 5 cache entries */ - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[0]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_ATTR_WITH_HINT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_ATTR_WITH_HINT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 10; - INSTRUCTION_STATS(LOAD_ATTR_WITH_HINT); - opcode = LOAD_ATTR_WITH_HINT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_LOAD_ATTR == 9, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef attr; - _PyStackRef *null; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - // _LOAD_ATTR_WITH_HINT - { - uint16_t hint = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - if (dict == NULL) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - PyDictKeysObject *dk = FT_ATOMIC_LOAD_PTR(dict->ma_keys); - assert(PyDict_CheckExact((PyObject *)dict)); - #ifdef Py_GIL_DISABLED - if (!_Py_IsOwnedByCurrentThread((PyObject *)dict) && !_PyObject_GC_IS_SHARED(dict)) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - #endif - PyObject *attr_o; - if (hint >= (size_t)FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_nentries)) { - if (true) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - if (dk->dk_kind != DICT_KEYS_UNICODE) { - if (true) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dk) + hint; - if (FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_key) != name) { - if (true) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - attr_o = FT_ATOMIC_LOAD_PTR(ep->me_value); - if (attr_o == NULL) { - if (true) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - STAT_INC(LOAD_ATTR, hit); - #ifdef Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(&ep->me_value, attr_o, &attr); - if (!increfed) { - if (true) { - UPDATE_MISS_STATS(LOAD_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_ATTR); - } - } - #else - attr = PyStackRef_FromPyObjectNew(attr_o); - #endif - stack_pointer[-1] = attr; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - /* Skip 5 cache entries */ - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[0]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer += (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_BUILD_CLASS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_BUILD_CLASS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_BUILD_CLASS); - opcode = LOAD_BUILD_CLASS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef bc; - PyObject *bc_o; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(BUILTINS(), &_Py_ID(__build_class__), &bc_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - if (bc_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_NameError, - "__build_class__ not found"); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - bc = PyStackRef_FromPyObjectSteal(bc_o); - stack_pointer[0] = bc; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_COMMON_CONSTANT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_COMMON_CONSTANT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_COMMON_CONSTANT); - opcode = LOAD_COMMON_CONSTANT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - assert(oparg < NUM_COMMON_CONSTANTS); - value = PyStackRef_FromPyObjectNew(tstate->interp->common_consts[oparg]); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_CONST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_CONST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_CONST); - opcode = LOAD_CONST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg); - value = PyStackRef_FromPyObjectBorrow(obj); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_DEREF) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_DEREF; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_DEREF); - opcode = LOAD_DEREF; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - _PyFrame_SetStackPointer(frame, stack_pointer); - value = _PyCell_GetStackRef(cell); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(value)) { - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_FAST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_FAST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST); - opcode = LOAD_FAST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_DUP(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_FAST_AND_CLEAR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_FAST_AND_CLEAR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_AND_CLEAR); - opcode = LOAD_FAST_AND_CLEAR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - value = GETLOCAL(oparg); - GETLOCAL(oparg) = PyStackRef_NULL; - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_FAST_BORROW) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_FAST_BORROW; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_BORROW); - opcode = LOAD_FAST_BORROW; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - assert(!PyStackRef_IsNull(GETLOCAL(oparg))); - value = PyStackRef_Borrow(GETLOCAL(oparg)); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_FAST_BORROW_LOAD_FAST_BORROW) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_FAST_BORROW_LOAD_FAST_BORROW; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_BORROW_LOAD_FAST_BORROW); - opcode = LOAD_FAST_BORROW_LOAD_FAST_BORROW; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value1; - _PyStackRef value2; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - value1 = PyStackRef_Borrow(GETLOCAL(oparg1)); - value2 = PyStackRef_Borrow(GETLOCAL(oparg2)); - stack_pointer[0] = value1; - stack_pointer[1] = value2; - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_FAST_CHECK) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_FAST_CHECK; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_CHECK); - opcode = LOAD_FAST_CHECK; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - _PyStackRef value_s = GETLOCAL(oparg); - if (PyStackRef_IsNull(value_s)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError, - UNBOUNDLOCAL_ERROR_MSG, - PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg) - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - value = PyStackRef_DUP(value_s); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_FAST_LOAD_FAST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_FAST_LOAD_FAST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FAST_LOAD_FAST); - opcode = LOAD_FAST_LOAD_FAST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value1; - _PyStackRef value2; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - value1 = PyStackRef_DUP(GETLOCAL(oparg1)); - value2 = PyStackRef_DUP(GETLOCAL(oparg2)); - stack_pointer[0] = value1; - stack_pointer[1] = value2; - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_FROM_DICT_OR_DEREF) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_FROM_DICT_OR_DEREF; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FROM_DICT_OR_DEREF); - opcode = LOAD_FROM_DICT_OR_DEREF; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef class_dict_st; - _PyStackRef value; - class_dict_st = stack_pointer[-1]; - PyObject *value_o; - PyObject *name; - PyObject *class_dict = PyStackRef_AsPyObjectBorrow(class_dict_st); - assert(class_dict); - assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); - name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(class_dict, name, &value_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - if (!value_o) { - PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - value_o = PyCell_GetRef(cell); - if (value_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(class_dict_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - value = PyStackRef_FromPyObjectSteal(value_o); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_FROM_DICT_OR_GLOBALS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_FROM_DICT_OR_GLOBALS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_FROM_DICT_OR_GLOBALS); - opcode = LOAD_FROM_DICT_OR_GLOBALS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef mod_or_class_dict; - _PyStackRef v; - mod_or_class_dict = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *v_o; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(PyStackRef_AsPyObjectBorrow(mod_or_class_dict), name, &v_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(mod_or_class_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - if (v_o == NULL) { - if (PyDict_CheckExact(GLOBALS()) - && PyDict_CheckExact(BUILTINS())) - { - _PyFrame_SetStackPointer(frame, stack_pointer); - v_o = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), - (PyDictObject *)BUILTINS(), - name); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (v_o == NULL) { - if (!_PyErr_Occurred(tstate)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_JUMP_TO_LABEL(error); - } - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(GLOBALS(), name, &v_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - if (v_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(BUILTINS(), name, &v_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - if (v_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - } - } - } - v = PyStackRef_FromPyObjectSteal(v_o); - stack_pointer[0] = v; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_GLOBAL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_GLOBAL; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL); - PREDICTED_TRACING_LOAD_GLOBAL:; - _Py_CODEUNIT* const this_instr = next_instr - 5; - (void)this_instr; - opcode = LOAD_GLOBAL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef *res; - _PyStackRef *null; - // _SPECIALIZE_LOAD_GLOBAL - { - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() - } - OPCODE_DEFERRED_INC(LOAD_GLOBAL); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 1 cache entry */ - /* Skip 1 cache entry */ - /* Skip 1 cache entry */ - // _LOAD_GLOBAL - { - res = &stack_pointer[0]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_LoadGlobalStackRef(GLOBALS(), BUILTINS(), name, res); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(*res)) { - TRACING_JUMP_TO_LABEL(error); - } - } - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[1]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_GLOBAL_BUILTIN) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_GLOBAL_BUILTIN; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL_BUILTIN); - opcode = LOAD_GLOBAL_BUILTIN; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); - _PyStackRef res; - _PyStackRef *null; - /* Skip 1 cache entry */ - // _GUARD_GLOBALS_VERSION - { - uint16_t version = read_u16(&this_instr[2].cache); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - if (!PyDict_CheckExact(dict)) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); - } - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); - } - assert(DK_IS_UNICODE(keys)); - } - // _LOAD_GLOBAL_BUILTINS - { - uint16_t version = read_u16(&this_instr[3].cache); - uint16_t index = read_u16(&this_instr[4].cache); - PyDictObject *dict = (PyDictObject *)BUILTINS(); - if (!PyDict_CheckExact(dict)) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); - } - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); - } - assert(DK_IS_UNICODE(keys)); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); - PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - if (res_o == NULL) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); - } - #if Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - if (!increfed) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); - } - #else - res = PyStackRef_FromPyObjectNew(res_o); - #endif - STAT_INC(LOAD_GLOBAL, hit); - } - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[1]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer[0] = res; - stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_GLOBAL_MODULE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_GLOBAL_MODULE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(LOAD_GLOBAL_MODULE); - opcode = LOAD_GLOBAL_MODULE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); - _PyStackRef res; - _PyStackRef *null; - /* Skip 1 cache entry */ - // _NOP - { - } - // _LOAD_GLOBAL_MODULE - { - uint16_t version = read_u16(&this_instr[2].cache); - uint16_t index = read_u16(&this_instr[4].cache); - PyDictObject *dict = (PyDictObject *)GLOBALS(); - if (!PyDict_CheckExact(dict)) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); - } - PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); - if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); - } - assert(DK_IS_UNICODE(keys)); - PyDictUnicodeEntry *entries = DK_UNICODE_ENTRIES(keys); - assert(index < DK_SIZE(keys)); - PyObject *res_o = FT_ATOMIC_LOAD_PTR_RELAXED(entries[index].me_value); - if (res_o == NULL) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); - } - #if Py_GIL_DISABLED - int increfed = _Py_TryIncrefCompareStackRef(&entries[index].me_value, res_o, &res); - if (!increfed) { - UPDATE_MISS_STATS(LOAD_GLOBAL); - assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - JUMP_TO_PREDICTED(TRACING_LOAD_GLOBAL); - } - #else - res = PyStackRef_FromPyObjectNew(res_o); - #endif - STAT_INC(LOAD_GLOBAL, hit); - } - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[1]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer[0] = res; - stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_LOCALS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_LOCALS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_LOCALS); - opcode = LOAD_LOCALS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef locals; - PyObject *l = LOCALS(); - if (l == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetString(tstate, PyExc_SystemError, - "no locals found"); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - locals = PyStackRef_FromPyObjectNew(l); - stack_pointer[0] = locals; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_NAME) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_NAME; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_NAME); - opcode = LOAD_NAME; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef v; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *v_o = _PyEval_LoadName(tstate, frame, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (v_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - v = PyStackRef_FromPyObjectSteal(v_o); - stack_pointer[0] = v; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_SMALL_INT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_SMALL_INT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_SMALL_INT); - opcode = LOAD_SMALL_INT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - assert(oparg < _PY_NSMALLPOSINTS); - PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg]; - value = PyStackRef_FromPyObjectBorrow(obj); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_SPECIAL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_SPECIAL; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(LOAD_SPECIAL); - opcode = LOAD_SPECIAL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef self; - _PyStackRef *method_and_self; - // _INSERT_NULL - { - self = stack_pointer[-1]; - method_and_self = &stack_pointer[-1]; - method_and_self[1] = self; - method_and_self[0] = PyStackRef_NULL; - } - // _LOAD_SPECIAL - { - method_and_self = &stack_pointer[-1]; - PyObject *name = _Py_SpecialMethods[oparg].name; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyObject_LookupSpecialMethod(name, method_and_self); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err <= 0) { - if (err == 0) { - PyObject *owner = PyStackRef_AsPyObjectBorrow(method_and_self[1]); - _PyFrame_SetStackPointer(frame, stack_pointer); - const char *errfmt = _PyEval_SpecialMethodCanSuggest(owner, oparg) - ? _Py_SpecialMethods[oparg].error_suggestion - : _Py_SpecialMethods[oparg].error; - stack_pointer = _PyFrame_GetStackPointer(frame); - assert(!_PyErr_Occurred(tstate)); - assert(errfmt != NULL); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_TypeError, errfmt, owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_SUPER_ATTR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_SUPER_ATTR; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR); - PREDICTED_TRACING_LOAD_SUPER_ATTR:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - opcode = LOAD_SUPER_ATTR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef global_super_st; - _PyStackRef class_st; - _PyStackRef self_st; - _PyStackRef attr; - _PyStackRef *null; - // _SPECIALIZE_LOAD_SUPER_ATTR - { - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - int load_method = oparg & 1; - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() - } - OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _LOAD_SUPER_ATTR - { - self_st = stack_pointer[-1]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = self_st; - self_st = PyStackRef_NULL; - stack_pointer[-1] = self_st; - PyStackRef_CLOSE(tmp); - tmp = class_st; - class_st = PyStackRef_NULL; - stack_pointer[-2] = class_st; - PyStackRef_CLOSE(tmp); - tmp = global_super_st; - global_super_st = PyStackRef_NULL; - stack_pointer[-3] = global_super_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - TRACING_JUMP_TO_LABEL(error); - } - } - PyObject *stack[] = {class, self}; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - if (super == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, global_super, arg); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_CLEAR(super); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = self_st; - self_st = PyStackRef_NULL; - stack_pointer[-1] = self_st; - PyStackRef_CLOSE(tmp); - tmp = class_st; - class_st = PyStackRef_NULL; - stack_pointer[-2] = class_st; - PyStackRef_CLOSE(tmp); - tmp = global_super_st; - global_super_st = PyStackRef_NULL; - stack_pointer[-3] = global_super_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - if (super == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = PyObject_GetAttr(super, name); - Py_DECREF(super); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - attr = PyStackRef_FromPyObjectSteal(attr_o); - } - // _PUSH_NULL_CONDITIONAL - { - null = &stack_pointer[1]; - if (oparg & 1) { - null[0] = PyStackRef_NULL; - } - } - stack_pointer[0] = attr; - stack_pointer += 1 + (oparg & 1); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_SUPER_ATTR_ATTR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_SUPER_ATTR_ATTR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR_ATTR); - opcode = LOAD_SUPER_ATTR_ATTR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); - _PyStackRef global_super_st; - _PyStackRef class_st; - _PyStackRef self_st; - _PyStackRef attr_st; - /* Skip 1 cache entry */ - self_st = stack_pointer[-1]; - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - assert(!(oparg & 1)); - if (global_super != (PyObject *)&PySuper_Type) { - UPDATE_MISS_STATS(LOAD_SUPER_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_SUPER_ATTR); - } - if (!PyType_Check(class)) { - UPDATE_MISS_STATS(LOAD_SUPER_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_SUPER_ATTR); - } - STAT_INC(LOAD_SUPER_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); - _PyStackRef tmp = self_st; - self_st = PyStackRef_NULL; - stack_pointer[-1] = self_st; - PyStackRef_CLOSE(tmp); - tmp = class_st; - class_st = PyStackRef_NULL; - stack_pointer[-2] = class_st; - PyStackRef_CLOSE(tmp); - tmp = global_super_st; - global_super_st = PyStackRef_NULL; - stack_pointer[-3] = global_super_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - if (attr == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - attr_st = PyStackRef_FromPyObjectSteal(attr); - stack_pointer[0] = attr_st; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(LOAD_SUPER_ATTR_METHOD) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = LOAD_SUPER_ATTR_METHOD; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(LOAD_SUPER_ATTR_METHOD); - opcode = LOAD_SUPER_ATTR_METHOD; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); - _PyStackRef global_super_st; - _PyStackRef class_st; - _PyStackRef self_st; - _PyStackRef attr; - _PyStackRef self_or_null; - /* Skip 1 cache entry */ - self_st = stack_pointer[-1]; - class_st = stack_pointer[-2]; - global_super_st = stack_pointer[-3]; - PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); - PyObject *class = PyStackRef_AsPyObjectBorrow(class_st); - PyObject *self = PyStackRef_AsPyObjectBorrow(self_st); - assert(oparg & 1); - if (global_super != (PyObject *)&PySuper_Type) { - UPDATE_MISS_STATS(LOAD_SUPER_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_SUPER_ATTR); - } - if (!PyType_Check(class)) { - UPDATE_MISS_STATS(LOAD_SUPER_ATTR); - assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); - JUMP_TO_PREDICTED(TRACING_LOAD_SUPER_ATTR); - } - STAT_INC(LOAD_SUPER_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - PyTypeObject *cls = (PyTypeObject *)class; - int method_found = 0; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attr_o = _PySuper_Lookup(cls, self, name, - Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - if (method_found) { - self_or_null = self_st; - } else { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(self_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - self_or_null = PyStackRef_NULL; - stack_pointer += 1; - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = global_super_st; - global_super_st = self_or_null; - stack_pointer[-2] = global_super_st; - PyStackRef_CLOSE(tmp); - tmp = class_st; - class_st = PyStackRef_NULL; - stack_pointer[-1] = class_st; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - attr = PyStackRef_FromPyObjectSteal(attr_o); - stack_pointer[0] = attr; - stack_pointer[1] = self_or_null; - stack_pointer += 2; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(MAKE_CELL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = MAKE_CELL; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MAKE_CELL); - opcode = MAKE_CELL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - PyObject *initial = PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - PyObject *cell = PyCell_New(initial); - if (cell == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = PyStackRef_FromPyObjectSteal(cell); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(MAKE_FUNCTION) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = MAKE_FUNCTION; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MAKE_FUNCTION); - opcode = MAKE_FUNCTION; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef codeobj_st; - _PyStackRef func; - codeobj_st = stack_pointer[-1]; - PyObject *codeobj = PyStackRef_AsPyObjectBorrow(codeobj_st); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyFunctionObject *func_obj = (PyFunctionObject *) - PyFunction_New(codeobj, GLOBALS()); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(codeobj_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (func_obj == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - _PyFunction_SetVersion( - func_obj, ((PyCodeObject *)codeobj)->co_version); - func = PyStackRef_FromPyObjectSteal((PyObject *)func_obj); - stack_pointer[0] = func; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(MAP_ADD) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = MAP_ADD; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MAP_ADD); - opcode = MAP_ADD; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef dict_st; - _PyStackRef key; - _PyStackRef value; - value = stack_pointer[-1]; - key = stack_pointer[-2]; - dict_st = stack_pointer[-3 - (oparg - 1)]; - PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - assert(PyDict_CheckExact(dict)); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyDict_SetItem_Take2( - (PyDictObject *)dict, - PyStackRef_AsPyObjectSteal(key), - PyStackRef_AsPyObjectSteal(value) - ); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(pop_2_error); - } - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(MATCH_CLASS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = MATCH_CLASS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_CLASS); - opcode = MATCH_CLASS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef subject; - _PyStackRef type; - _PyStackRef names; - _PyStackRef attrs; - names = stack_pointer[-1]; - type = stack_pointer[-2]; - subject = stack_pointer[-3]; - assert(PyTuple_CheckExact(PyStackRef_AsPyObjectBorrow(names))); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *attrs_o = _PyEval_MatchClass(tstate, - PyStackRef_AsPyObjectBorrow(subject), - PyStackRef_AsPyObjectBorrow(type), oparg, - PyStackRef_AsPyObjectBorrow(names)); - _PyStackRef tmp = names; - names = PyStackRef_NULL; - stack_pointer[-1] = names; - PyStackRef_CLOSE(tmp); - tmp = type; - type = PyStackRef_NULL; - stack_pointer[-2] = type; - PyStackRef_CLOSE(tmp); - tmp = subject; - subject = PyStackRef_NULL; - stack_pointer[-3] = subject; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - if (attrs_o) { - assert(PyTuple_CheckExact(attrs_o)); - attrs = PyStackRef_FromPyObjectSteal(attrs_o); - } - else { - if (_PyErr_Occurred(tstate)) { - TRACING_JUMP_TO_LABEL(error); - } - attrs = PyStackRef_None; - } - stack_pointer[0] = attrs; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(MATCH_KEYS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = MATCH_KEYS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_KEYS); - opcode = MATCH_KEYS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef subject; - _PyStackRef keys; - _PyStackRef values_or_none; - keys = stack_pointer[-1]; - subject = stack_pointer[-2]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *values_or_none_o = _PyEval_MatchKeys(tstate, - PyStackRef_AsPyObjectBorrow(subject), PyStackRef_AsPyObjectBorrow(keys)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (values_or_none_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - values_or_none = PyStackRef_FromPyObjectSteal(values_or_none_o); - stack_pointer[0] = values_or_none; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(MATCH_MAPPING) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = MATCH_MAPPING; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_MAPPING); - opcode = MATCH_MAPPING; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef subject; - _PyStackRef res; - subject = stack_pointer[-1]; - int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_MAPPING; - res = match ? PyStackRef_True : PyStackRef_False; - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(MATCH_SEQUENCE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = MATCH_SEQUENCE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(MATCH_SEQUENCE); - opcode = MATCH_SEQUENCE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef subject; - _PyStackRef res; - subject = stack_pointer[-1]; - int match = PyStackRef_TYPE(subject)->tp_flags & Py_TPFLAGS_SEQUENCE; - res = match ? PyStackRef_True : PyStackRef_False; - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(NOP) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = NOP; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(NOP); - opcode = NOP; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - TRACING_DISPATCH(); - } - - TRACING_TARGET(NOT_TAKEN) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = NOT_TAKEN; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(NOT_TAKEN); - opcode = NOT_TAKEN; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - TRACING_DISPATCH(); - } - - TRACING_TARGET(POP_EXCEPT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = POP_EXCEPT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(POP_EXCEPT); - opcode = POP_EXCEPT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef exc_value; - exc_value = stack_pointer[-1]; - _PyErr_StackItem *exc_info = tstate->exc_info; - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_XSETREF(exc_info->exc_value, - PyStackRef_IsNone(exc_value) - ? NULL : PyStackRef_AsPyObjectSteal(exc_value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(POP_ITER) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = POP_ITER; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(POP_ITER); - opcode = POP_ITER; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef iter; - _PyStackRef index_or_null; - index_or_null = stack_pointer[-1]; - iter = stack_pointer[-2]; - (void)index_or_null; - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iter); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(POP_JUMP_IF_FALSE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = POP_JUMP_IF_FALSE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_FALSE); - opcode = POP_JUMP_IF_FALSE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef cond; - /* Skip 1 cache entry */ - cond = stack_pointer[-1]; - assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsFalse(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(POP_JUMP_IF_NONE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = POP_JUMP_IF_NONE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_NONE); - opcode = POP_JUMP_IF_NONE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - _PyStackRef b; - _PyStackRef cond; - /* Skip 1 cache entry */ - // _IS_NONE - { - value = stack_pointer[-1]; - if (PyStackRef_IsNone(value)) { - b = PyStackRef_True; - } - else { - b = PyStackRef_False; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = value; - value = b; - stack_pointer[-1] = value; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - // _POP_JUMP_IF_TRUE - { - cond = b; - assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsTrue(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(POP_JUMP_IF_NOT_NONE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = POP_JUMP_IF_NOT_NONE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_NOT_NONE); - opcode = POP_JUMP_IF_NOT_NONE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - _PyStackRef b; - _PyStackRef cond; - /* Skip 1 cache entry */ - // _IS_NONE - { - value = stack_pointer[-1]; - if (PyStackRef_IsNone(value)) { - b = PyStackRef_True; - } - else { - b = PyStackRef_False; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = value; - value = b; - stack_pointer[-1] = value; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - } - // _POP_JUMP_IF_FALSE - { - cond = b; - assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsFalse(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(POP_JUMP_IF_TRUE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = POP_JUMP_IF_TRUE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(POP_JUMP_IF_TRUE); - opcode = POP_JUMP_IF_TRUE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef cond; - /* Skip 1 cache entry */ - cond = stack_pointer[-1]; - assert(PyStackRef_BoolCheck(cond)); - int flag = PyStackRef_IsTrue(cond); - RECORD_BRANCH_TAKEN(this_instr[1].cache, flag); - JUMPBY(flag ? oparg : next_instr->op.code == NOT_TAKEN); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(POP_TOP) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = POP_TOP; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(POP_TOP); - opcode = POP_TOP; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - value = stack_pointer[-1]; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(PUSH_EXC_INFO) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = PUSH_EXC_INFO; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(PUSH_EXC_INFO); - opcode = PUSH_EXC_INFO; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef exc; - _PyStackRef prev_exc; - _PyStackRef new_exc; - exc = stack_pointer[-1]; - _PyErr_StackItem *exc_info = tstate->exc_info; - if (exc_info->exc_value != NULL) { - prev_exc = PyStackRef_FromPyObjectSteal(exc_info->exc_value); - } - else { - prev_exc = PyStackRef_None; - } - assert(PyStackRef_ExceptionInstanceCheck(exc)); - exc_info->exc_value = PyStackRef_AsPyObjectNew(exc); - new_exc = exc; - stack_pointer[-1] = prev_exc; - stack_pointer[0] = new_exc; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(PUSH_NULL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = PUSH_NULL; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(PUSH_NULL); - opcode = PUSH_NULL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef res; - res = PyStackRef_NULL; - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(RAISE_VARARGS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = RAISE_VARARGS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RAISE_VARARGS); - opcode = RAISE_VARARGS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef *args; - args = &stack_pointer[-oparg]; - assert(oparg < 3); - PyObject *cause = oparg == 2 ? PyStackRef_AsPyObjectSteal(args[1]) : NULL; - PyObject *exc = oparg > 0 ? PyStackRef_AsPyObjectSteal(args[0]) : NULL; - stack_pointer += -oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = do_raise(tstate, exc, cause); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - assert(oparg == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - monitor_reraise(tstate, frame, this_instr); - TRACING_JUMP_TO_LABEL(exception_unwind); - } - TRACING_JUMP_TO_LABEL(error); - } - - TRACING_TARGET(RERAISE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = RERAISE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RERAISE); - opcode = RERAISE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef *values; - _PyStackRef exc_st; - exc_st = stack_pointer[-1]; - values = &stack_pointer[-1 - oparg]; - PyObject *exc = PyStackRef_AsPyObjectSteal(exc_st); - assert(oparg >= 0 && oparg <= 2); - if (oparg) { - frame->instr_ptr = _PyFrame_GetBytecode(frame) + PyStackRef_UntagInt(values[0]); - } - assert(exc && PyExceptionInstance_Check(exc)); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_SetRaisedException(tstate, exc); - monitor_reraise(tstate, frame, this_instr); - TRACING_JUMP_TO_LABEL(exception_unwind); - } - - TRACING_TARGET(RESERVED) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = RESERVED; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RESERVED); - opcode = RESERVED; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - assert(0 && "Executing RESERVED instruction."); - Py_FatalError("Executing RESERVED instruction."); - TRACING_DISPATCH(); - } - - TRACING_TARGET(RESUME) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = RESUME; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RESUME); - PREDICTED_TRACING_RESUME:; - _Py_CODEUNIT* const this_instr = next_instr - 1; - (void)this_instr; - opcode = RESUME; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - // _LOAD_BYTECODE - { - #ifdef Py_GIL_DISABLED - if (frame->tlbc_index != - ((_PyThreadStateImpl *)tstate)->tlbc_index) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_CODEUNIT *bytecode = - _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (bytecode == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame); - frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index; - frame->instr_ptr = bytecode + off; - next_instr = frame->instr_ptr; - TRACING_DISPATCH(); - } - #endif - } - // _MAYBE_INSTRUMENT - { - #ifdef Py_GIL_DISABLED - - int check_instrumentation = 1; - #else - int check_instrumentation = (tstate->tracing == 0); - #endif - if (check_instrumentation) { - uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; - uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - if (code_version != global_version) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - next_instr = this_instr; - TRACING_DISPATCH(); - } - } - } - // _QUICKEN_RESUME - { - #if ENABLE_SPECIALIZATION_FT - if (tstate->tracing == 0 && this_instr->op.code == RESUME) { - FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); - } - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _CHECK_PERIODIC_IF_NOT_YIELD_FROM - { - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = check_periodics(tstate); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err != 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(RESUME_CHECK) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = RESUME_CHECK; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RESUME_CHECK); - opcode = RESUME_CHECK; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(0 == 0, "incorrect cache size"); - #if defined(__EMSCRIPTEN__) - if (_Py_emscripten_signal_clock == 0) { - UPDATE_MISS_STATS(RESUME); - assert(_PyOpcode_Deopt[opcode] == (RESUME)); - JUMP_TO_PREDICTED(TRACING_RESUME); - } - _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; - #endif - uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); - uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - assert((version & _PY_EVAL_EVENTS_MASK) == 0); - if (eval_breaker != version) { - UPDATE_MISS_STATS(RESUME); - assert(_PyOpcode_Deopt[opcode] == (RESUME)); - JUMP_TO_PREDICTED(TRACING_RESUME); - } - #ifdef Py_GIL_DISABLED - if (frame->tlbc_index != - ((_PyThreadStateImpl *)tstate)->tlbc_index) { - UPDATE_MISS_STATS(RESUME); - assert(_PyOpcode_Deopt[opcode] == (RESUME)); - JUMP_TO_PREDICTED(TRACING_RESUME); - } - #endif - TRACING_DISPATCH(); - } - - TRACING_TARGET(RETURN_GENERATOR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = RETURN_GENERATOR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RETURN_GENERATOR); - opcode = RETURN_GENERATOR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef res; - assert(PyStackRef_FunctionCheck(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (gen == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - frame->instr_ptr++; - _PyFrame_Copy(frame, gen_frame); - assert(frame->frame_obj == NULL); - gen->gi_frame_state = FRAME_CREATED; - gen_frame->owner = FRAME_OWNED_BY_GENERATOR; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *prev = frame->previous; - _PyThreadState_PopFrame(tstate, frame); - frame = tstate->current_frame = prev; - #if TIER_ONE - LOAD_IP(frame->return_offset); - #endif - #if TIER_TWO - TIER2_STORE_IP(frame->return_offset); - #endif - stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); - LLTRACE_RESUME_FRAME(); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(RETURN_VALUE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = RETURN_VALUE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(RETURN_VALUE); - opcode = RETURN_VALUE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef retval; - _PyStackRef res; - retval = stack_pointer[-1]; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - _PyStackRef temp = PyStackRef_MakeHeapSafe(retval); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(STACK_LEVEL() == 0); - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *dying = frame; - frame = tstate->current_frame = dying->previous; - _PyEval_FrameClearAndPop(tstate, dying); - stack_pointer = _PyFrame_GetStackPointer(frame); - #if TIER_ONE - LOAD_IP(frame->return_offset); - #endif - #if TIER_TWO - TIER2_STORE_IP(frame->return_offset); - #endif - res = temp; - LLTRACE_RESUME_FRAME(); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(SEND) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = SEND; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(SEND); - PREDICTED_TRACING_SEND:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - opcode = SEND; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef receiver; - _PyStackRef v; - _PyStackRef retval; - // _SPECIALIZE_SEND - { - receiver = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_Send(receiver, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() - } - OPCODE_DEFERRED_INC(SEND); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _SEND - { - v = stack_pointer[-1]; - PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver); - PyObject *retval_o; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - if ((tstate->interp->eval_frame == NULL) && - (Py_TYPE(receiver_o) == &PyGen_Type || Py_TYPE(receiver_o) == &PyCoro_Type) && - ((PyGenObject *)receiver_o)->gi_frame_state < FRAME_EXECUTING) - { - PyGenObject *gen = (PyGenObject *)receiver_o; - _PyInterpreterFrame *gen_frame = &gen->gi_iframe; - _PyFrame_StackPush(gen_frame, PyStackRef_MakeHeapSafe(v)); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - assert( 2u + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)( 2u + oparg); - assert(gen_frame->previous == NULL); - gen_frame->previous = frame; - TRACING_DISPATCH_INLINED(gen_frame); - } - if (PyStackRef_IsNone(v) && PyIter_Check(receiver_o)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - retval_o = Py_TYPE(receiver_o)->tp_iternext(receiver_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - retval_o = PyObject_CallMethodOneArg(receiver_o, - &_Py_ID(send), - PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - if (retval_o == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (matches) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyEval_MonitorRaise(tstate, frame, this_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyGen_FetchStopIterationValue(&retval_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err == 0) { - assert(retval_o != NULL); - JUMPBY(oparg); - RECORD_DYNAMIC_JUMP_TAKEN(); - } - else { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); - retval = PyStackRef_FromPyObjectSteal(retval_o); - } - stack_pointer[0] = retval; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(SEND_GEN) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = SEND_GEN; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(SEND_GEN); - opcode = SEND_GEN; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); - _PyStackRef receiver; - _PyStackRef v; - _PyStackRef gen_frame; - _PyStackRef new_frame; - /* Skip 1 cache entry */ - // _CHECK_PEP_523 - { - if (tstate->interp->eval_frame) { - UPDATE_MISS_STATS(SEND); - assert(_PyOpcode_Deopt[opcode] == (SEND)); - JUMP_TO_PREDICTED(TRACING_SEND); - } - } - // _SEND_GEN_FRAME - { - v = stack_pointer[-1]; - receiver = stack_pointer[-2]; - PyGenObject *gen = (PyGenObject *)PyStackRef_AsPyObjectBorrow(receiver); - if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { - UPDATE_MISS_STATS(SEND); - assert(_PyOpcode_Deopt[opcode] == (SEND)); - JUMP_TO_PREDICTED(TRACING_SEND); - } - if (gen->gi_frame_state >= FRAME_EXECUTING) { - UPDATE_MISS_STATS(SEND); - assert(_PyOpcode_Deopt[opcode] == (SEND)); - JUMP_TO_PREDICTED(TRACING_SEND); - } - STAT_INC(SEND, hit); - _PyInterpreterFrame *pushed_frame = &gen->gi_iframe; - _PyFrame_StackPush(pushed_frame, PyStackRef_MakeHeapSafe(v)); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - assert( 2u + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)( 2u + oparg); - pushed_frame->previous = frame; - gen_frame = PyStackRef_Wrap(pushed_frame); - } - // _PUSH_FRAME - { - new_frame = gen_frame; - assert(tstate->interp->eval_frame == NULL); - _PyInterpreterFrame *temp = PyStackRef_Unwrap(new_frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - assert(temp->previous == frame || temp->previous->previous == frame); - CALL_STAT_INC(inlined_py_calls); - frame = tstate->current_frame = temp; - tstate->py_recursion_remaining--; - LOAD_SP(); - LOAD_IP(0); - LLTRACE_RESUME_FRAME(); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(SETUP_ANNOTATIONS) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = SETUP_ANNOTATIONS; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SETUP_ANNOTATIONS); - opcode = SETUP_ANNOTATIONS; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - PyObject *ann_dict; - if (LOCALS() == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_SystemError, - "no locals found when setting up annotations"); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - if (ann_dict == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - ann_dict = PyDict_New(); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (ann_dict == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), - ann_dict); - Py_DECREF(ann_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(ann_dict); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(SET_ADD) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = SET_ADD; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SET_ADD); - opcode = SET_ADD; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef set; - _PyStackRef v; - v = stack_pointer[-1]; - set = stack_pointer[-2 - (oparg-1)]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PySet_AddTakeRef((PySetObject *)PyStackRef_AsPyObjectBorrow(set), - PyStackRef_AsPyObjectSteal(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(pop_1_error); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(SET_FUNCTION_ATTRIBUTE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = SET_FUNCTION_ATTRIBUTE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SET_FUNCTION_ATTRIBUTE); - opcode = SET_FUNCTION_ATTRIBUTE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef attr_st; - _PyStackRef func_in; - _PyStackRef func_out; - func_in = stack_pointer[-1]; - attr_st = stack_pointer[-2]; - PyObject *func = PyStackRef_AsPyObjectBorrow(func_in); - PyObject *attr = PyStackRef_AsPyObjectSteal(attr_st); - func_out = func_in; - assert(PyFunction_Check(func)); - size_t offset = _Py_FunctionAttributeOffsets[oparg]; - assert(offset != 0); - PyObject **ptr = (PyObject **)(((char *)func) + offset); - assert(*ptr == NULL); - *ptr = attr; - stack_pointer[-2] = func_out; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(SET_UPDATE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = SET_UPDATE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SET_UPDATE); - opcode = SET_UPDATE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef set; - _PyStackRef iterable; - iterable = stack_pointer[-1]; - set = stack_pointer[-2 - (oparg-1)]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PySet_Update(PyStackRef_AsPyObjectBorrow(set), - PyStackRef_AsPyObjectBorrow(iterable)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(iterable); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_ATTR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_ATTR; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR); - PREDICTED_TRACING_STORE_ATTR:; - _Py_CODEUNIT* const this_instr = next_instr - 5; - (void)this_instr; - opcode = STORE_ATTR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef owner; - _PyStackRef v; - // _SPECIALIZE_STORE_ATTR - { - owner = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_StoreAttr(owner, next_instr, name); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() - } - OPCODE_DEFERRED_INC(STORE_ATTR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 3 cache entries */ - // _STORE_ATTR - { - v = stack_pointer[-2]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_SetAttr(PyStackRef_AsPyObjectBorrow(owner), - name, PyStackRef_AsPyObjectBorrow(v)); - _PyStackRef tmp = owner; - owner = PyStackRef_NULL; - stack_pointer[-1] = owner; - PyStackRef_CLOSE(tmp); - tmp = v; - v = PyStackRef_NULL; - stack_pointer[-2] = v; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_ATTR_INSTANCE_VALUE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_ATTR_INSTANCE_VALUE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_INSTANCE_VALUE); - opcode = STORE_ATTR_INSTANCE_VALUE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef value; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION_AND_LOCK - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(type_version != 0); - if (!LOCK_OBJECT(owner_o)) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - PyTypeObject *tp = Py_TYPE(owner_o); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UNLOCK_OBJECT(owner_o); - if (true) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - } - } - // _GUARD_DORV_NO_DICT - { - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_dictoffset < 0); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_INLINE_VALUES); - if (_PyObject_GetManagedDict(owner_o) || - !FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { - UNLOCK_OBJECT(owner_o); - if (true) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - } - } - // _STORE_ATTR_INSTANCE_VALUE - { - value = stack_pointer[-2]; - uint16_t offset = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - STAT_INC(STORE_ATTR, hit); - assert(_PyObject_GetManagedDict(owner_o) == NULL); - PyObject **value_ptr = (PyObject**)(((char *)owner_o) + offset); - PyObject *old_value = *value_ptr; - FT_ATOMIC_STORE_PTR_RELEASE(*value_ptr, PyStackRef_AsPyObjectSteal(value)); - if (old_value == NULL) { - PyDictValues *values = _PyObject_InlineValues(owner_o); - Py_ssize_t index = value_ptr - values->values; - _PyDictValues_AddToInsertionOrder(values, index); - } - UNLOCK_OBJECT(owner_o); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - Py_XDECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_ATTR_SLOT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_ATTR_SLOT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_SLOT); - opcode = STORE_ATTR_SLOT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef value; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - } - // _STORE_ATTR_SLOT - { - value = stack_pointer[-2]; - uint16_t index = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - if (!LOCK_OBJECT(owner_o)) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - char *addr = (char *)owner_o + index; - STAT_INC(STORE_ATTR, hit); - PyObject *old_value = *(PyObject **)addr; - FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, PyStackRef_AsPyObjectSteal(value)); - UNLOCK_OBJECT(owner_o); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - Py_XDECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_ATTR_WITH_HINT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_ATTR_WITH_HINT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 5; - INSTRUCTION_STATS(STORE_ATTR_WITH_HINT); - opcode = STORE_ATTR_WITH_HINT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef value; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - } - // _STORE_ATTR_WITH_HINT - { - value = stack_pointer[-2]; - uint16_t hint = read_u16(&this_instr[4].cache); - PyObject *owner_o = PyStackRef_AsPyObjectBorrow(owner); - assert(Py_TYPE(owner_o)->tp_flags & Py_TPFLAGS_MANAGED_DICT); - PyDictObject *dict = _PyObject_GetManagedDict(owner_o); - if (dict == NULL) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - if (!LOCK_OBJECT(dict)) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - assert(PyDict_CheckExact((PyObject *)dict)); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (hint >= (size_t)dict->ma_keys->dk_nentries || - !DK_IS_UNICODE(dict->ma_keys)) { - UNLOCK_OBJECT(dict); - if (true) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - } - PyDictUnicodeEntry *ep = DK_UNICODE_ENTRIES(dict->ma_keys) + hint; - if (ep->me_key != name) { - UNLOCK_OBJECT(dict); - if (true) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - } - PyObject *old_value = ep->me_value; - if (old_value == NULL) { - UNLOCK_OBJECT(dict); - if (true) { - UPDATE_MISS_STATS(STORE_ATTR); - assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - JUMP_TO_PREDICTED(TRACING_STORE_ATTR); - } - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyDict_NotifyEvent(tstate->interp, PyDict_EVENT_MODIFIED, dict, name, PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - FT_ATOMIC_STORE_PTR_RELEASE(ep->me_value, PyStackRef_AsPyObjectSteal(value)); - UNLOCK_OBJECT(dict); - STAT_INC(STORE_ATTR, hit); - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - Py_XDECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_DEREF) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_DEREF; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_DEREF); - opcode = STORE_DEREF; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef v; - v = stack_pointer[-1]; - PyCellObject *cell = (PyCellObject *)PyStackRef_AsPyObjectBorrow(GETLOCAL(oparg)); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCell_SetTakeRef(cell, PyStackRef_AsPyObjectSteal(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_FAST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_FAST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST); - opcode = STORE_FAST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - value = stack_pointer[-1]; - _PyStackRef tmp = GETLOCAL(oparg); - GETLOCAL(oparg) = value; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_FAST_LOAD_FAST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_FAST_LOAD_FAST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST_LOAD_FAST); - opcode = STORE_FAST_LOAD_FAST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value1; - _PyStackRef value2; - value1 = stack_pointer[-1]; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - _PyStackRef tmp = GETLOCAL(oparg1); - GETLOCAL(oparg1) = value1; - value2 = PyStackRef_DUP(GETLOCAL(oparg2)); - stack_pointer[-1] = value2; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_FAST_STORE_FAST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_FAST_STORE_FAST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_FAST_STORE_FAST); - opcode = STORE_FAST_STORE_FAST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value2; - _PyStackRef value1; - value1 = stack_pointer[-1]; - value2 = stack_pointer[-2]; - uint32_t oparg1 = oparg >> 4; - uint32_t oparg2 = oparg & 15; - _PyStackRef tmp = GETLOCAL(oparg1); - GETLOCAL(oparg1) = value1; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - tmp = GETLOCAL(oparg2); - GETLOCAL(oparg2) = value2; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_XCLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_GLOBAL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_GLOBAL; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_GLOBAL); - opcode = STORE_GLOBAL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef v; - v = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_NAME) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_NAME; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_NAME); - opcode = STORE_NAME; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef v; - v = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *ns = LOCALS(); - int err; - if (ns == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyErr_Format(tstate, PyExc_SystemError, - "no locals found when storing %R", name); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_JUMP_TO_LABEL(error); - } - if (PyDict_CheckExact(ns)) { - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyDict_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - else { - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_SetItem(ns, name, PyStackRef_AsPyObjectBorrow(v)); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(v); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_SLICE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_SLICE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(STORE_SLICE); - opcode = STORE_SLICE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef v; - _PyStackRef container; - _PyStackRef start; - _PyStackRef stop; - // _SPECIALIZE_STORE_SLICE - { - #if ENABLE_SPECIALIZATION - OPCODE_DEFERRED_INC(STORE_SLICE); - #endif /* ENABLE_SPECIALIZATION */ - } - // _STORE_SLICE - { - stop = stack_pointer[-1]; - start = stack_pointer[-2]; - container = stack_pointer[-3]; - v = stack_pointer[-4]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start), - PyStackRef_AsPyObjectSteal(stop)); - stack_pointer = _PyFrame_GetStackPointer(frame); - int err; - if (slice == NULL) { - err = 1; - } - else { - stack_pointer += -2; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v)); - Py_DECREF(slice); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += 2; - } - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = container; - container = PyStackRef_NULL; - stack_pointer[-3] = container; - PyStackRef_CLOSE(tmp); - tmp = v; - v = PyStackRef_NULL; - stack_pointer[-4] = v; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -4; - assert(WITHIN_STACK_BOUNDS()); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_SUBSCR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_SUBSCR; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR); - PREDICTED_TRACING_STORE_SUBSCR:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - opcode = STORE_SUBSCR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef container; - _PyStackRef sub; - _PyStackRef v; - // _SPECIALIZE_STORE_SUBSCR - { - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_StoreSubscr(container, sub, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() - } - OPCODE_DEFERRED_INC(STORE_SUBSCR); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - // _STORE_SUBSCR - { - v = stack_pointer[-3]; - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), PyStackRef_AsPyObjectBorrow(sub), PyStackRef_AsPyObjectBorrow(v)); - _PyStackRef tmp = sub; - sub = PyStackRef_NULL; - stack_pointer[-1] = sub; - PyStackRef_CLOSE(tmp); - tmp = container; - container = PyStackRef_NULL; - stack_pointer[-2] = container; - PyStackRef_CLOSE(tmp); - tmp = v; - v = PyStackRef_NULL; - stack_pointer[-3] = v; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_SUBSCR_DICT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_SUBSCR_DICT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR_DICT); - opcode = STORE_SUBSCR_DICT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); - _PyStackRef nos; - _PyStackRef value; - _PyStackRef dict_st; - _PyStackRef sub; - // _GUARD_NOS_DICT - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyDict_CheckExact(o)) { - UPDATE_MISS_STATS(STORE_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); - } - } - /* Skip 1 cache entry */ - // _STORE_SUBSCR_DICT - { - sub = stack_pointer[-1]; - dict_st = nos; - value = stack_pointer[-3]; - PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - assert(PyDict_CheckExact(dict)); - STAT_INC(STORE_SUBSCR, hit); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = _PyDict_SetItem_Take2((PyDictObject *)dict, - PyStackRef_AsPyObjectSteal(sub), - PyStackRef_AsPyObjectSteal(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(dict_st); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err) { - TRACING_JUMP_TO_LABEL(error); - } - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(STORE_SUBSCR_LIST_INT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = STORE_SUBSCR_LIST_INT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(STORE_SUBSCR_LIST_INT); - opcode = STORE_SUBSCR_LIST_INT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_STORE_SUBSCR == 1, "incorrect cache size"); - _PyStackRef value; - _PyStackRef nos; - _PyStackRef list_st; - _PyStackRef sub_st; - // _GUARD_TOS_INT - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!_PyLong_CheckExactAndCompact(value_o)) { - UPDATE_MISS_STATS(STORE_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); - } - } - // _GUARD_NOS_LIST - { - nos = stack_pointer[-2]; - PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyList_CheckExact(o)) { - UPDATE_MISS_STATS(STORE_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); - } - } - /* Skip 1 cache entry */ - // _STORE_SUBSCR_LIST_INT - { - sub_st = value; - list_st = nos; - value = stack_pointer[-3]; - PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - PyObject *list = PyStackRef_AsPyObjectBorrow(list_st); - assert(PyLong_CheckExact(sub)); - assert(PyList_CheckExact(list)); - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - UPDATE_MISS_STATS(STORE_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); - } - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (!LOCK_OBJECT(list)) { - UPDATE_MISS_STATS(STORE_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); - } - if (index >= PyList_GET_SIZE(list)) { - UNLOCK_OBJECT(list); - if (true) { - UPDATE_MISS_STATS(STORE_SUBSCR); - assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - JUMP_TO_PREDICTED(TRACING_STORE_SUBSCR); - } - } - STAT_INC(STORE_SUBSCR, hit); - PyObject *old_value = PyList_GET_ITEM(list, index); - FT_ATOMIC_STORE_PTR_RELEASE(_PyList_ITEMS(list)[index], - PyStackRef_AsPyObjectSteal(value)); - assert(old_value != NULL); - UNLOCK_OBJECT(list); - PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc); - stack_pointer += -3; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(list_st); - Py_DECREF(old_value); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(SWAP) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = SWAP; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(SWAP); - opcode = SWAP; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef bottom; - _PyStackRef top; - top = stack_pointer[-1]; - bottom = stack_pointer[-2 - (oparg-2)]; - _PyStackRef temp = bottom; - bottom = top; - top = temp; - stack_pointer[-2 - (oparg-2)] = bottom; - stack_pointer[-1] = top; - TRACING_DISPATCH(); - } - - TRACING_TARGET(TO_BOOL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = TO_BOOL; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL); - PREDICTED_TRACING_TO_BOOL:; - _Py_CODEUNIT* const this_instr = next_instr - 4; - (void)this_instr; - opcode = TO_BOOL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - _PyStackRef res; - // _SPECIALIZE_TO_BOOL - { - value = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_ToBool(value, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() - } - OPCODE_DEFERRED_INC(TO_BOOL); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - } - /* Skip 2 cache entries */ - // _TO_BOOL - { - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - TRACING_JUMP_TO_LABEL(error); - } - res = err ? PyStackRef_True : PyStackRef_False; - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(TO_BOOL_ALWAYS_TRUE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = TO_BOOL_ALWAYS_TRUE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); - opcode = TO_BOOL_ALWAYS_TRUE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef owner; - _PyStackRef value; - _PyStackRef res; - /* Skip 1 cache entry */ - // _GUARD_TYPE_VERSION - { - owner = stack_pointer[-1]; - uint32_t type_version = read_u32(&this_instr[2].cache); - PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(owner)); - assert(type_version != 0); - if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { - UPDATE_MISS_STATS(TO_BOOL); - assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - JUMP_TO_PREDICTED(TRACING_TO_BOOL); - } - } - // _REPLACE_WITH_TRUE - { - value = owner; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_True; - } - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(TO_BOOL_BOOL) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = TO_BOOL_BOOL; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_BOOL); - opcode = TO_BOOL_BOOL; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef value; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - value = stack_pointer[-1]; - if (!PyStackRef_BoolCheck(value)) { - UPDATE_MISS_STATS(TO_BOOL); - assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - JUMP_TO_PREDICTED(TRACING_TO_BOOL); - } - STAT_INC(TO_BOOL, hit); - TRACING_DISPATCH(); - } - - TRACING_TARGET(TO_BOOL_INT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = TO_BOOL_INT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_INT); - opcode = TO_BOOL_INT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef value; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyLong_CheckExact(value_o)) { - UPDATE_MISS_STATS(TO_BOOL); - assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - JUMP_TO_PREDICTED(TRACING_TO_BOOL); - } - STAT_INC(TO_BOOL, hit); - if (_PyLong_IsZero((PyLongObject *)value_o)) { - assert(_Py_IsImmortal(value_o)); - res = PyStackRef_False; - } - else { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_True; - stack_pointer += 1; - } - stack_pointer[-1] = res; - TRACING_DISPATCH(); - } - - TRACING_TARGET(TO_BOOL_LIST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = TO_BOOL_LIST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_LIST); - opcode = TO_BOOL_LIST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef tos; - _PyStackRef value; - _PyStackRef res; - // _GUARD_TOS_LIST - { - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyList_CheckExact(o)) { - UPDATE_MISS_STATS(TO_BOOL); - assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - JUMP_TO_PREDICTED(TRACING_TO_BOOL); - } - } - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _TO_BOOL_LIST - { - value = tos; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - assert(PyList_CheckExact(value_o)); - STAT_INC(TO_BOOL, hit); - res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False; - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyStackRef tmp = value; - value = res; - stack_pointer[-1] = value; - PyStackRef_CLOSE(tmp); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(TO_BOOL_NONE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = TO_BOOL_NONE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_NONE); - opcode = TO_BOOL_NONE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef value; - _PyStackRef res; - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - value = stack_pointer[-1]; - if (!PyStackRef_IsNone(value)) { - UPDATE_MISS_STATS(TO_BOOL); - assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - JUMP_TO_PREDICTED(TRACING_TO_BOOL); - } - STAT_INC(TO_BOOL, hit); - res = PyStackRef_False; - stack_pointer[-1] = res; - TRACING_DISPATCH(); - } - - TRACING_TARGET(TO_BOOL_STR) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = TO_BOOL_STR; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 4; - INSTRUCTION_STATS(TO_BOOL_STR); - opcode = TO_BOOL_STR; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); - _PyStackRef value; - _PyStackRef res; - // _GUARD_TOS_UNICODE - { - value = stack_pointer[-1]; - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (!PyUnicode_CheckExact(value_o)) { - UPDATE_MISS_STATS(TO_BOOL); - assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - JUMP_TO_PREDICTED(TRACING_TO_BOOL); - } - } - /* Skip 1 cache entry */ - /* Skip 2 cache entries */ - // _TO_BOOL_STR - { - STAT_INC(TO_BOOL, hit); - PyObject *value_o = PyStackRef_AsPyObjectBorrow(value); - if (value_o == &_Py_STR(empty)) { - assert(_Py_IsImmortal(value_o)); - res = PyStackRef_False; - } - else { - assert(Py_SIZE(value_o)); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - res = PyStackRef_True; - stack_pointer += 1; - } - } - stack_pointer[-1] = res; - TRACING_DISPATCH(); - } - - TRACING_TARGET(UNARY_INVERT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = UNARY_INVERT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_INVERT); - opcode = UNARY_INVERT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(UNARY_NEGATIVE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = UNARY_NEGATIVE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_NEGATIVE); - opcode = UNARY_NEGATIVE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value)); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(value); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(UNARY_NOT) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = UNARY_NOT; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNARY_NOT); - opcode = UNARY_NOT; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef value; - _PyStackRef res; - value = stack_pointer[-1]; - assert(PyStackRef_BoolCheck(value)); - res = PyStackRef_IsFalse(value) - ? PyStackRef_True : PyStackRef_False; - stack_pointer[-1] = res; - TRACING_DISPATCH(); - } - - TRACING_TARGET(UNPACK_EX) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = UNPACK_EX; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(UNPACK_EX); - opcode = UNPACK_EX; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef seq; - _PyStackRef *top; - seq = stack_pointer[-1]; - top = &stack_pointer[(oparg & 0xFF) + (oparg >> 8)]; - PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg & 0xFF, oparg >> 8, top); - Py_DECREF(seq_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res == 0) { - TRACING_JUMP_TO_LABEL(error); - } - stack_pointer += 1 + (oparg & 0xFF) + (oparg >> 8); - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(UNPACK_SEQUENCE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = UNPACK_SEQUENCE; - (void)(opcode); - #endif - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE); - PREDICTED_TRACING_UNPACK_SEQUENCE:; - _Py_CODEUNIT* const this_instr = next_instr - 2; - (void)this_instr; - opcode = UNPACK_SEQUENCE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef seq; - _PyStackRef *top; - // _SPECIALIZE_UNPACK_SEQUENCE - { - seq = stack_pointer[-1]; - uint16_t counter = read_u16(&this_instr[1].cache); - (void)counter; - #if ENABLE_SPECIALIZATION_FT - if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { - next_instr = this_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - _Py_Specialize_UnpackSequence(seq, next_instr, oparg); - stack_pointer = _PyFrame_GetStackPointer(frame); - TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() - } - OPCODE_DEFERRED_INC(UNPACK_SEQUENCE); - ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION_FT */ - (void)seq; - (void)counter; - } - // _UNPACK_SEQUENCE - { - top = &stack_pointer[-1 + oparg]; - PyObject *seq_o = PyStackRef_AsPyObjectSteal(seq); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - int res = _PyEval_UnpackIterableStackRef(tstate, seq_o, oparg, -1, top); - Py_DECREF(seq_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res == 0) { - TRACING_JUMP_TO_LABEL(error); - } - } - stack_pointer += oparg; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(UNPACK_SEQUENCE_LIST) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = UNPACK_SEQUENCE_LIST; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_LIST); - opcode = UNPACK_SEQUENCE_LIST; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - _PyStackRef tos; - _PyStackRef seq; - _PyStackRef *values; - // _GUARD_TOS_LIST - { - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyList_CheckExact(o)) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); - } - } - /* Skip 1 cache entry */ - // _UNPACK_SEQUENCE_LIST - { - seq = tos; - values = &stack_pointer[-1]; - PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - assert(PyList_CheckExact(seq_o)); - if (!LOCK_OBJECT(seq_o)) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); - } - if (PyList_GET_SIZE(seq_o) != oparg) { - UNLOCK_OBJECT(seq_o); - if (true) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); - } - } - STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyList_ITEMS(seq_o); - for (int i = oparg; --i >= 0; ) { - *values++ = PyStackRef_FromPyObjectNew(items[i]); - } - UNLOCK_OBJECT(seq_o); - stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(UNPACK_SEQUENCE_TUPLE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = UNPACK_SEQUENCE_TUPLE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_TUPLE); - opcode = UNPACK_SEQUENCE_TUPLE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - _PyStackRef tos; - _PyStackRef seq; - _PyStackRef *values; - // _GUARD_TOS_TUPLE - { - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyTuple_CheckExact(o)) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); - } - } - /* Skip 1 cache entry */ - // _UNPACK_SEQUENCE_TUPLE - { - seq = tos; - values = &stack_pointer[-1]; - PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - assert(PyTuple_CheckExact(seq_o)); - if (PyTuple_GET_SIZE(seq_o) != oparg) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); - } - STAT_INC(UNPACK_SEQUENCE, hit); - PyObject **items = _PyTuple_ITEMS(seq_o); - for (int i = oparg; --i >= 0; ) { - *values++ = PyStackRef_FromPyObjectNew(items[i]); - } - stack_pointer += -1 + oparg; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(UNPACK_SEQUENCE_TWO_TUPLE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = UNPACK_SEQUENCE_TWO_TUPLE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 2; - INSTRUCTION_STATS(UNPACK_SEQUENCE_TWO_TUPLE); - opcode = UNPACK_SEQUENCE_TWO_TUPLE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); - _PyStackRef tos; - _PyStackRef seq; - _PyStackRef val1; - _PyStackRef val0; - // _GUARD_TOS_TUPLE - { - tos = stack_pointer[-1]; - PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyTuple_CheckExact(o)) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); - } - } - /* Skip 1 cache entry */ - // _UNPACK_SEQUENCE_TWO_TUPLE - { - seq = tos; - assert(oparg == 2); - PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); - assert(PyTuple_CheckExact(seq_o)); - if (PyTuple_GET_SIZE(seq_o) != 2) { - UPDATE_MISS_STATS(UNPACK_SEQUENCE); - assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - JUMP_TO_PREDICTED(TRACING_UNPACK_SEQUENCE); - } - STAT_INC(UNPACK_SEQUENCE, hit); - val0 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 0)); - val1 = PyStackRef_FromPyObjectNew(PyTuple_GET_ITEM(seq_o, 1)); - stack_pointer[-1] = val1; - stack_pointer[0] = val0; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(seq); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - TRACING_DISPATCH(); - } - - TRACING_TARGET(WITH_EXCEPT_START) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = WITH_EXCEPT_START; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(WITH_EXCEPT_START); - opcode = WITH_EXCEPT_START; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef exit_func; - _PyStackRef exit_self; - _PyStackRef lasti; - _PyStackRef val; - _PyStackRef res; - val = stack_pointer[-1]; - lasti = stack_pointer[-3]; - exit_self = stack_pointer[-4]; - exit_func = stack_pointer[-5]; - PyObject *exc, *tb; - PyObject *val_o = PyStackRef_AsPyObjectBorrow(val); - PyObject *exit_func_o = PyStackRef_AsPyObjectBorrow(exit_func); - assert(val_o && PyExceptionInstance_Check(val_o)); - exc = PyExceptionInstance_Class(val_o); - PyObject *original_tb = tb = PyException_GetTraceback(val_o); - if (tb == NULL) { - tb = Py_None; - } - assert(PyStackRef_IsTaggedInt(lasti)); - (void)lasti; - PyObject *stack[5] = {NULL, PyStackRef_AsPyObjectBorrow(exit_self), exc, val_o, tb}; - int has_self = !PyStackRef_IsNull(exit_self); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *res_o = PyObject_Vectorcall(exit_func_o, stack + 2 - has_self, - (3 + has_self) | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); - Py_XDECREF(original_tb); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (res_o == NULL) { - TRACING_JUMP_TO_LABEL(error); - } - res = PyStackRef_FromPyObjectSteal(res_o); - stack_pointer[0] = res; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - - TRACING_TARGET(YIELD_VALUE) { - assert(IS_JIT_TRACING()); - #if _Py_TAIL_CALL_INTERP - int opcode = YIELD_VALUE; - (void)(opcode); - #endif - _Py_CODEUNIT* const this_instr = next_instr; - (void)this_instr; - frame->instr_ptr = next_instr; - next_instr += 1; - INSTRUCTION_STATS(YIELD_VALUE); - opcode = YIELD_VALUE; - PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); - (void)old_code; - PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - (void)old_func; - int _jump_taken = false; - (void)_jump_taken; - int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0; - (void)(_old_stack_level); - _PyStackRef retval; - _PyStackRef value; - retval = stack_pointer[-1]; - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - frame->instr_ptr++; - PyGenObject *gen = _PyGen_GetGeneratorFromFrame(frame); - assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); - assert(oparg == 0 || oparg == 1); - gen->gi_frame_state = FRAME_SUSPENDED + oparg; - _PyStackRef temp = retval; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - tstate->exc_info = gen->gi_exc_state.previous_item; - gen->gi_exc_state.previous_item = NULL; - _Py_LeaveRecursiveCallPy(tstate); - _PyInterpreterFrame *gen_frame = frame; - frame = tstate->current_frame = frame->previous; - gen_frame->previous = NULL; - assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - #if TIER_ONE - assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || - frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); - #endif - stack_pointer = _PyFrame_GetStackPointer(frame); - #if TIER_ONE - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - #endif - #if TIER_TWO - TIER2_STORE_IP(1 + INLINE_CACHE_ENTRIES_SEND); - #endif - value = PyStackRef_MakeHeapSafe(temp); - LLTRACE_RESUME_FRAME(); - stack_pointer[0] = value; - stack_pointer += 1; - assert(WITHIN_STACK_BOUNDS()); - TRACING_DISPATCH(); - } - #endif /* END TRACING INSTRUCTIONS */ - #undef TRACING_JIT - #undef TIER_ONE diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 66abd6d4623e1c..0d28b3d4e69055 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -105,6 +105,7 @@ static void *opcode_targets_table[256] = { &&TARGET_POP_JUMP_IF_NOT_NONE, &&TARGET_POP_JUMP_IF_TRUE, &&TARGET_RAISE_VARARGS, + &&TARGET_RECORD_PREVIOUS_INST, &&TARGET_RERAISE, &&TARGET_SEND, &&TARGET_SET_ADD, @@ -127,7 +128,6 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_RESUME, &&TARGET_BINARY_OP_ADD_FLOAT, &&TARGET_BINARY_OP_ADD_INT, @@ -259,216 +259,216 @@ static void *opcode_targets_table[256] = { }; #if _Py_TIER2 static void *opcode_tracing_targets_table[256] = { - &&TARGET_TRACING_CACHE, - &&TARGET_TRACING_BINARY_SLICE, - &&TARGET_TRACING_BUILD_TEMPLATE, - &&TARGET_TRACING_BINARY_OP_INPLACE_ADD_UNICODE, - &&TARGET_TRACING_CALL_FUNCTION_EX, - &&TARGET_TRACING_CHECK_EG_MATCH, - &&TARGET_TRACING_CHECK_EXC_MATCH, - &&TARGET_TRACING_CLEANUP_THROW, - &&TARGET_TRACING_DELETE_SUBSCR, - &&TARGET_TRACING_END_FOR, - &&TARGET_TRACING_END_SEND, - &&TARGET_TRACING_EXIT_INIT_CHECK, - &&TARGET_TRACING_FORMAT_SIMPLE, - &&TARGET_TRACING_FORMAT_WITH_SPEC, - &&TARGET_TRACING_GET_AITER, - &&TARGET_TRACING_GET_ANEXT, - &&TARGET_TRACING_GET_ITER, - &&TARGET_TRACING_RESERVED, - &&TARGET_TRACING_GET_LEN, - &&TARGET_TRACING_GET_YIELD_FROM_ITER, - &&TARGET_TRACING_INTERPRETER_EXIT, - &&TARGET_TRACING_LOAD_BUILD_CLASS, - &&TARGET_TRACING_LOAD_LOCALS, - &&TARGET_TRACING_MAKE_FUNCTION, - &&TARGET_TRACING_MATCH_KEYS, - &&TARGET_TRACING_MATCH_MAPPING, - &&TARGET_TRACING_MATCH_SEQUENCE, - &&TARGET_TRACING_NOP, - &&TARGET_TRACING_NOT_TAKEN, - &&TARGET_TRACING_POP_EXCEPT, - &&TARGET_TRACING_POP_ITER, - &&TARGET_TRACING_POP_TOP, - &&TARGET_TRACING_PUSH_EXC_INFO, - &&TARGET_TRACING_PUSH_NULL, - &&TARGET_TRACING_RETURN_GENERATOR, - &&TARGET_TRACING_RETURN_VALUE, - &&TARGET_TRACING_SETUP_ANNOTATIONS, - &&TARGET_TRACING_STORE_SLICE, - &&TARGET_TRACING_STORE_SUBSCR, - &&TARGET_TRACING_TO_BOOL, - &&TARGET_TRACING_UNARY_INVERT, - &&TARGET_TRACING_UNARY_NEGATIVE, - &&TARGET_TRACING_UNARY_NOT, - &&TARGET_TRACING_WITH_EXCEPT_START, - &&TARGET_TRACING_BINARY_OP, - &&TARGET_TRACING_BUILD_INTERPOLATION, - &&TARGET_TRACING_BUILD_LIST, - &&TARGET_TRACING_BUILD_MAP, - &&TARGET_TRACING_BUILD_SET, - &&TARGET_TRACING_BUILD_SLICE, - &&TARGET_TRACING_BUILD_STRING, - &&TARGET_TRACING_BUILD_TUPLE, - &&TARGET_TRACING_CALL, - &&TARGET_TRACING_CALL_INTRINSIC_1, - &&TARGET_TRACING_CALL_INTRINSIC_2, - &&TARGET_TRACING_CALL_KW, - &&TARGET_TRACING_COMPARE_OP, - &&TARGET_TRACING_CONTAINS_OP, - &&TARGET_TRACING_CONVERT_VALUE, - &&TARGET_TRACING_COPY, - &&TARGET_TRACING_COPY_FREE_VARS, - &&TARGET_TRACING_DELETE_ATTR, - &&TARGET_TRACING_DELETE_DEREF, - &&TARGET_TRACING_DELETE_FAST, - &&TARGET_TRACING_DELETE_GLOBAL, - &&TARGET_TRACING_DELETE_NAME, - &&TARGET_TRACING_DICT_MERGE, - &&TARGET_TRACING_DICT_UPDATE, - &&TARGET_TRACING_END_ASYNC_FOR, - &&TARGET_TRACING_EXTENDED_ARG, - &&TARGET_TRACING_FOR_ITER, - &&TARGET_TRACING_GET_AWAITABLE, - &&TARGET_TRACING_IMPORT_FROM, - &&TARGET_TRACING_IMPORT_NAME, - &&TARGET_TRACING_IS_OP, - &&TARGET_TRACING_JUMP_BACKWARD, - &&TARGET_TRACING_JUMP_BACKWARD_NO_INTERRUPT, - &&TARGET_TRACING_JUMP_FORWARD, - &&TARGET_TRACING_LIST_APPEND, - &&TARGET_TRACING_LIST_EXTEND, - &&TARGET_TRACING_LOAD_ATTR, - &&TARGET_TRACING_LOAD_COMMON_CONSTANT, - &&TARGET_TRACING_LOAD_CONST, - &&TARGET_TRACING_LOAD_DEREF, - &&TARGET_TRACING_LOAD_FAST, - &&TARGET_TRACING_LOAD_FAST_AND_CLEAR, - &&TARGET_TRACING_LOAD_FAST_BORROW, - &&TARGET_TRACING_LOAD_FAST_BORROW_LOAD_FAST_BORROW, - &&TARGET_TRACING_LOAD_FAST_CHECK, - &&TARGET_TRACING_LOAD_FAST_LOAD_FAST, - &&TARGET_TRACING_LOAD_FROM_DICT_OR_DEREF, - &&TARGET_TRACING_LOAD_FROM_DICT_OR_GLOBALS, - &&TARGET_TRACING_LOAD_GLOBAL, - &&TARGET_TRACING_LOAD_NAME, - &&TARGET_TRACING_LOAD_SMALL_INT, - &&TARGET_TRACING_LOAD_SPECIAL, - &&TARGET_TRACING_LOAD_SUPER_ATTR, - &&TARGET_TRACING_MAKE_CELL, - &&TARGET_TRACING_MAP_ADD, - &&TARGET_TRACING_MATCH_CLASS, - &&TARGET_TRACING_POP_JUMP_IF_FALSE, - &&TARGET_TRACING_POP_JUMP_IF_NONE, - &&TARGET_TRACING_POP_JUMP_IF_NOT_NONE, - &&TARGET_TRACING_POP_JUMP_IF_TRUE, - &&TARGET_TRACING_RAISE_VARARGS, - &&TARGET_TRACING_RERAISE, - &&TARGET_TRACING_SEND, - &&TARGET_TRACING_SET_ADD, - &&TARGET_TRACING_SET_FUNCTION_ATTRIBUTE, - &&TARGET_TRACING_SET_UPDATE, - &&TARGET_TRACING_STORE_ATTR, - &&TARGET_TRACING_STORE_DEREF, - &&TARGET_TRACING_STORE_FAST, - &&TARGET_TRACING_STORE_FAST_LOAD_FAST, - &&TARGET_TRACING_STORE_FAST_STORE_FAST, - &&TARGET_TRACING_STORE_GLOBAL, - &&TARGET_TRACING_STORE_NAME, - &&TARGET_TRACING_SWAP, - &&TARGET_TRACING_UNPACK_EX, - &&TARGET_TRACING_UNPACK_SEQUENCE, - &&TARGET_TRACING_YIELD_VALUE, - &&_unknown_opcode, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&TARGET_TRACING_RESUME, - &&TARGET_TRACING_BINARY_OP_ADD_FLOAT, - &&TARGET_TRACING_BINARY_OP_ADD_INT, - &&TARGET_TRACING_BINARY_OP_ADD_UNICODE, - &&TARGET_TRACING_BINARY_OP_EXTEND, - &&TARGET_TRACING_BINARY_OP_MULTIPLY_FLOAT, - &&TARGET_TRACING_BINARY_OP_MULTIPLY_INT, - &&TARGET_TRACING_BINARY_OP_SUBSCR_DICT, - &&TARGET_TRACING_BINARY_OP_SUBSCR_GETITEM, - &&TARGET_TRACING_BINARY_OP_SUBSCR_LIST_INT, - &&TARGET_TRACING_BINARY_OP_SUBSCR_LIST_SLICE, - &&TARGET_TRACING_BINARY_OP_SUBSCR_STR_INT, - &&TARGET_TRACING_BINARY_OP_SUBSCR_TUPLE_INT, - &&TARGET_TRACING_BINARY_OP_SUBTRACT_FLOAT, - &&TARGET_TRACING_BINARY_OP_SUBTRACT_INT, - &&TARGET_TRACING_CALL_ALLOC_AND_ENTER_INIT, - &&TARGET_TRACING_CALL_BOUND_METHOD_EXACT_ARGS, - &&TARGET_TRACING_CALL_BOUND_METHOD_GENERAL, - &&TARGET_TRACING_CALL_BUILTIN_CLASS, - &&TARGET_TRACING_CALL_BUILTIN_FAST, - &&TARGET_TRACING_CALL_BUILTIN_FAST_WITH_KEYWORDS, - &&TARGET_TRACING_CALL_BUILTIN_O, - &&TARGET_TRACING_CALL_ISINSTANCE, - &&TARGET_TRACING_CALL_KW_BOUND_METHOD, - &&TARGET_TRACING_CALL_KW_NON_PY, - &&TARGET_TRACING_CALL_KW_PY, - &&TARGET_TRACING_CALL_LEN, - &&TARGET_TRACING_CALL_LIST_APPEND, - &&TARGET_TRACING_CALL_METHOD_DESCRIPTOR_FAST, - &&TARGET_TRACING_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, - &&TARGET_TRACING_CALL_METHOD_DESCRIPTOR_NOARGS, - &&TARGET_TRACING_CALL_METHOD_DESCRIPTOR_O, - &&TARGET_TRACING_CALL_NON_PY_GENERAL, - &&TARGET_TRACING_CALL_PY_EXACT_ARGS, - &&TARGET_TRACING_CALL_PY_GENERAL, - &&TARGET_TRACING_CALL_STR_1, - &&TARGET_TRACING_CALL_TUPLE_1, - &&TARGET_TRACING_CALL_TYPE_1, - &&TARGET_TRACING_COMPARE_OP_FLOAT, - &&TARGET_TRACING_COMPARE_OP_INT, - &&TARGET_TRACING_COMPARE_OP_STR, - &&TARGET_TRACING_CONTAINS_OP_DICT, - &&TARGET_TRACING_CONTAINS_OP_SET, - &&TARGET_TRACING_FOR_ITER_GEN, - &&TARGET_TRACING_FOR_ITER_LIST, - &&TARGET_TRACING_FOR_ITER_RANGE, - &&TARGET_TRACING_FOR_ITER_TUPLE, - &&TARGET_TRACING_JUMP_BACKWARD_JIT, - &&TARGET_TRACING_JUMP_BACKWARD_NO_JIT, - &&TARGET_TRACING_LOAD_ATTR_CLASS, - &&TARGET_TRACING_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK, - &&TARGET_TRACING_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, - &&TARGET_TRACING_LOAD_ATTR_INSTANCE_VALUE, - &&TARGET_TRACING_LOAD_ATTR_METHOD_LAZY_DICT, - &&TARGET_TRACING_LOAD_ATTR_METHOD_NO_DICT, - &&TARGET_TRACING_LOAD_ATTR_METHOD_WITH_VALUES, - &&TARGET_TRACING_LOAD_ATTR_MODULE, - &&TARGET_TRACING_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, - &&TARGET_TRACING_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, - &&TARGET_TRACING_LOAD_ATTR_PROPERTY, - &&TARGET_TRACING_LOAD_ATTR_SLOT, - &&TARGET_TRACING_LOAD_ATTR_WITH_HINT, - &&TARGET_TRACING_LOAD_GLOBAL_BUILTIN, - &&TARGET_TRACING_LOAD_GLOBAL_MODULE, - &&TARGET_TRACING_LOAD_SUPER_ATTR_ATTR, - &&TARGET_TRACING_LOAD_SUPER_ATTR_METHOD, - &&TARGET_TRACING_RESUME_CHECK, - &&TARGET_TRACING_SEND_GEN, - &&TARGET_TRACING_STORE_ATTR_INSTANCE_VALUE, - &&TARGET_TRACING_STORE_ATTR_SLOT, - &&TARGET_TRACING_STORE_ATTR_WITH_HINT, - &&TARGET_TRACING_STORE_SUBSCR_DICT, - &&TARGET_TRACING_STORE_SUBSCR_LIST_INT, - &&TARGET_TRACING_TO_BOOL_ALWAYS_TRUE, - &&TARGET_TRACING_TO_BOOL_BOOL, - &&TARGET_TRACING_TO_BOOL_INT, - &&TARGET_TRACING_TO_BOOL_LIST, - &&TARGET_TRACING_TO_BOOL_NONE, - &&TARGET_TRACING_TO_BOOL_STR, - &&TARGET_TRACING_UNPACK_SEQUENCE_LIST, - &&TARGET_TRACING_UNPACK_SEQUENCE_TUPLE, - &&TARGET_TRACING_UNPACK_SEQUENCE_TWO_TUPLE, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -493,28 +493,28 @@ static void *opcode_tracing_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&TARGET_TRACING_INSTRUMENTED_END_FOR, - &&TARGET_TRACING_INSTRUMENTED_POP_ITER, - &&TARGET_TRACING_INSTRUMENTED_END_SEND, - &&TARGET_TRACING_INSTRUMENTED_FOR_ITER, - &&TARGET_TRACING_INSTRUMENTED_INSTRUCTION, - &&TARGET_TRACING_INSTRUMENTED_JUMP_FORWARD, - &&TARGET_TRACING_INSTRUMENTED_NOT_TAKEN, - &&TARGET_TRACING_INSTRUMENTED_POP_JUMP_IF_TRUE, - &&TARGET_TRACING_INSTRUMENTED_POP_JUMP_IF_FALSE, - &&TARGET_TRACING_INSTRUMENTED_POP_JUMP_IF_NONE, - &&TARGET_TRACING_INSTRUMENTED_POP_JUMP_IF_NOT_NONE, - &&TARGET_TRACING_INSTRUMENTED_RESUME, - &&TARGET_TRACING_INSTRUMENTED_RETURN_VALUE, - &&TARGET_TRACING_INSTRUMENTED_YIELD_VALUE, - &&TARGET_TRACING_INSTRUMENTED_END_ASYNC_FOR, - &&TARGET_TRACING_INSTRUMENTED_LOAD_SUPER_ATTR, - &&TARGET_TRACING_INSTRUMENTED_CALL, - &&TARGET_TRACING_INSTRUMENTED_CALL_KW, - &&TARGET_TRACING_INSTRUMENTED_CALL_FUNCTION_EX, - &&TARGET_TRACING_INSTRUMENTED_JUMP_BACKWARD, - &&TARGET_TRACING_INSTRUMENTED_LINE, - &&TARGET_TRACING_ENTER_EXECUTOR, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_RECORD_PREVIOUS_INST, }; #endif #else /* _Py_TAIL_CALL_INTERP */ @@ -530,455 +530,231 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_start_frame(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_INT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_ADD_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_UNICODE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_ADD_UNICODE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_EXTEND(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_EXTEND(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_INPLACE_ADD_UNICODE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_INPLACE_ADD_UNICODE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_FLOAT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_MULTIPLY_FLOAT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_MULTIPLY_INT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_MULTIPLY_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBSCR_DICT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_SUBSCR_DICT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBSCR_GETITEM(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_SUBSCR_GETITEM(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBSCR_LIST_SLICE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_SUBSCR_LIST_SLICE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBSCR_STR_INT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_SUBSCR_STR_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBSCR_TUPLE_INT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_SUBSCR_TUPLE_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_FLOAT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_SUBTRACT_FLOAT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_SUBTRACT_INT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_OP_SUBTRACT_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_SLICE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BINARY_SLICE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_INTERPOLATION(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BUILD_INTERPOLATION(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_LIST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BUILD_LIST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_MAP(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BUILD_MAP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SET(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BUILD_SET(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_SLICE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BUILD_SLICE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_STRING(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BUILD_STRING(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_TEMPLATE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BUILD_TEMPLATE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BUILD_TUPLE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_BUILD_TUPLE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CACHE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CACHE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ALLOC_AND_ENTER_INIT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_ALLOC_AND_ENTER_INIT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_BOUND_METHOD_EXACT_ARGS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_BOUND_METHOD_GENERAL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_CLASS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_BUILTIN_CLASS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_BUILTIN_FAST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_BUILTIN_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_BUILTIN_O(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_BUILTIN_O(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_1(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_INTRINSIC_1(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_INTRINSIC_2(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_INTRINSIC_2(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_ISINSTANCE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_ISINSTANCE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_KW(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_KW_BOUND_METHOD(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_NON_PY(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_KW_NON_PY(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_KW_PY(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_KW_PY(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LEN(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_LEN(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_LIST_APPEND(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_METHOD_DESCRIPTOR_FAST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_NOARGS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_METHOD_DESCRIPTOR_NOARGS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_METHOD_DESCRIPTOR_O(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_METHOD_DESCRIPTOR_O(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_NON_PY_GENERAL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_NON_PY_GENERAL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_PY_EXACT_ARGS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_PY_GENERAL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_PY_GENERAL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_STR_1(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_STR_1(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TUPLE_1(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_TUPLE_1(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CALL_TYPE_1(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CALL_TYPE_1(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EG_MATCH(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CHECK_EG_MATCH(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CHECK_EXC_MATCH(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CHECK_EXC_MATCH(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CLEANUP_THROW(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CLEANUP_THROW(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_COMPARE_OP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_FLOAT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_COMPARE_OP_FLOAT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_INT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_COMPARE_OP_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COMPARE_OP_STR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_COMPARE_OP_STR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CONTAINS_OP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_DICT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CONTAINS_OP_DICT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONTAINS_OP_SET(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CONTAINS_OP_SET(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_CONVERT_VALUE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_CONVERT_VALUE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_COPY(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_COPY_FREE_VARS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_COPY_FREE_VARS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_ATTR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_DELETE_ATTR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_DEREF(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_DELETE_DEREF(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_FAST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_DELETE_FAST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_GLOBAL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_DELETE_GLOBAL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_NAME(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_DELETE_NAME(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DELETE_SUBSCR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_DELETE_SUBSCR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_MERGE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_DICT_MERGE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_DICT_UPDATE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_DICT_UPDATE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_ASYNC_FOR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_END_ASYNC_FOR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_FOR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_END_FOR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_END_SEND(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_END_SEND(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_ENTER_EXECUTOR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_ENTER_EXECUTOR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXIT_INIT_CHECK(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_EXIT_INIT_CHECK(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_EXTENDED_ARG(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_EXTENDED_ARG(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_SIMPLE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_FORMAT_SIMPLE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FORMAT_WITH_SPEC(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_FORMAT_WITH_SPEC(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_FOR_ITER(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_GEN(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_FOR_ITER_GEN(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_LIST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_FOR_ITER_LIST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_RANGE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_FOR_ITER_RANGE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_FOR_ITER_TUPLE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_FOR_ITER_TUPLE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AITER(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_GET_AITER(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ANEXT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_GET_ANEXT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_AWAITABLE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_GET_AWAITABLE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_ITER(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_GET_ITER(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_LEN(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_GET_LEN(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_GET_YIELD_FROM_ITER(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_GET_YIELD_FROM_ITER(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_FROM(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_IMPORT_FROM(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IMPORT_NAME(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_IMPORT_NAME(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_CALL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_CALL_FUNCTION_EX(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_CALL_KW(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_CALL_KW(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_ASYNC_FOR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_END_ASYNC_FOR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_FOR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_END_FOR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_END_SEND(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_END_SEND(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_FOR_ITER(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_FOR_ITER(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_INSTRUCTION(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_INSTRUCTION(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_BACKWARD(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_JUMP_BACKWARD(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_JUMP_FORWARD(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_JUMP_FORWARD(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LINE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_LINE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_NOT_TAKEN(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_NOT_TAKEN(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_ITER(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_POP_ITER(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RESUME(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_RESUME(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_RETURN_VALUE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_RETURN_VALUE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INSTRUMENTED_YIELD_VALUE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INSTRUMENTED_YIELD_VALUE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_INTERPRETER_EXIT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_INTERPRETER_EXIT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_IS_OP(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_IS_OP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_JUMP_BACKWARD(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_JIT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_JUMP_BACKWARD_JIT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_INTERRUPT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_JUMP_BACKWARD_NO_INTERRUPT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_BACKWARD_NO_JIT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_JUMP_BACKWARD_NO_JIT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_JUMP_FORWARD(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_JUMP_FORWARD(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_APPEND(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LIST_APPEND(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LIST_EXTEND(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LIST_EXTEND(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_CLASS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_METHOD_LAZY_DICT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_NO_DICT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_METHOD_NO_DICT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_METHOD_WITH_VALUES(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_METHOD_WITH_VALUES(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_MODULE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_MODULE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_NONDESCRIPTOR_NO_DICT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_PROPERTY(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_PROPERTY(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_SLOT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_ATTR_WITH_HINT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_ATTR_WITH_HINT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_BUILD_CLASS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_BUILD_CLASS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_COMMON_CONSTANT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_COMMON_CONSTANT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_CONST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_CONST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_DEREF(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_DEREF(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_FAST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_AND_CLEAR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_FAST_AND_CLEAR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_BORROW(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_FAST_BORROW(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_BORROW_LOAD_FAST_BORROW(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_FAST_BORROW_LOAD_FAST_BORROW(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_CHECK(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_FAST_CHECK(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FAST_LOAD_FAST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_FAST_LOAD_FAST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_DEREF(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_FROM_DICT_OR_DEREF(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_FROM_DICT_OR_GLOBALS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_FROM_DICT_OR_GLOBALS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_GLOBAL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_GLOBAL_BUILTIN(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_GLOBAL_MODULE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_GLOBAL_MODULE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_LOCALS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_LOCALS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_NAME(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_NAME(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SMALL_INT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_SMALL_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SPECIAL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_SPECIAL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_SUPER_ATTR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_SUPER_ATTR_ATTR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_LOAD_SUPER_ATTR_METHOD(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_LOAD_SUPER_ATTR_METHOD(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_CELL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_MAKE_CELL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAKE_FUNCTION(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_MAKE_FUNCTION(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MAP_ADD(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_MAP_ADD(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_CLASS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_MATCH_CLASS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_KEYS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_MATCH_KEYS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_MAPPING(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_MATCH_MAPPING(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_MATCH_SEQUENCE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_MATCH_SEQUENCE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOP(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_NOP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_NOT_TAKEN(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_NOT_TAKEN(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_EXCEPT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_POP_EXCEPT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_ITER(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_POP_ITER(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_POP_JUMP_IF_FALSE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_POP_JUMP_IF_NONE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_POP_JUMP_IF_NOT_NONE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_POP_JUMP_IF_TRUE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_TOP(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_POP_TOP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_EXC_INFO(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_PUSH_EXC_INFO(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_NULL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_PUSH_NULL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RAISE_VARARGS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_RAISE_VARARGS(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RECORD_PREVIOUS_INST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RERAISE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_RERAISE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESERVED(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_RESERVED(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_RESUME(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME_CHECK(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_RESUME_CHECK(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_GENERATOR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_RETURN_GENERATOR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RETURN_VALUE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_RETURN_VALUE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_SEND(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SEND_GEN(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_SEND_GEN(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SETUP_ANNOTATIONS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_SETUP_ANNOTATIONS(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_ADD(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_SET_ADD(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_FUNCTION_ATTRIBUTE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_SET_FUNCTION_ATTRIBUTE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SET_UPDATE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_SET_UPDATE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_ATTR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_ATTR_INSTANCE_VALUE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_SLOT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_ATTR_SLOT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_ATTR_WITH_HINT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_ATTR_WITH_HINT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_DEREF(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_DEREF(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_FAST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_LOAD_FAST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_FAST_LOAD_FAST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_FAST_STORE_FAST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_FAST_STORE_FAST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_GLOBAL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_GLOBAL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_NAME(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_NAME(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SLICE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_SLICE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_SUBSCR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_DICT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_SUBSCR_DICT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_STORE_SUBSCR_LIST_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_SWAP(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_SWAP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_TO_BOOL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_TO_BOOL_ALWAYS_TRUE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_BOOL(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_TO_BOOL_BOOL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_INT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_TO_BOOL_INT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_LIST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_TO_BOOL_LIST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_NONE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_TO_BOOL_NONE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TO_BOOL_STR(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_TO_BOOL_STR(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_INVERT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_UNARY_INVERT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NEGATIVE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_UNARY_NEGATIVE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNARY_NOT(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_UNARY_NOT(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_EX(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_UNPACK_EX(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_UNPACK_SEQUENCE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_LIST(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_UNPACK_SEQUENCE_LIST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_UNPACK_SEQUENCE_TUPLE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_UNPACK_SEQUENCE_TWO_TUPLE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_WITH_EXCEPT_START(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_WITH_EXCEPT_START(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_YIELD_VALUE(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_TRACING_YIELD_VALUE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_UNKNOWN_OPCODE(TAIL_CALL_PARAMS) { int opcode = next_instr->op.code; @@ -1172,6 +948,7 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [PUSH_EXC_INFO] = _TAIL_CALL_PUSH_EXC_INFO, [PUSH_NULL] = _TAIL_CALL_PUSH_NULL, [RAISE_VARARGS] = _TAIL_CALL_RAISE_VARARGS, + [RECORD_PREVIOUS_INST] = _TAIL_CALL_RECORD_PREVIOUS_INST, [RERAISE] = _TAIL_CALL_RERAISE, [RESERVED] = _TAIL_CALL_RESERVED, [RESUME] = _TAIL_CALL_RESUME, @@ -1216,7 +993,6 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE, [WITH_EXCEPT_START] = _TAIL_CALL_WITH_EXCEPT_START, [YIELD_VALUE] = _TAIL_CALL_YIELD_VALUE, - [121] = _TAIL_CALL_UNKNOWN_OPCODE, [122] = _TAIL_CALL_UNKNOWN_OPCODE, [123] = _TAIL_CALL_UNKNOWN_OPCODE, [124] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -1249,232 +1025,232 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [233] = _TAIL_CALL_UNKNOWN_OPCODE, }; static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { - [BINARY_OP] = _TAIL_CALL_TRACING_BINARY_OP, - [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_TRACING_BINARY_OP_ADD_FLOAT, - [BINARY_OP_ADD_INT] = _TAIL_CALL_TRACING_BINARY_OP_ADD_INT, - [BINARY_OP_ADD_UNICODE] = _TAIL_CALL_TRACING_BINARY_OP_ADD_UNICODE, - [BINARY_OP_EXTEND] = _TAIL_CALL_TRACING_BINARY_OP_EXTEND, - [BINARY_OP_INPLACE_ADD_UNICODE] = _TAIL_CALL_TRACING_BINARY_OP_INPLACE_ADD_UNICODE, - [BINARY_OP_MULTIPLY_FLOAT] = _TAIL_CALL_TRACING_BINARY_OP_MULTIPLY_FLOAT, - [BINARY_OP_MULTIPLY_INT] = _TAIL_CALL_TRACING_BINARY_OP_MULTIPLY_INT, - [BINARY_OP_SUBSCR_DICT] = _TAIL_CALL_TRACING_BINARY_OP_SUBSCR_DICT, - [BINARY_OP_SUBSCR_GETITEM] = _TAIL_CALL_TRACING_BINARY_OP_SUBSCR_GETITEM, - [BINARY_OP_SUBSCR_LIST_INT] = _TAIL_CALL_TRACING_BINARY_OP_SUBSCR_LIST_INT, - [BINARY_OP_SUBSCR_LIST_SLICE] = _TAIL_CALL_TRACING_BINARY_OP_SUBSCR_LIST_SLICE, - [BINARY_OP_SUBSCR_STR_INT] = _TAIL_CALL_TRACING_BINARY_OP_SUBSCR_STR_INT, - [BINARY_OP_SUBSCR_TUPLE_INT] = _TAIL_CALL_TRACING_BINARY_OP_SUBSCR_TUPLE_INT, - [BINARY_OP_SUBTRACT_FLOAT] = _TAIL_CALL_TRACING_BINARY_OP_SUBTRACT_FLOAT, - [BINARY_OP_SUBTRACT_INT] = _TAIL_CALL_TRACING_BINARY_OP_SUBTRACT_INT, - [BINARY_SLICE] = _TAIL_CALL_TRACING_BINARY_SLICE, - [BUILD_INTERPOLATION] = _TAIL_CALL_TRACING_BUILD_INTERPOLATION, - [BUILD_LIST] = _TAIL_CALL_TRACING_BUILD_LIST, - [BUILD_MAP] = _TAIL_CALL_TRACING_BUILD_MAP, - [BUILD_SET] = _TAIL_CALL_TRACING_BUILD_SET, - [BUILD_SLICE] = _TAIL_CALL_TRACING_BUILD_SLICE, - [BUILD_STRING] = _TAIL_CALL_TRACING_BUILD_STRING, - [BUILD_TEMPLATE] = _TAIL_CALL_TRACING_BUILD_TEMPLATE, - [BUILD_TUPLE] = _TAIL_CALL_TRACING_BUILD_TUPLE, - [CACHE] = _TAIL_CALL_TRACING_CACHE, - [CALL] = _TAIL_CALL_TRACING_CALL, - [CALL_ALLOC_AND_ENTER_INIT] = _TAIL_CALL_TRACING_CALL_ALLOC_AND_ENTER_INIT, - [CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_TRACING_CALL_BOUND_METHOD_EXACT_ARGS, - [CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_TRACING_CALL_BOUND_METHOD_GENERAL, - [CALL_BUILTIN_CLASS] = _TAIL_CALL_TRACING_CALL_BUILTIN_CLASS, - [CALL_BUILTIN_FAST] = _TAIL_CALL_TRACING_CALL_BUILTIN_FAST, - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = _TAIL_CALL_TRACING_CALL_BUILTIN_FAST_WITH_KEYWORDS, - [CALL_BUILTIN_O] = _TAIL_CALL_TRACING_CALL_BUILTIN_O, - [CALL_FUNCTION_EX] = _TAIL_CALL_TRACING_CALL_FUNCTION_EX, - [CALL_INTRINSIC_1] = _TAIL_CALL_TRACING_CALL_INTRINSIC_1, - [CALL_INTRINSIC_2] = _TAIL_CALL_TRACING_CALL_INTRINSIC_2, - [CALL_ISINSTANCE] = _TAIL_CALL_TRACING_CALL_ISINSTANCE, - [CALL_KW] = _TAIL_CALL_TRACING_CALL_KW, - [CALL_KW_BOUND_METHOD] = _TAIL_CALL_TRACING_CALL_KW_BOUND_METHOD, - [CALL_KW_NON_PY] = _TAIL_CALL_TRACING_CALL_KW_NON_PY, - [CALL_KW_PY] = _TAIL_CALL_TRACING_CALL_KW_PY, - [CALL_LEN] = _TAIL_CALL_TRACING_CALL_LEN, - [CALL_LIST_APPEND] = _TAIL_CALL_TRACING_CALL_LIST_APPEND, - [CALL_METHOD_DESCRIPTOR_FAST] = _TAIL_CALL_TRACING_CALL_METHOD_DESCRIPTOR_FAST, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = _TAIL_CALL_TRACING_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, - [CALL_METHOD_DESCRIPTOR_NOARGS] = _TAIL_CALL_TRACING_CALL_METHOD_DESCRIPTOR_NOARGS, - [CALL_METHOD_DESCRIPTOR_O] = _TAIL_CALL_TRACING_CALL_METHOD_DESCRIPTOR_O, - [CALL_NON_PY_GENERAL] = _TAIL_CALL_TRACING_CALL_NON_PY_GENERAL, - [CALL_PY_EXACT_ARGS] = _TAIL_CALL_TRACING_CALL_PY_EXACT_ARGS, - [CALL_PY_GENERAL] = _TAIL_CALL_TRACING_CALL_PY_GENERAL, - [CALL_STR_1] = _TAIL_CALL_TRACING_CALL_STR_1, - [CALL_TUPLE_1] = _TAIL_CALL_TRACING_CALL_TUPLE_1, - [CALL_TYPE_1] = _TAIL_CALL_TRACING_CALL_TYPE_1, - [CHECK_EG_MATCH] = _TAIL_CALL_TRACING_CHECK_EG_MATCH, - [CHECK_EXC_MATCH] = _TAIL_CALL_TRACING_CHECK_EXC_MATCH, - [CLEANUP_THROW] = _TAIL_CALL_TRACING_CLEANUP_THROW, - [COMPARE_OP] = _TAIL_CALL_TRACING_COMPARE_OP, - [COMPARE_OP_FLOAT] = _TAIL_CALL_TRACING_COMPARE_OP_FLOAT, - [COMPARE_OP_INT] = _TAIL_CALL_TRACING_COMPARE_OP_INT, - [COMPARE_OP_STR] = _TAIL_CALL_TRACING_COMPARE_OP_STR, - [CONTAINS_OP] = _TAIL_CALL_TRACING_CONTAINS_OP, - [CONTAINS_OP_DICT] = _TAIL_CALL_TRACING_CONTAINS_OP_DICT, - [CONTAINS_OP_SET] = _TAIL_CALL_TRACING_CONTAINS_OP_SET, - [CONVERT_VALUE] = _TAIL_CALL_TRACING_CONVERT_VALUE, - [COPY] = _TAIL_CALL_TRACING_COPY, - [COPY_FREE_VARS] = _TAIL_CALL_TRACING_COPY_FREE_VARS, - [DELETE_ATTR] = _TAIL_CALL_TRACING_DELETE_ATTR, - [DELETE_DEREF] = _TAIL_CALL_TRACING_DELETE_DEREF, - [DELETE_FAST] = _TAIL_CALL_TRACING_DELETE_FAST, - [DELETE_GLOBAL] = _TAIL_CALL_TRACING_DELETE_GLOBAL, - [DELETE_NAME] = _TAIL_CALL_TRACING_DELETE_NAME, - [DELETE_SUBSCR] = _TAIL_CALL_TRACING_DELETE_SUBSCR, - [DICT_MERGE] = _TAIL_CALL_TRACING_DICT_MERGE, - [DICT_UPDATE] = _TAIL_CALL_TRACING_DICT_UPDATE, - [END_ASYNC_FOR] = _TAIL_CALL_TRACING_END_ASYNC_FOR, - [END_FOR] = _TAIL_CALL_TRACING_END_FOR, - [END_SEND] = _TAIL_CALL_TRACING_END_SEND, - [ENTER_EXECUTOR] = _TAIL_CALL_TRACING_ENTER_EXECUTOR, - [EXIT_INIT_CHECK] = _TAIL_CALL_TRACING_EXIT_INIT_CHECK, - [EXTENDED_ARG] = _TAIL_CALL_TRACING_EXTENDED_ARG, - [FORMAT_SIMPLE] = _TAIL_CALL_TRACING_FORMAT_SIMPLE, - [FORMAT_WITH_SPEC] = _TAIL_CALL_TRACING_FORMAT_WITH_SPEC, - [FOR_ITER] = _TAIL_CALL_TRACING_FOR_ITER, - [FOR_ITER_GEN] = _TAIL_CALL_TRACING_FOR_ITER_GEN, - [FOR_ITER_LIST] = _TAIL_CALL_TRACING_FOR_ITER_LIST, - [FOR_ITER_RANGE] = _TAIL_CALL_TRACING_FOR_ITER_RANGE, - [FOR_ITER_TUPLE] = _TAIL_CALL_TRACING_FOR_ITER_TUPLE, - [GET_AITER] = _TAIL_CALL_TRACING_GET_AITER, - [GET_ANEXT] = _TAIL_CALL_TRACING_GET_ANEXT, - [GET_AWAITABLE] = _TAIL_CALL_TRACING_GET_AWAITABLE, - [GET_ITER] = _TAIL_CALL_TRACING_GET_ITER, - [GET_LEN] = _TAIL_CALL_TRACING_GET_LEN, - [GET_YIELD_FROM_ITER] = _TAIL_CALL_TRACING_GET_YIELD_FROM_ITER, - [IMPORT_FROM] = _TAIL_CALL_TRACING_IMPORT_FROM, - [IMPORT_NAME] = _TAIL_CALL_TRACING_IMPORT_NAME, - [INSTRUMENTED_CALL] = _TAIL_CALL_TRACING_INSTRUMENTED_CALL, - [INSTRUMENTED_CALL_FUNCTION_EX] = _TAIL_CALL_TRACING_INSTRUMENTED_CALL_FUNCTION_EX, - [INSTRUMENTED_CALL_KW] = _TAIL_CALL_TRACING_INSTRUMENTED_CALL_KW, - [INSTRUMENTED_END_ASYNC_FOR] = _TAIL_CALL_TRACING_INSTRUMENTED_END_ASYNC_FOR, - [INSTRUMENTED_END_FOR] = _TAIL_CALL_TRACING_INSTRUMENTED_END_FOR, - [INSTRUMENTED_END_SEND] = _TAIL_CALL_TRACING_INSTRUMENTED_END_SEND, - [INSTRUMENTED_FOR_ITER] = _TAIL_CALL_TRACING_INSTRUMENTED_FOR_ITER, - [INSTRUMENTED_INSTRUCTION] = _TAIL_CALL_TRACING_INSTRUMENTED_INSTRUCTION, - [INSTRUMENTED_JUMP_BACKWARD] = _TAIL_CALL_TRACING_INSTRUMENTED_JUMP_BACKWARD, - [INSTRUMENTED_JUMP_FORWARD] = _TAIL_CALL_TRACING_INSTRUMENTED_JUMP_FORWARD, - [INSTRUMENTED_LINE] = _TAIL_CALL_TRACING_INSTRUMENTED_LINE, - [INSTRUMENTED_LOAD_SUPER_ATTR] = _TAIL_CALL_TRACING_INSTRUMENTED_LOAD_SUPER_ATTR, - [INSTRUMENTED_NOT_TAKEN] = _TAIL_CALL_TRACING_INSTRUMENTED_NOT_TAKEN, - [INSTRUMENTED_POP_ITER] = _TAIL_CALL_TRACING_INSTRUMENTED_POP_ITER, - [INSTRUMENTED_POP_JUMP_IF_FALSE] = _TAIL_CALL_TRACING_INSTRUMENTED_POP_JUMP_IF_FALSE, - [INSTRUMENTED_POP_JUMP_IF_NONE] = _TAIL_CALL_TRACING_INSTRUMENTED_POP_JUMP_IF_NONE, - [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_TRACING_INSTRUMENTED_POP_JUMP_IF_NOT_NONE, - [INSTRUMENTED_POP_JUMP_IF_TRUE] = _TAIL_CALL_TRACING_INSTRUMENTED_POP_JUMP_IF_TRUE, - [INSTRUMENTED_RESUME] = _TAIL_CALL_TRACING_INSTRUMENTED_RESUME, - [INSTRUMENTED_RETURN_VALUE] = _TAIL_CALL_TRACING_INSTRUMENTED_RETURN_VALUE, - [INSTRUMENTED_YIELD_VALUE] = _TAIL_CALL_TRACING_INSTRUMENTED_YIELD_VALUE, - [INTERPRETER_EXIT] = _TAIL_CALL_TRACING_INTERPRETER_EXIT, - [IS_OP] = _TAIL_CALL_TRACING_IS_OP, - [JUMP_BACKWARD] = _TAIL_CALL_TRACING_JUMP_BACKWARD, - [JUMP_BACKWARD_JIT] = _TAIL_CALL_TRACING_JUMP_BACKWARD_JIT, - [JUMP_BACKWARD_NO_INTERRUPT] = _TAIL_CALL_TRACING_JUMP_BACKWARD_NO_INTERRUPT, - [JUMP_BACKWARD_NO_JIT] = _TAIL_CALL_TRACING_JUMP_BACKWARD_NO_JIT, - [JUMP_FORWARD] = _TAIL_CALL_TRACING_JUMP_FORWARD, - [LIST_APPEND] = _TAIL_CALL_TRACING_LIST_APPEND, - [LIST_EXTEND] = _TAIL_CALL_TRACING_LIST_EXTEND, - [LOAD_ATTR] = _TAIL_CALL_TRACING_LOAD_ATTR, - [LOAD_ATTR_CLASS] = _TAIL_CALL_TRACING_LOAD_ATTR_CLASS, - [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = _TAIL_CALL_TRACING_LOAD_ATTR_CLASS_WITH_METACLASS_CHECK, - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = _TAIL_CALL_TRACING_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, - [LOAD_ATTR_INSTANCE_VALUE] = _TAIL_CALL_TRACING_LOAD_ATTR_INSTANCE_VALUE, - [LOAD_ATTR_METHOD_LAZY_DICT] = _TAIL_CALL_TRACING_LOAD_ATTR_METHOD_LAZY_DICT, - [LOAD_ATTR_METHOD_NO_DICT] = _TAIL_CALL_TRACING_LOAD_ATTR_METHOD_NO_DICT, - [LOAD_ATTR_METHOD_WITH_VALUES] = _TAIL_CALL_TRACING_LOAD_ATTR_METHOD_WITH_VALUES, - [LOAD_ATTR_MODULE] = _TAIL_CALL_TRACING_LOAD_ATTR_MODULE, - [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = _TAIL_CALL_TRACING_LOAD_ATTR_NONDESCRIPTOR_NO_DICT, - [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = _TAIL_CALL_TRACING_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, - [LOAD_ATTR_PROPERTY] = _TAIL_CALL_TRACING_LOAD_ATTR_PROPERTY, - [LOAD_ATTR_SLOT] = _TAIL_CALL_TRACING_LOAD_ATTR_SLOT, - [LOAD_ATTR_WITH_HINT] = _TAIL_CALL_TRACING_LOAD_ATTR_WITH_HINT, - [LOAD_BUILD_CLASS] = _TAIL_CALL_TRACING_LOAD_BUILD_CLASS, - [LOAD_COMMON_CONSTANT] = _TAIL_CALL_TRACING_LOAD_COMMON_CONSTANT, - [LOAD_CONST] = _TAIL_CALL_TRACING_LOAD_CONST, - [LOAD_DEREF] = _TAIL_CALL_TRACING_LOAD_DEREF, - [LOAD_FAST] = _TAIL_CALL_TRACING_LOAD_FAST, - [LOAD_FAST_AND_CLEAR] = _TAIL_CALL_TRACING_LOAD_FAST_AND_CLEAR, - [LOAD_FAST_BORROW] = _TAIL_CALL_TRACING_LOAD_FAST_BORROW, - [LOAD_FAST_BORROW_LOAD_FAST_BORROW] = _TAIL_CALL_TRACING_LOAD_FAST_BORROW_LOAD_FAST_BORROW, - [LOAD_FAST_CHECK] = _TAIL_CALL_TRACING_LOAD_FAST_CHECK, - [LOAD_FAST_LOAD_FAST] = _TAIL_CALL_TRACING_LOAD_FAST_LOAD_FAST, - [LOAD_FROM_DICT_OR_DEREF] = _TAIL_CALL_TRACING_LOAD_FROM_DICT_OR_DEREF, - [LOAD_FROM_DICT_OR_GLOBALS] = _TAIL_CALL_TRACING_LOAD_FROM_DICT_OR_GLOBALS, - [LOAD_GLOBAL] = _TAIL_CALL_TRACING_LOAD_GLOBAL, - [LOAD_GLOBAL_BUILTIN] = _TAIL_CALL_TRACING_LOAD_GLOBAL_BUILTIN, - [LOAD_GLOBAL_MODULE] = _TAIL_CALL_TRACING_LOAD_GLOBAL_MODULE, - [LOAD_LOCALS] = _TAIL_CALL_TRACING_LOAD_LOCALS, - [LOAD_NAME] = _TAIL_CALL_TRACING_LOAD_NAME, - [LOAD_SMALL_INT] = _TAIL_CALL_TRACING_LOAD_SMALL_INT, - [LOAD_SPECIAL] = _TAIL_CALL_TRACING_LOAD_SPECIAL, - [LOAD_SUPER_ATTR] = _TAIL_CALL_TRACING_LOAD_SUPER_ATTR, - [LOAD_SUPER_ATTR_ATTR] = _TAIL_CALL_TRACING_LOAD_SUPER_ATTR_ATTR, - [LOAD_SUPER_ATTR_METHOD] = _TAIL_CALL_TRACING_LOAD_SUPER_ATTR_METHOD, - [MAKE_CELL] = _TAIL_CALL_TRACING_MAKE_CELL, - [MAKE_FUNCTION] = _TAIL_CALL_TRACING_MAKE_FUNCTION, - [MAP_ADD] = _TAIL_CALL_TRACING_MAP_ADD, - [MATCH_CLASS] = _TAIL_CALL_TRACING_MATCH_CLASS, - [MATCH_KEYS] = _TAIL_CALL_TRACING_MATCH_KEYS, - [MATCH_MAPPING] = _TAIL_CALL_TRACING_MATCH_MAPPING, - [MATCH_SEQUENCE] = _TAIL_CALL_TRACING_MATCH_SEQUENCE, - [NOP] = _TAIL_CALL_TRACING_NOP, - [NOT_TAKEN] = _TAIL_CALL_TRACING_NOT_TAKEN, - [POP_EXCEPT] = _TAIL_CALL_TRACING_POP_EXCEPT, - [POP_ITER] = _TAIL_CALL_TRACING_POP_ITER, - [POP_JUMP_IF_FALSE] = _TAIL_CALL_TRACING_POP_JUMP_IF_FALSE, - [POP_JUMP_IF_NONE] = _TAIL_CALL_TRACING_POP_JUMP_IF_NONE, - [POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_TRACING_POP_JUMP_IF_NOT_NONE, - [POP_JUMP_IF_TRUE] = _TAIL_CALL_TRACING_POP_JUMP_IF_TRUE, - [POP_TOP] = _TAIL_CALL_TRACING_POP_TOP, - [PUSH_EXC_INFO] = _TAIL_CALL_TRACING_PUSH_EXC_INFO, - [PUSH_NULL] = _TAIL_CALL_TRACING_PUSH_NULL, - [RAISE_VARARGS] = _TAIL_CALL_TRACING_RAISE_VARARGS, - [RERAISE] = _TAIL_CALL_TRACING_RERAISE, - [RESERVED] = _TAIL_CALL_TRACING_RESERVED, - [RESUME] = _TAIL_CALL_TRACING_RESUME, - [RESUME_CHECK] = _TAIL_CALL_TRACING_RESUME_CHECK, - [RETURN_GENERATOR] = _TAIL_CALL_TRACING_RETURN_GENERATOR, - [RETURN_VALUE] = _TAIL_CALL_TRACING_RETURN_VALUE, - [SEND] = _TAIL_CALL_TRACING_SEND, - [SEND_GEN] = _TAIL_CALL_TRACING_SEND_GEN, - [SETUP_ANNOTATIONS] = _TAIL_CALL_TRACING_SETUP_ANNOTATIONS, - [SET_ADD] = _TAIL_CALL_TRACING_SET_ADD, - [SET_FUNCTION_ATTRIBUTE] = _TAIL_CALL_TRACING_SET_FUNCTION_ATTRIBUTE, - [SET_UPDATE] = _TAIL_CALL_TRACING_SET_UPDATE, - [STORE_ATTR] = _TAIL_CALL_TRACING_STORE_ATTR, - [STORE_ATTR_INSTANCE_VALUE] = _TAIL_CALL_TRACING_STORE_ATTR_INSTANCE_VALUE, - [STORE_ATTR_SLOT] = _TAIL_CALL_TRACING_STORE_ATTR_SLOT, - [STORE_ATTR_WITH_HINT] = _TAIL_CALL_TRACING_STORE_ATTR_WITH_HINT, - [STORE_DEREF] = _TAIL_CALL_TRACING_STORE_DEREF, - [STORE_FAST] = _TAIL_CALL_TRACING_STORE_FAST, - [STORE_FAST_LOAD_FAST] = _TAIL_CALL_TRACING_STORE_FAST_LOAD_FAST, - [STORE_FAST_STORE_FAST] = _TAIL_CALL_TRACING_STORE_FAST_STORE_FAST, - [STORE_GLOBAL] = _TAIL_CALL_TRACING_STORE_GLOBAL, - [STORE_NAME] = _TAIL_CALL_TRACING_STORE_NAME, - [STORE_SLICE] = _TAIL_CALL_TRACING_STORE_SLICE, - [STORE_SUBSCR] = _TAIL_CALL_TRACING_STORE_SUBSCR, - [STORE_SUBSCR_DICT] = _TAIL_CALL_TRACING_STORE_SUBSCR_DICT, - [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_TRACING_STORE_SUBSCR_LIST_INT, - [SWAP] = _TAIL_CALL_TRACING_SWAP, - [TO_BOOL] = _TAIL_CALL_TRACING_TO_BOOL, - [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_TRACING_TO_BOOL_ALWAYS_TRUE, - [TO_BOOL_BOOL] = _TAIL_CALL_TRACING_TO_BOOL_BOOL, - [TO_BOOL_INT] = _TAIL_CALL_TRACING_TO_BOOL_INT, - [TO_BOOL_LIST] = _TAIL_CALL_TRACING_TO_BOOL_LIST, - [TO_BOOL_NONE] = _TAIL_CALL_TRACING_TO_BOOL_NONE, - [TO_BOOL_STR] = _TAIL_CALL_TRACING_TO_BOOL_STR, - [UNARY_INVERT] = _TAIL_CALL_TRACING_UNARY_INVERT, - [UNARY_NEGATIVE] = _TAIL_CALL_TRACING_UNARY_NEGATIVE, - [UNARY_NOT] = _TAIL_CALL_TRACING_UNARY_NOT, - [UNPACK_EX] = _TAIL_CALL_TRACING_UNPACK_EX, - [UNPACK_SEQUENCE] = _TAIL_CALL_TRACING_UNPACK_SEQUENCE, - [UNPACK_SEQUENCE_LIST] = _TAIL_CALL_TRACING_UNPACK_SEQUENCE_LIST, - [UNPACK_SEQUENCE_TUPLE] = _TAIL_CALL_TRACING_UNPACK_SEQUENCE_TUPLE, - [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_TRACING_UNPACK_SEQUENCE_TWO_TUPLE, - [WITH_EXCEPT_START] = _TAIL_CALL_TRACING_WITH_EXCEPT_START, - [YIELD_VALUE] = _TAIL_CALL_TRACING_YIELD_VALUE, - [121] = _TAIL_CALL_UNKNOWN_OPCODE, + [BINARY_OP] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BINARY_OP_ADD_INT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BINARY_OP_ADD_UNICODE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BINARY_OP_EXTEND] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BINARY_OP_INPLACE_ADD_UNICODE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BINARY_OP_MULTIPLY_FLOAT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BINARY_OP_MULTIPLY_INT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BINARY_OP_SUBSCR_DICT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BINARY_OP_SUBSCR_GETITEM] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BINARY_OP_SUBSCR_LIST_INT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BINARY_OP_SUBSCR_LIST_SLICE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BINARY_OP_SUBSCR_STR_INT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BINARY_OP_SUBSCR_TUPLE_INT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BINARY_OP_SUBTRACT_FLOAT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BINARY_OP_SUBTRACT_INT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BINARY_SLICE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BUILD_INTERPOLATION] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BUILD_LIST] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BUILD_MAP] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BUILD_SET] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BUILD_SLICE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BUILD_STRING] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BUILD_TEMPLATE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BUILD_TUPLE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CACHE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_ALLOC_AND_ENTER_INIT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_BUILTIN_CLASS] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_BUILTIN_FAST] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_BUILTIN_O] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_FUNCTION_EX] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_INTRINSIC_1] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_INTRINSIC_2] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_ISINSTANCE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_KW] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_KW_BOUND_METHOD] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_KW_NON_PY] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_KW_PY] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_LEN] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_LIST_APPEND] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_METHOD_DESCRIPTOR_FAST] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_METHOD_DESCRIPTOR_NOARGS] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_METHOD_DESCRIPTOR_O] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_NON_PY_GENERAL] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_PY_EXACT_ARGS] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_PY_GENERAL] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_STR_1] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_TUPLE_1] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CALL_TYPE_1] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CHECK_EG_MATCH] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CHECK_EXC_MATCH] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CLEANUP_THROW] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [COMPARE_OP] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [COMPARE_OP_FLOAT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [COMPARE_OP_INT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [COMPARE_OP_STR] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CONTAINS_OP] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CONTAINS_OP_DICT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CONTAINS_OP_SET] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [CONVERT_VALUE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [COPY] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [COPY_FREE_VARS] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [DELETE_ATTR] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [DELETE_DEREF] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [DELETE_FAST] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [DELETE_GLOBAL] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [DELETE_NAME] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [DELETE_SUBSCR] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [DICT_MERGE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [DICT_UPDATE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [END_ASYNC_FOR] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [END_FOR] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [END_SEND] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [ENTER_EXECUTOR] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [EXIT_INIT_CHECK] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [EXTENDED_ARG] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [FORMAT_SIMPLE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [FORMAT_WITH_SPEC] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [FOR_ITER] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [FOR_ITER_GEN] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [FOR_ITER_LIST] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [FOR_ITER_RANGE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [FOR_ITER_TUPLE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [GET_AITER] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [GET_ANEXT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [GET_AWAITABLE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [GET_ITER] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [GET_LEN] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [GET_YIELD_FROM_ITER] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [IMPORT_FROM] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [IMPORT_NAME] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INSTRUMENTED_CALL] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INSTRUMENTED_CALL_FUNCTION_EX] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INSTRUMENTED_CALL_KW] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INSTRUMENTED_END_ASYNC_FOR] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INSTRUMENTED_END_FOR] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INSTRUMENTED_END_SEND] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INSTRUMENTED_FOR_ITER] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INSTRUMENTED_INSTRUCTION] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INSTRUMENTED_JUMP_BACKWARD] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INSTRUMENTED_JUMP_FORWARD] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INSTRUMENTED_LINE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INSTRUMENTED_LOAD_SUPER_ATTR] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INSTRUMENTED_NOT_TAKEN] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INSTRUMENTED_POP_ITER] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INSTRUMENTED_POP_JUMP_IF_NONE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INSTRUMENTED_RESUME] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INSTRUMENTED_RETURN_VALUE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INSTRUMENTED_YIELD_VALUE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [INTERPRETER_EXIT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [IS_OP] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [JUMP_BACKWARD] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [JUMP_BACKWARD_JIT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [JUMP_BACKWARD_NO_INTERRUPT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [JUMP_BACKWARD_NO_JIT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [JUMP_FORWARD] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LIST_APPEND] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LIST_EXTEND] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_ATTR] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_ATTR_CLASS] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_ATTR_INSTANCE_VALUE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_ATTR_METHOD_LAZY_DICT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_ATTR_METHOD_NO_DICT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_ATTR_METHOD_WITH_VALUES] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_ATTR_MODULE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_ATTR_PROPERTY] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_ATTR_SLOT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_ATTR_WITH_HINT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_BUILD_CLASS] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_COMMON_CONSTANT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_CONST] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_DEREF] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_FAST] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_FAST_AND_CLEAR] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_FAST_BORROW] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_FAST_BORROW_LOAD_FAST_BORROW] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_FAST_CHECK] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_FAST_LOAD_FAST] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_FROM_DICT_OR_DEREF] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_FROM_DICT_OR_GLOBALS] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_GLOBAL] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_GLOBAL_BUILTIN] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_GLOBAL_MODULE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_LOCALS] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_NAME] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_SMALL_INT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_SPECIAL] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_SUPER_ATTR] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_SUPER_ATTR_ATTR] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [LOAD_SUPER_ATTR_METHOD] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [MAKE_CELL] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [MAKE_FUNCTION] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [MAP_ADD] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [MATCH_CLASS] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [MATCH_KEYS] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [MATCH_MAPPING] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [MATCH_SEQUENCE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [NOP] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [NOT_TAKEN] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [POP_EXCEPT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [POP_ITER] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [POP_JUMP_IF_FALSE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [POP_JUMP_IF_NONE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [POP_JUMP_IF_TRUE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [POP_TOP] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [PUSH_EXC_INFO] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [PUSH_NULL] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [RAISE_VARARGS] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [RECORD_PREVIOUS_INST] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [RERAISE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [RESERVED] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [RESUME] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [RESUME_CHECK] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [RETURN_GENERATOR] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [RETURN_VALUE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [SEND] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [SEND_GEN] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [SETUP_ANNOTATIONS] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [SET_ADD] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [SET_FUNCTION_ATTRIBUTE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [SET_UPDATE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [STORE_ATTR] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [STORE_ATTR_INSTANCE_VALUE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [STORE_ATTR_SLOT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [STORE_ATTR_WITH_HINT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [STORE_DEREF] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [STORE_FAST] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [STORE_FAST_LOAD_FAST] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [STORE_FAST_STORE_FAST] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [STORE_GLOBAL] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [STORE_NAME] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [STORE_SLICE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [STORE_SUBSCR] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [STORE_SUBSCR_DICT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [SWAP] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [TO_BOOL] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [TO_BOOL_BOOL] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [TO_BOOL_INT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [TO_BOOL_LIST] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [TO_BOOL_NONE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [TO_BOOL_STR] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [UNARY_INVERT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [UNARY_NEGATIVE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [UNARY_NOT] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [UNPACK_EX] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [UNPACK_SEQUENCE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [UNPACK_SEQUENCE_LIST] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [UNPACK_SEQUENCE_TUPLE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [WITH_EXCEPT_START] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [YIELD_VALUE] = _TAIL_CALL_RECORD_PREVIOUS_INST, [122] = _TAIL_CALL_UNKNOWN_OPCODE, [123] = _TAIL_CALL_UNKNOWN_OPCODE, [124] = _TAIL_CALL_UNKNOWN_OPCODE, diff --git a/Python/optimizer.c b/Python/optimizer.c index 4d9e9e0ae4237f..a3d847b6eab884 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -172,7 +172,7 @@ _PyOptimizer_Optimize( executor->vm_data.code = NULL; } if (chain_depth > 0) { - _PyExitData *prev_exit = tstate->interp->jit_state.previous_exit; + _PyExitData *prev_exit = tstate->interp->jit_state.prev_exit; assert(prev_exit != NULL); prev_exit->executor = executor;; } @@ -554,16 +554,16 @@ int _PyJit_translate_single_bytecode_to_trace( PyThreadState *tstate, _PyInterpreterFrame *frame, - _Py_CODEUNIT *this_instr, - _Py_CODEUNIT *next_instr, - PyCodeObject *old_code, - PyFunctionObject *func, - int old_stack_level, - int opcode, - int oparg, - int jump_taken) + _Py_CODEUNIT *next_instr) { + PyCodeObject *old_code = tstate->interp->jit_state.prev_instr_code; + // Something else finalized the trace. This can happen in multi-threaded scenarios as our trace + // addition from bytecode execution to here is not atomic. + // Though in GIL builds, the GIL protects the rest. + if (old_code == NULL) { + return 0; + } bool progress_needed = (tstate->interp->jit_state.initial_chain_depth % MAX_CHAIN_DEPTH) == 0;; _PyBloomFilter *dependencies = &tstate->interp->jit_state.dependencies; _Py_BloomFilter_Add(dependencies, old_code); @@ -583,6 +583,7 @@ _PyJit_translate_single_bytecode_to_trace( lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that } #endif + _Py_CODEUNIT *this_instr = tstate->interp->jit_state.prev_instr; _Py_CODEUNIT *target_instr = this_instr; uint32_t target = 0; @@ -590,11 +591,17 @@ _PyJit_translate_single_bytecode_to_trace( // Rewind EXTENDED_ARG so that we see the whole thing. // We must point to the first EXTENDED_ARG when deopting. + int oparg = tstate->interp->jit_state.prev_instr_oparg; + int opcode = this_instr->op.code; int rewind_oparg = oparg; while (rewind_oparg > 255) { rewind_oparg >>= 8; target--; } + + bool needs_guard_ip = _PyOpcode_NeedsGuardIp[opcode]; + DPRINTF(2, "%p %d: %s(%d) %d\n", old_code, target, _PyOpcode_OpName[opcode], oparg, needs_guard_ip); + #ifdef Py_DEBUG if (oparg > 255) { assert(_Py_GetBaseCodeUnit(old_code, target).op.code == EXTENDED_ARG); @@ -605,44 +612,52 @@ _PyJit_translate_single_bytecode_to_trace( goto done; } - DPRINTF(2, "%p %d: %s(%d) %d\n", old_code, target, _PyOpcode_OpName[opcode], oparg, progress_needed); + int old_stack_level = tstate->interp->jit_state.prev_instr_oparg; + // Strange control-flow, unsupported opcode, etc. + if (tstate->interp->jit_state.dynamic_jump_taken) { + goto unsupported; + } - bool needs_guard_ip = _PyOpcode_NeedsGuardIp[opcode]; + // This happens when a recursive call happens that we can't trace. Such as Python -> C -> Python calls + // If we haven't guarded the IP, then it's untraceable. + if (frame != tstate->interp->jit_state.prev_instr_frame && !needs_guard_ip) { + goto unsupported; + } - // Strange control-flow, unsupported opcode, etc. - if (jump_taken || - // This happens when a recursive call happens that we can't trace. Such as Python -> C -> Python calls - // If we haven't guarded the IP, then it's untraceable. - (frame != tstate->interp->jit_state.current_frame && !needs_guard_ip) || - (oparg > 0xFFFF) || - // TODO (gh-140277): The constituent use one extra stack slot. So we need to check for headroom. - (opcode == BINARY_OP_SUBSCR_GETITEM && old_stack_level + 1 > old_code->co_stacksize)|| - // Exception stuff, could be handled in the future maybe? - opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO || - frame->owner >= FRAME_OWNED_BY_INTERPRETER - ) { + if (oparg > 0xFFFF) { + goto unsupported; + } + + // TODO (gh-140277): The constituent use one extra stack slot. So we need to check for headroom. + if (opcode == BINARY_OP_SUBSCR_GETITEM && old_stack_level + 1 > old_code->co_stacksize) { + goto unsupported; + } + + if (opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO) { + goto unsupported; + } + + if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) { unsupported: - { - // Rewind to previous instruction and replace with _EXIT_TRACE. - _PyUOpInstruction *curr = &trace[trace_length-1]; - while (curr->opcode != _SET_IP && trace_length > 2) { - trace_length--; - curr = &trace[trace_length-1]; - } - assert(curr->opcode == _SET_IP || trace_length == 2); - if (curr->opcode == _SET_IP) { - int32_t old_target = (int32_t)uop_get_target(curr); - curr++; - trace_length++; - curr->opcode = _EXIT_TRACE; - curr->format = UOP_FORMAT_TARGET; - curr->target = old_target; - } - goto done; - } + { + // Rewind to previous instruction and replace with _EXIT_TRACE. + _PyUOpInstruction *curr = &trace[trace_length-1]; + while (curr->opcode != _SET_IP && trace_length > 2) { + trace_length--; + curr = &trace[trace_length-1]; + } + assert(curr->opcode == _SET_IP || trace_length == 2); + if (curr->opcode == _SET_IP) { + int32_t old_target = (int32_t)uop_get_target(curr); + curr++; + trace_length++; + curr->opcode = _EXIT_TRACE; + curr->format = UOP_FORMAT_TARGET; + curr->target = old_target; + } + goto done; } - - tstate->interp->jit_state.current_frame = frame; + } if (opcode == NOP) { return 1; @@ -721,7 +736,9 @@ _PyJit_translate_single_bytecode_to_trace( { if ((next_instr != tstate->interp->jit_state.close_loop_instr) && (next_instr != tstate->interp->jit_state.insert_exec_instr) && - tstate->interp->jit_state.code_curr_size > 5) { + tstate->interp->jit_state.code_curr_size > 5 && + // These are coroutines, and we want to unroll those usually. + opcode != JUMP_BACKWARD_NO_INTERRUPT) { // We encountered a JUMP_BACKWARD but not to the top of our own loop. // We don't want to continue tracing as we might get stuck in the // inner loop. Instead, end the trace where the executor of the @@ -846,9 +863,9 @@ _PyJit_translate_single_bytecode_to_trace( } if (uop == _BINARY_OP_INPLACE_ADD_UNICODE) { assert(i + 1 == nuops); - _Py_CODEUNIT *next_instr = target_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; - assert(next_instr->op.code == STORE_FAST); - operand = next_instr->op.arg; + _Py_CODEUNIT *next = target_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]; + assert(next->op.code == STORE_FAST); + operand = next->op.arg; } // All other instructions ADD_TO_TRACE(uop, oparg, operand, target); @@ -889,7 +906,7 @@ _PyJit_translate_single_bytecode_to_trace( } void -_PyJit_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *insert_exec_instr, _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit) +_PyJit_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *curr_instr, _Py_CODEUNIT *insert_exec_instr, _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit, int oparg) { PyCodeObject *code = _PyFrame_GetCode(frame); #ifdef Py_DEBUG @@ -914,13 +931,19 @@ _PyJit_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_ tstate->interp->jit_state.close_loop_instr = close_loop_instr; tstate->interp->jit_state.initial_code = (PyCodeObject *)Py_NewRef(code); tstate->interp->jit_state.initial_func = (PyFunctionObject *)Py_NewRef(_PyFrame_GetFunction(frame)); - tstate->interp->jit_state.previous_exit = exit; - _Py_BloomFilter_Init(&tstate->interp->jit_state.dependencies); + tstate->interp->jit_state.prev_exit = exit; tstate->interp->jit_state.initial_stack_depth = curr_stackdepth; tstate->interp->jit_state.initial_chain_depth = chain_depth % MAX_CHAIN_DEPTH; - tstate->interp->jit_state.current_frame = frame; + tstate->interp->jit_state.prev_instr_frame = frame; tstate->interp->jit_state.dependencies_still_valid = true; - tstate->interp->jit_state.last_specialized_instr = NULL; + tstate->interp->jit_state.do_not_specialize = false; + tstate->interp->jit_state.prev_instr_code = (PyCodeObject *)Py_NewRef(_PyFrame_GetCode(frame)); + tstate->interp->jit_state.prev_instr = curr_instr; + tstate->interp->jit_state.prev_instr_frame = frame; + tstate->interp->jit_state.prev_instr_oparg = oparg; + tstate->interp->jit_state.prev_instr_stacklevel = curr_stackdepth; + tstate->interp->jit_state.dynamic_jump_taken = false; + _Py_BloomFilter_Init(&tstate->interp->jit_state.dependencies); } void @@ -928,6 +951,7 @@ _PyJit_FinalizeTracing(PyThreadState *tstate) { Py_CLEAR(tstate->interp->jit_state.initial_code); Py_CLEAR(tstate->interp->jit_state.initial_func); + Py_CLEAR(tstate->interp->jit_state.prev_instr_code); tstate->interp->jit_state.code_curr_size = 2; tstate->interp->jit_state.code_max_size = UOP_MAX_TRACE_LENGTH - 1; } diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 3edefb110834a9..4a66b7d5118064 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -128,7 +128,6 @@ def __init__(self, out: CWriter, labels: dict[str, Label], cannot_escape: bool = "DISPATCH": self.dispatch, "INSTRUCTION_SIZE": self.instruction_size, "stack_pointer": self.stack_pointer, - "RECORD_DYNAMIC_JUMP_TAKEN": self.record_dynamic_jump_taken, } self.out = out self.labels = labels @@ -476,19 +475,6 @@ def instruction_size(self, self.out.emit(f" {uop.instruction_size}u ") return True - def record_dynamic_jump_taken( - self, - tkn: Token, - tkn_iter: TokenIterator, - uop: CodeSection, - storage: Storage, - inst: Instruction | None, - ) -> bool: - next(tkn_iter); - next(tkn_iter); - next(tkn_iter); - return True - def _print_storage(self, reason:str, storage: Storage) -> None: if DEBUG: self.out.start_line() diff --git a/Tools/cases_generator/target_generator.py b/Tools/cases_generator/target_generator.py index 8e5dc10f759a15..209411ca6f50d9 100644 --- a/Tools/cases_generator/target_generator.py +++ b/Tools/cases_generator/target_generator.py @@ -34,7 +34,7 @@ def write_opcode_targets(analysis: Analysis, out: CWriter) -> None: targets = ["&&_unknown_opcode,\n"] * 256 for name, op in analysis.opmap.items(): if op < 256: - targets[op] = f"&&TARGET_TRACING_{name},\n" + targets[op] = f"&&TARGET_RECORD_PREVIOUS_INST,\n" out.emit("#if _Py_TIER2\n") out.emit("static void *opcode_tracing_targets_table[256] = {\n") for target in targets: @@ -58,10 +58,9 @@ def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: out.emit(f"{function_proto(name)};\n") out.emit("\n") - # Emit function prototypes for opcode handlers. + # Emit function prototypes for opcode handlers. for name in sorted(analysis.instructions.keys()): out.emit(f"{function_proto(name)};\n") - out.emit(f"{function_proto('TRACING_' + name)};\n") out.emit("\n") # Emit unknown opcode handler. @@ -85,7 +84,7 @@ def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: # Emit the tracing dispatch table. out.emit("static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = {\n") for name in sorted(analysis.instructions.keys()): - out.emit(f"[{name}] = _TAIL_CALL_TRACING_{name},\n") + out.emit(f"[{name}] = _TAIL_CALL_RECORD_PREVIOUS_INST,\n") named_values = analysis.opmap.values() for rest in range(256): if rest not in named_values: diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index 25a7442a0ef611..a7961f4fbc7cce 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -216,12 +216,35 @@ def get_popped(inst: Instruction, analysis: Analysis) -> str: stack = get_stack_effect(inst) return (-stack.base_offset).to_c() + +def generate_record_previous_inst(out: CWriter, emitter: Emitter, inst: Instruction) -> None: + out.emit("\n") + out.emit(f"TARGET(RECORD_PREVIOUS_INST) {{\n") + out.emit(f"INSTRUCTION_STATS(RECORD_PREVIOUS_INST);\n") + declare_variables(inst, out) + offset = 1 # The instruction itself + stack = Stack() + for part in inst.parts: + # Only emit braces if more than one uop + insert_braces = len([p for p in inst.parts if isinstance(p, Uop)]) > 1 + reachable, offset, stack = write_uop(part, emitter, offset, stack, inst, insert_braces) + out.start_line() + if reachable: # type: ignore[possibly-undefined] + stack.flush(out) + out.emit(f"DISPATCH();\n") + out.start_line() + out.emit("}") + out.emit("\n") + def generate_tier1_cases( analysis: Analysis, out: CWriter, emitter: Emitter ) -> None: out.emit("\n") for name, inst in sorted(analysis.instructions.items()): out.emit("\n") + if name == "RECORD_PREVIOUS_INST": + generate_record_previous_inst(out, emitter, inst) + continue out.emit(f"TARGET({name}) {{\n") popped = get_popped(inst, analysis) # We need to ifdef it because this breaks platforms diff --git a/Tools/cases_generator/tracer_generator.py b/Tools/cases_generator/tracer_generator.py deleted file mode 100644 index 1b7d9a5acea6bc..00000000000000 --- a/Tools/cases_generator/tracer_generator.py +++ /dev/null @@ -1,222 +0,0 @@ -import argparse - -from analyzer import ( - Analysis, - Instruction, - Uop, - Label, - CodeSection, - Part, - analyze_files, - Skip, - Flush, - analysis_error, - StackItem, -) -from generators_common import ( - DEFAULT_INPUT, - ROOT, - write_header, - type_and_null, - Emitter, - TokenIterator, - always_true, - emit_to, - ReplacementFunctionType, -) -from cwriter import CWriter -from typing import TextIO -from lexer import Token -from stack import Local, Stack, StackError, get_stack_effect, Storage -from tier1_generator import get_popped, declare_variables, write_uop - -DEFAULT_OUTPUT = ROOT / "Python/generated_tracer_cases.c.h" - -class TracerEmitter(Emitter): - out: CWriter - labels: dict[str, Label] - _replacers: dict[str, ReplacementFunctionType] - cannot_escape: bool - - def __init__(self, out: CWriter, labels: dict[str, Label], cannot_escape: bool = False): - super().__init__(out, labels, cannot_escape, jump_prefix="TRACING_") - self._replacers = { - **self._replacers, - "DISPATCH": self.dispatch, - "DISPATCH_INLINED": self.dispatch_inlined, - "DISPATCH_SAME_OPARG": self.dispatch_same_oparg, - } - - def dispatch( - self, - tkn: Token, - tkn_iter: TokenIterator, - uop: CodeSection, - storage: Storage, - inst: Instruction | None, - ) -> bool: - if storage.spilled: - raise analysis_error("stack_pointer needs reloading before dispatch", tkn) - storage.stack.flush(self.out) - self.out.start_line() - self.emit("TRACING_DISPATCH") - return False - - def dispatch_inlined( - self, - tkn: Token, - tkn_iter: TokenIterator, - uop: CodeSection, - storage: Storage, - inst: Instruction | None, - ) -> bool: - if storage.spilled: - raise analysis_error("stack_pointer needs reloading before dispatch", tkn) - storage.stack.flush(self.out) - self.out.start_line() - self.emit("TRACING_DISPATCH_INLINED") - return False - - def dispatch_same_oparg( - self, - tkn: Token, - tkn_iter: TokenIterator, - uop: CodeSection, - storage: Storage, - inst: Instruction | None, - ) -> bool: - if storage.spilled: - raise analysis_error("stack_pointer needs reloading before dispatch", tkn) - storage.stack.flush(self.out) - self.out.start_line() - if isinstance(uop, Uop) and "specializing" in uop.annotations: - self.emit("TRACING_SPECIALIZE_DISPATCH_SAME_OPARG") - else: - self.emit(tkn) - emit_to(self.out, tkn_iter, "SEMI") - return False - - def record_dynamic_jump_taken( - self, - tkn: Token, - tkn_iter: TokenIterator, - uop: CodeSection, - storage: Storage, - inst: Instruction | None, - ) -> bool: - self.out.emit(tkn) - emit_to(self.out, tkn_iter, "SEMI") - self.out.emit(";\n") - return True - -def generate_tier1_tracer_cases( - analysis: Analysis, out: CWriter, emitter: Emitter -) -> None: - out.emit("\n") - for name, inst in sorted(analysis.instructions.items()): - out.emit("\n") - out.emit(f"TRACING_TARGET({name}) {{\n") - out.emit(f"assert(IS_JIT_TRACING());\n") - # We need to ifdef it because this breaks platforms - # without computed gotos/tail calling. - out.emit(f"#if _Py_TAIL_CALL_INTERP\n") - out.emit(f"int opcode = {name};\n") - out.emit(f"(void)(opcode);\n") - out.emit(f"#endif\n") - unused_guard = "(void)this_instr;\n" - if inst.properties.needs_prev: - out.emit(f"_Py_CODEUNIT* const prev_instr = frame->instr_ptr;\n") - if not inst.is_target: - out.emit(f"_Py_CODEUNIT* const this_instr = next_instr;\n") - out.emit(unused_guard) - if not inst.properties.no_save_ip: - out.emit(f"frame->instr_ptr = next_instr;\n") - - out.emit(f"next_instr += {inst.size};\n") - out.emit(f"INSTRUCTION_STATS({name});\n") - if inst.is_target: - out.emit(f"PREDICTED_TRACING_{name}:;\n") - out.emit(f"_Py_CODEUNIT* const this_instr = next_instr - {inst.size};\n") - out.emit(unused_guard) - # This is required so that the predicted ops reflect the correct opcode. - out.emit(f"opcode = {name};\n") - out.emit(f"PyCodeObject *old_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable);\n") - out.emit(f"(void)old_code;\n") - out.emit(f"PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj);\n") - out.emit(f"(void)old_func;\n") - out.emit(f"int _jump_taken = false;\n") - out.emit(f"(void)_jump_taken;\n") - out.emit(f"int _old_stack_level = !PyStackRef_IsNull(frame->f_executable) ? STACK_LEVEL() : 0;\n") - out.emit(f"(void)(_old_stack_level);\n") - if inst.family is not None: - out.emit( - f"static_assert({inst.family.size} == {inst.size-1}" - ', "incorrect cache size");\n' - ) - declare_variables(inst, out) - offset = 1 # The instruction itself - stack = Stack() - for part in inst.parts: - # Only emit braces if more than one uop - insert_braces = len([p for p in inst.parts if isinstance(p, Uop)]) > 1 - reachable, offset, stack = write_uop(part, emitter, offset, stack, inst, insert_braces) - out.start_line() - if reachable: # type: ignore[possibly-undefined] - stack.flush(out) - out.emit(f"TRACING_DISPATCH();\n") - out.start_line() - out.emit("}") - out.emit("\n") - - -def generate_tracer_cases( - analysis: Analysis, out: CWriter -) -> None: - out.emit(f"#ifdef _Py_TIER2 /* BEGIN TRACING INSTRUCTIONS */\n") - generate_tier1_tracer_cases(analysis, out, TracerEmitter(out, analysis.labels)) - out.emit(f"#endif /* END TRACING INSTRUCTIONS */\n") - -def generate_tracer( - filenames: list[str], analysis: Analysis, outfile: TextIO, lines: bool -) -> None: - write_header(__file__, filenames, outfile) - out = CWriter(outfile, 2, lines) - out.emit("#define TIER_ONE 1\n") - out.emit("#define TRACING_JIT 1\n") - generate_tracer_cases(analysis, out) - out.emit("#undef TRACING_JIT\n") - out.emit("#undef TIER_ONE\n") - -# For use in unittest -def generate_tracer_from_files( - filenames: list[str], outfilename: str, lines: bool -) -> None: - data = analyze_files(filenames) - with open(outfilename, "w") as outfile: - generate_tracer(filenames, data, outfile, lines) - - -arg_parser = argparse.ArgumentParser( - description="Generate the code for the interpreter switch.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter, -) - -arg_parser.add_argument( - "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT -) - -arg_parser.add_argument( - "-l", "--emit-line-directives", help="Emit #line directives", action="store_true" -) - -arg_parser.add_argument( - "input", nargs=argparse.REMAINDER, help="Instruction definition file(s)" -) - -if __name__ == "__main__": - args = arg_parser.parse_args() - if len(args.input) == 0: - args.input.append(DEFAULT_INPUT) - data = analyze_files(args.input) - with open(args.output, "w") as outfile: - generate_tracer(args.input, data, outfile, args.emit_line_directives) From 676faf8bf465fcb7d9d86db9fbacc3c3927fef43 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 24 Oct 2025 00:20:58 +0100 Subject: [PATCH 090/190] Fix ifdefs --- Python/ceval_macros.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index bfc84e8c66230b..9a006e185bcea2 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -133,7 +133,7 @@ # define LABEL(name) name: #endif -#if _Py_TAIL_CALL_INTERP || USE_COMPUTED_GOTOS +#if (_Py_TAIL_CALL_INTERP || USE_COMPUTED_GOTOS) && _Py_TIER2 # define IS_JIT_TRACING() (DISPATCH_TABLE_VAR == TRACING_DISPATCH_TABLE) // Required to not get stuck in infinite pecialization loops due to specialization failure. # define IS_JIT_TRACING_MAKING_PROGRESS() (IS_JIT_TRACING() && !tstate->interp->jit_state.do_not_specialize) @@ -141,9 +141,13 @@ DISPATCH_TABLE_VAR = TRACING_DISPATCH_TABLE; # define LEAVE_TRACING() \ DISPATCH_TABLE_VAR = DISPATCH_TABLE; +#else +# define IS_JIT_TRACING() (0) +# define IS_JIT_TRACING_MAKING_PROGRESS() (0) +# define ENTER_TRACING() +# define LEAVE_TRACING() #endif - /* PRE_DISPATCH_GOTO() does lltrace if enabled. Normally a no-op */ #ifdef Py_DEBUG #define PRE_DISPATCH_GOTO() if (frame->lltrace >= 5) { \ @@ -324,14 +328,10 @@ GETITEM(PyObject *v, Py_ssize_t i) { /* This takes a uint16_t instead of a _Py_BackoffCounter, * because it is used directly on the cache entry in generated code, * which is always an integral type. */ -#if _Py_TIER2 // Force re-specialization when tracing a side exit to get good side exits. #define ADAPTIVE_COUNTER_TRIGGERS(COUNTER) \ backoff_counter_triggers(forge_backoff_counter((COUNTER))) || IS_JIT_TRACING_MAKING_PROGRESS() -#else -#define ADAPTIVE_COUNTER_TRIGGERS(COUNTER) \ - backoff_counter_triggers(forge_backoff_counter((COUNTER))) -#endif + #define ADVANCE_ADAPTIVE_COUNTER(COUNTER) \ do { \ (COUNTER) = advance_backoff_counter((COUNTER)); \ From cdcce30be410c34e75c1481e22f870edb8378dcf Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 24 Oct 2025 00:26:49 +0100 Subject: [PATCH 091/190] Address review of macros --- ...-10-18-21-50-44.gh-issue-139109.9QQOzN.rst | 2 +- Python/bytecodes.c | 12 +++++----- Python/ceval_macros.h | 24 +++++++++++++------ Python/executor_cases.c.h | 12 +++++----- 4 files changed, 30 insertions(+), 20 deletions(-) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-50-44.gh-issue-139109.9QQOzN.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-50-44.gh-issue-139109.9QQOzN.rst index 01e2efa4df4590..3d6f45b2b44781 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-50-44.gh-issue-139109.9QQOzN.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-50-44.gh-issue-139109.9QQOzN.rst @@ -1 +1 @@ -A new tracing frontend for the JIT compiler has been implemented. Patch by Ken Jin. Design for CPython by Brandt Bucher, Mark Shannon and Ken Jin. +A new tracing frontend for the JIT compiler has been implemented. Patch by Ken Jin. Design for CPython by Mark Shannon, Ken Jin and Brandt Bucher. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 73ee9fdb8fcca3..9c0759af7c897d 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5420,12 +5420,12 @@ dummy_func( } tier2 op(_DEOPT, (--)) { - GOTO_TIER_ONE(_PyFrame_GetBytecode(frame) + CURRENT_TARGET(), 0); + GOTO_TIER_ONE(_PyFrame_GetBytecode(frame) + CURRENT_TARGET()); } tier2 op(_HANDLE_PENDING_AND_DEOPT, (--)) { int err = _Py_HandlePending(tstate); - GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET(), 0); + GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); } tier2 op(_ERROR_POP_N, (target/2 --)) { @@ -5435,7 +5435,7 @@ dummy_func( // gh-140104: The exception handler expects frame->instr_ptr to be pointing to next_instr, not this_instr! frame->instr_ptr = next_instr; SYNC_SP(); - GOTO_TIER_ONE(NULL, 0); + GOTO_TIER_ONE(NULL); } /* Progress is guaranteed if we DEOPT on the eval breaker, because @@ -5457,7 +5457,7 @@ dummy_func( _Py_BackoffCounter temperature = exit->temperature; if (!backoff_counter_triggers(temperature)) { exit->temperature = advance_backoff_counter(temperature); - GOTO_TIER_ONE(target, 0); + GOTO_TIER_ONE(target); } _PyExecutorObject *executor; if (target->op.code == ENTER_EXECUTOR) { @@ -5474,7 +5474,7 @@ dummy_func( // So setting it to anything else is wrong. _PyJit_InitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, target->op.arg); exit->temperature = initial_temperature_backoff_counter(); - GOTO_TIER_ONE(target, 1); + GOTO_TIER_ONE_CONTINUE_TRACING(target); } assert(tstate->jit_exit == exit); exit->executor = executor; @@ -5508,7 +5508,7 @@ dummy_func( TIER2_TO_TIER2(executor); } else { - GOTO_TIER_ONE(target, 0); + GOTO_TIER_ONE(target); } } diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 9a006e185bcea2..27cc8b19f715c3 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -440,13 +440,23 @@ do { \ goto tier2_start; \ } while (0) -#define GOTO_TIER_ONE(TARGET, SHOULD_CONTINUE_TRACING) \ - do \ - { \ - tstate->current_executor = NULL; \ - OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); \ - _PyFrame_SetStackPointer(frame, stack_pointer); \ - return (_Py_CODEUNIT *)(((uintptr_t)(TARGET)) | SHOULD_CONTINUE_TRACING); \ +#define GOTO_TIER_ONE_SETUP \ + tstate->current_executor = NULL; \ + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); \ + _PyFrame_SetStackPointer(frame, stack_pointer); + +#define GOTO_TIER_ONE(TARGET) \ + do \ + { \ + GOTO_TIER_ONE_SETUP \ + return (_Py_CODEUNIT *)(TARGET); \ + } while (0) + +#define GOTO_TIER_ONE_CONTINUE_TRACING(TARGET) \ + do \ + { \ + GOTO_TIER_ONE_SETUP \ + return (_Py_CODEUNIT *)(((uintptr_t)(TARGET))| 1); \ } while (0) #define CURRENT_OPARG() (next_uop[-1].oparg) diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index f839b9579ef38c..6d155cf8179661 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7442,7 +7442,7 @@ } case _DEOPT: { - GOTO_TIER_ONE(_PyFrame_GetBytecode(frame) + CURRENT_TARGET(), 0); + GOTO_TIER_ONE(_PyFrame_GetBytecode(frame) + CURRENT_TARGET()); break; } @@ -7450,7 +7450,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); int err = _Py_HandlePending(tstate); stack_pointer = _PyFrame_GetStackPointer(frame); - GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET(), 0); + GOTO_TIER_ONE(err ? NULL : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); break; } @@ -7461,7 +7461,7 @@ _Py_CODEUNIT *current_instr = _PyFrame_GetBytecode(frame) + target; _Py_CODEUNIT *next_instr = current_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[current_instr->op.code]]; frame->instr_ptr = next_instr; - GOTO_TIER_ONE(NULL, 0); + GOTO_TIER_ONE(NULL); break; } @@ -7489,7 +7489,7 @@ _Py_BackoffCounter temperature = exit->temperature; if (!backoff_counter_triggers(temperature)) { exit->temperature = advance_backoff_counter(temperature); - GOTO_TIER_ONE(target, 0); + GOTO_TIER_ONE(target); } _PyExecutorObject *executor; if (target->op.code == ENTER_EXECUTOR) { @@ -7503,7 +7503,7 @@ int chain_depth = previous_executor->vm_data.chain_depth + 1; _PyJit_InitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, target->op.arg); exit->temperature = initial_temperature_backoff_counter(); - GOTO_TIER_ONE(target, 1); + GOTO_TIER_ONE_CONTINUE_TRACING(target); } assert(tstate->jit_exit == exit); exit->executor = executor; @@ -7544,7 +7544,7 @@ TIER2_TO_TIER2(executor); } else { - GOTO_TIER_ONE(target, 0); + GOTO_TIER_ONE(target); } break; } From 1a3f12993ee7a882cddab1abee4e5ad7640e318b Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 24 Oct 2025 00:49:18 +0100 Subject: [PATCH 092/190] fix a tracing bug, ifdef out code --- Python/bytecodes.c | 4 ++++ Python/ceval_macros.h | 1 - Python/generated_cases.c.h | 4 ++++ Python/optimizer.c | 3 +++ 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 9c0759af7c897d..be98f34ce819e4 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5219,6 +5219,7 @@ dummy_func( } tier1 no_save_ip inst(RECORD_PREVIOUS_INST, (--)) { +#if _Py_TIER2 assert(IS_JIT_TRACING()); int opcode = next_instr->op.code; int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr); @@ -5236,6 +5237,9 @@ dummy_func( tstate->interp->jit_state.prev_instr_oparg = oparg; tstate->interp->jit_state.prev_instr_stacklevel = STACK_LEVEL(); DISPATCH_GOTO_NON_TRACING(); +#else + Py_FatalError("JIT instruction executed in non-jit build."); +#endif } ///////// Tier-2 only opcodes ///////// diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 27cc8b19f715c3..b741707377ae22 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -413,7 +413,6 @@ _PyFrame_SetStackPointer(frame, stack_pointer) #define TIER1_TO_TIER2(EXECUTOR) \ do { \ - LEAVE_TRACING(); \ OPT_STAT_INC(traces_executed); \ next_instr = _Py_jit_entry((EXECUTOR), frame, stack_pointer, tstate); \ frame = tstate->current_frame; \ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 516d42a3972d76..0665f985ff08cc 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -10199,6 +10199,7 @@ TARGET(RECORD_PREVIOUS_INST) { INSTRUCTION_STATS(RECORD_PREVIOUS_INST); + #if _Py_TIER2 assert(IS_JIT_TRACING()); int opcode = next_instr->op.code; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -10224,6 +10225,9 @@ tstate->interp->jit_state.prev_instr_oparg = oparg; tstate->interp->jit_state.prev_instr_stacklevel = STACK_LEVEL(); DISPATCH_GOTO_NON_TRACING(); + #else + Py_FatalError("JIT instruction executed in non-jit build."); + #endif } TARGET(RERAISE) { diff --git a/Python/optimizer.c b/Python/optimizer.c index a3d847b6eab884..c11cd22cf9dfbc 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -884,14 +884,17 @@ _PyJit_translate_single_bytecode_to_trace( ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0, 0); goto done; } + DPRINTF(2, "Trace continuing\n"); tstate->interp->jit_state.code_curr_size = trace_length; tstate->interp->jit_state.code_max_size = max_length; return 1; done: + DPRINTF(2, "Trace done\n"); tstate->interp->jit_state.code_curr_size = trace_length; tstate->interp->jit_state.code_max_size = max_length; return 0; full: + DPRINTF(2, "Trace full\n"); if (!is_terminator(&tstate->interp->jit_state.code_buffer[trace_length-1])) { // Undo the last few instructions. trace_length = tstate->interp->jit_state.code_curr_size; From e8fff006c1e4977c693956a9d74446c8efdd4e10 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 24 Oct 2025 00:52:17 +0100 Subject: [PATCH 093/190] fix JIT builds --- Tools/jit/template.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 471e052c2d8d52..9516d25086231c 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -55,14 +55,6 @@ do { \ __attribute__((musttail)) return jitted(frame, stack_pointer, tstate); \ } while (0) -#undef GOTO_TIER_ONE -#define GOTO_TIER_ONE(TARGET, SHOULD_CONTINUE_TRACING) \ -do { \ - tstate->current_executor = NULL; \ - _PyFrame_SetStackPointer(frame, stack_pointer); \ - return (_Py_CODEUNIT *)(((uintptr_t)(TARGET)) | SHOULD_CONTINUE_TRACING); \ -} while (0) - #undef LOAD_IP #define LOAD_IP(UNUSED) \ do { \ From 7b2a8ca57df72a411250ef4d7a2a34ce0607bf38 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 24 Oct 2025 01:04:12 +0100 Subject: [PATCH 094/190] regen frozenmain --- Programs/test_frozenmain.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index dbeedb7ffe0ce6..1794032e2b1315 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -2,14 +2,14 @@ unsigned char M_test_frozenmain[] = { 227,0,0,0,0,0,0,0,0,0,0,0,0,9,0,0, 0,0,0,0,0,243,184,0,0,0,128,0,94,0,82,1, - 73,0,116,0,94,0,82,1,73,1,116,1,93,2,33,0, + 73,0,117,0,94,0,82,1,73,1,117,1,93,2,33,0, 82,2,52,1,0,0,0,0,0,0,31,0,93,2,33,0, 82,3,93,0,80,6,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,52,2,0,0,0,0,0,0, 31,0,93,1,80,8,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,33,0,52,0,0,0,0,0, 0,0,82,4,44,26,0,0,0,0,0,0,0,0,0,0, - 116,5,82,7,16,0,70,24,0,0,116,6,93,2,33,0, + 117,5,82,7,16,0,70,24,0,0,117,6,93,2,33,0, 82,5,93,6,12,0,82,6,93,5,93,6,44,26,0,0, 0,0,0,0,0,0,0,0,12,0,50,4,52,1,0,0, 0,0,0,0,31,0,75,26,0,0,9,0,30,0,82,1, From abb17571c321dc57c11fa14c6f9282de8b21ba30 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 24 Oct 2025 09:01:07 +0100 Subject: [PATCH 095/190] fix build on non-JIT --- Python/ceval.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Python/ceval.c b/Python/ceval.c index f4bf7b244dc794..91bb38acdb42d1 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -984,6 +984,7 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch) } } +#if _Py_TIER2 // 0 for success, -1 for error. static int bail_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame) @@ -996,6 +997,7 @@ bail_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame) _PyJit_FinalizeTracing(tstate); return err; } +#endif /* _PyEval_EvalFrameDefault is too large to optimize for speed with PGO on MSVC. */ From 0fee4e9b3ff6c18607073947112b9eef4890d939 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 24 Oct 2025 09:06:08 +0100 Subject: [PATCH 096/190] fix pystats jit build --- Tools/jit/template.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 9516d25086231c..85aa6f988755fc 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -55,6 +55,11 @@ do { \ __attribute__((musttail)) return jitted(frame, stack_pointer, tstate); \ } while (0) +#undef GOTO_TIER_ONE_SETUP +#define GOTO_TIER_ONE_SETUP \ + tstate->current_executor = NULL; \ + _PyFrame_SetStackPointer(frame, stack_pointer); + #undef LOAD_IP #define LOAD_IP(UNUSED) \ do { \ From eb970e06fdfd2bc28e2d79fa29e110803090b48d Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 24 Oct 2025 11:27:20 +0100 Subject: [PATCH 097/190] specialization and deopt fixes --- Include/internal/pycore_interp_structs.h | 3 +- Python/bytecodes.c | 15 +- Python/ceval_macros.h | 6 +- Python/generated_cases.c.h | 732 ++++++++++++++++++++- Python/optimizer.c | 20 +- Tools/cases_generator/generators_common.py | 3 + 6 files changed, 767 insertions(+), 12 deletions(-) diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index c2cddfe47d3f73..bc0ae6ea97d654 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -759,14 +759,15 @@ typedef _Py_CODEUNIT *(*_PyJitEntryFuncPtr)(struct _PyExecutorObject *exec, _PyI typedef struct _PyJitTracerState { bool dependencies_still_valid; - bool do_not_specialize; bool dynamic_jump_taken; + bool prev_instr_is_super; int code_max_size; int code_curr_size; int initial_stack_depth; int initial_chain_depth; int prev_instr_oparg; int prev_instr_stacklevel; + int specialize_counter; _PyUOpInstruction *code_buffer; _Py_CODEUNIT *insert_exec_instr; _Py_CODEUNIT *close_loop_instr; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index be98f34ce819e4..e872dabda87416 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5229,10 +5229,21 @@ dummy_func( ERROR_IF(err < 0); DISPATCH_GOTO_NON_TRACING(); } - tstate->interp->jit_state.do_not_specialize = false; + // Super instructions. Instruction deopted, There's a mismatch in what the stack expects + // in the optimizer. So we have to reflect in the trace correctly. + if ((tstate->interp->jit_state.prev_instr->op.code == CALL_LIST_APPEND && + opcode == POP_TOP) || + (tstate->interp->jit_state.prev_instr->op.code == BINARY_OP_INPLACE_ADD_UNICODE && + opcode == STORE_FAST)) { + tstate->interp->jit_state.prev_instr_is_super = true; + } + else { + tstate->interp->jit_state.prev_instr = next_instr; + } + tstate->interp->jit_state.specialize_counter = 0; PyCodeObject *prev_code = (PyCodeObject *)Py_NewRef(_PyFrame_GetCode(frame)); Py_SETREF(tstate->interp->jit_state.prev_instr_code, prev_code); - tstate->interp->jit_state.prev_instr = next_instr; + tstate->interp->jit_state.prev_instr_frame = frame; tstate->interp->jit_state.prev_instr_oparg = oparg; tstate->interp->jit_state.prev_instr_stacklevel = STACK_LEVEL(); diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index b741707377ae22..abef480b902a5c 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -135,8 +135,8 @@ #if (_Py_TAIL_CALL_INTERP || USE_COMPUTED_GOTOS) && _Py_TIER2 # define IS_JIT_TRACING() (DISPATCH_TABLE_VAR == TRACING_DISPATCH_TABLE) -// Required to not get stuck in infinite pecialization loops due to specialization failure. -# define IS_JIT_TRACING_MAKING_PROGRESS() (IS_JIT_TRACING() && !tstate->interp->jit_state.do_not_specialize) +// Required to not get stuck in infinite specialization loops due to specialization failure. +# define IS_JIT_TRACING_MAKING_PROGRESS() (IS_JIT_TRACING() && tstate->interp->jit_state.specialize_counter <= 2) # define ENTER_TRACING() \ DISPATCH_TABLE_VAR = TRACING_DISPATCH_TABLE; # define LEAVE_TRACING() \ @@ -195,7 +195,7 @@ do { \ #if _Py_TIER2 #define DISPATCH_SAME_OPARG() \ { \ - tstate->interp->jit_state.do_not_specialize = true; \ + tstate->interp->jit_state.specialize_counter++; \ opcode = next_instr->op.code; \ PRE_DISPATCH_GOTO(); \ DISPATCH_GOTO_NON_TRACING(); \ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 0665f985ff08cc..7b759811b3928d 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -103,6 +103,9 @@ if (!PyFloat_CheckExact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -113,6 +116,9 @@ if (!PyFloat_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -161,6 +167,9 @@ if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -171,6 +180,9 @@ if (!_PyLong_CheckExactAndCompact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -188,6 +200,9 @@ if (PyStackRef_IsNull(res)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); @@ -222,6 +237,9 @@ if (!PyUnicode_CheckExact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -232,6 +250,9 @@ if (!PyUnicode_CheckExact(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -290,6 +311,9 @@ if (!res) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -345,6 +369,9 @@ if (!PyUnicode_CheckExact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -355,6 +382,9 @@ if (!PyUnicode_CheckExact(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -378,6 +408,9 @@ if (PyStackRef_AsPyObjectBorrow(*target_local) != left_o) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } STAT_INC(BINARY_OP, hit); @@ -428,6 +461,9 @@ if (!PyFloat_CheckExact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -438,6 +474,9 @@ if (!PyFloat_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -486,6 +525,9 @@ if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -496,6 +538,9 @@ if (!_PyLong_CheckExactAndCompact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -513,6 +558,9 @@ if (PyStackRef_IsNull(res)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); @@ -546,6 +594,9 @@ if (!PyDict_CheckExact(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -611,6 +662,9 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -621,6 +675,9 @@ if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; @@ -628,6 +685,9 @@ if (getitem_o == NULL) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } assert(PyFunction_Check(getitem_o)); @@ -635,6 +695,9 @@ if (((PyFunctionObject *)getitem_o)->func_version != cached_version) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); @@ -642,6 +705,9 @@ if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } getitem = PyStackRef_FromPyObjectNew(getitem_o); @@ -697,6 +763,9 @@ if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -707,6 +776,9 @@ if (!PyList_CheckExact(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -722,6 +794,9 @@ if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; @@ -732,6 +807,9 @@ if (res_o == NULL) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } STAT_INC(BINARY_OP, hit); @@ -740,6 +818,9 @@ if (index >= PyList_GET_SIZE(list)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } STAT_INC(BINARY_OP, hit); @@ -787,6 +868,9 @@ if (!PySlice_Check(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -797,6 +881,9 @@ if (!PyList_CheckExact(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -859,6 +946,9 @@ if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -869,6 +959,9 @@ if (!PyUnicode_CheckExact(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -884,18 +977,27 @@ if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; if (PyUnicode_GET_LENGTH(str) <= index) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } Py_UCS4 c = PyUnicode_READ_CHAR(str, index); if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } STAT_INC(BINARY_OP, hit); @@ -937,6 +1039,9 @@ if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -947,6 +1052,9 @@ if (!PyTuple_CheckExact(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -962,12 +1070,18 @@ if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; if (index >= PyTuple_GET_SIZE(tuple)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } STAT_INC(BINARY_OP, hit); @@ -1009,6 +1123,9 @@ if (!PyFloat_CheckExact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -1019,6 +1136,9 @@ if (!PyFloat_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -1067,6 +1187,9 @@ if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -1077,6 +1200,9 @@ if (!_PyLong_CheckExactAndCompact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -1094,6 +1220,9 @@ if (PyStackRef_IsNull(res)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(BINARY_OP); } PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); @@ -1709,6 +1838,9 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -1721,17 +1853,26 @@ if (!PyStackRef_IsNull(self_or_null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } if (!PyType_Check(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } PyTypeObject *tp = (PyTypeObject *)callable_o; if (FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } assert(tp->tp_new == PyBaseObject_Type.tp_new); @@ -1743,6 +1884,9 @@ if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); @@ -1828,6 +1972,9 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -1838,11 +1985,17 @@ if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -1869,12 +2022,18 @@ if (!PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -1887,6 +2046,9 @@ if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -1898,6 +2060,9 @@ if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -1906,6 +2071,9 @@ if (tstate->py_recursion_remaining <= 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -1971,6 +2139,9 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -1983,22 +2154,34 @@ if (Py_TYPE(callable_o) != &PyMethod_Type) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } PyObject *func = ((PyMethodObject *)callable_o)->im_func; if (!PyFunction_Check(func)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } if (((PyFunctionObject *)func)->func_version != func_version) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -2024,6 +2207,9 @@ if (tstate->py_recursion_remaining <= 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -2103,6 +2289,9 @@ if (!PyType_Check(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } PyTypeObject *tp = (PyTypeObject *)callable_o; @@ -2115,6 +2304,9 @@ if (tp->tp_vectorcall == NULL) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); @@ -2214,11 +2406,17 @@ if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } if (PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); @@ -2323,11 +2521,17 @@ if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } if (PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); @@ -2431,21 +2635,33 @@ if (total_args != 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } if (_Py_ReachedRecursionLimit(tstate)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); @@ -2744,6 +2960,9 @@ if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -2755,6 +2974,9 @@ if (callable_o != interp->callable_cache.isinstance) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -3001,6 +3223,9 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL_KW); } } @@ -3013,22 +3238,34 @@ if (Py_TYPE(callable_o) != &PyMethod_Type) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL_KW); } PyObject *func = ((PyMethodObject *)callable_o)->im_func; if (!PyFunction_Check(func)) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL_KW); } if (((PyFunctionObject *)func)->func_version != func_version) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL_KW); } if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL_KW); } } @@ -3134,11 +3371,17 @@ if (PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL_KW); } if (Py_TYPE(callable_o) == &PyMethod_Type) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL_KW); } } @@ -3257,6 +3500,9 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL_KW); } } @@ -3268,12 +3514,18 @@ if (!PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL_KW); } PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL_KW); } } @@ -3282,6 +3534,9 @@ if (tstate->py_recursion_remaining <= 1) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL_KW); } } @@ -3368,6 +3623,9 @@ if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -3379,6 +3637,9 @@ if (callable_o != interp->callable_cache.len) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -3442,6 +3703,9 @@ if (callable_o != interp->callable_cache.list_append) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -3452,6 +3716,9 @@ if (o == NULL) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -3461,6 +3728,9 @@ if (!PyList_CheckExact(o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -3473,11 +3743,17 @@ if (!PyList_CheckExact(self_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } if (!LOCK_OBJECT(self_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); @@ -3537,18 +3813,27 @@ if (total_args == 0) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } PyMethodDef *meth = method->d_method; if (meth->ml_flags != METH_FASTCALL) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); @@ -3556,6 +3841,9 @@ if (!Py_IS_TYPE(self, method->d_common.d_type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); @@ -3658,18 +3946,27 @@ if (total_args == 0) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } PyMethodDef *meth = method->d_method; if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } PyTypeObject *d_type = method->d_common.d_type; @@ -3678,6 +3975,9 @@ if (!Py_IS_TYPE(self, d_type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); @@ -3781,12 +4081,18 @@ if (total_args != 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } PyMethodDef *meth = method->d_method; @@ -3795,16 +4101,25 @@ if (!Py_IS_TYPE(self, method->d_common.d_type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } if (meth->ml_flags != METH_NOARGS) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } if (_Py_ReachedRecursionLimit(tstate)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); @@ -3875,22 +4190,34 @@ if (total_args != 2) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } PyMethodDef *meth = method->d_method; if (meth->ml_flags != METH_O) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } if (_Py_ReachedRecursionLimit(tstate)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } _PyStackRef arg_stackref = arguments[1]; @@ -3899,6 +4226,9 @@ method->d_common.d_type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); @@ -3973,11 +4303,17 @@ if (PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } if (Py_TYPE(callable_o) == &PyMethod_Type) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -4084,6 +4420,9 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -4095,12 +4434,18 @@ if (!PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -4114,6 +4459,9 @@ if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -4125,6 +4473,9 @@ if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -4133,6 +4484,9 @@ if (tstate->py_recursion_remaining <= 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -4197,6 +4551,9 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -4208,12 +4565,18 @@ if (!PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -4222,6 +4585,9 @@ if (tstate->py_recursion_remaining <= 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -4299,6 +4665,9 @@ if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -4309,6 +4678,9 @@ if (callable_o != (PyObject *)&PyUnicode_Type) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -4371,6 +4743,9 @@ if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -4381,6 +4756,9 @@ if (callable_o != (PyObject *)&PyTuple_Type) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -4443,6 +4821,9 @@ if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -4453,6 +4834,9 @@ if (callable_o != (PyObject *)&PyType_Type) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CALL); } } @@ -4737,6 +5121,9 @@ if (!PyFloat_CheckExact(value_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(COMPARE_OP); } } @@ -4747,6 +5134,9 @@ if (!PyFloat_CheckExact(left_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(COMPARE_OP); } } @@ -4792,6 +5182,9 @@ if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(COMPARE_OP); } } @@ -4802,6 +5195,9 @@ if (!_PyLong_CheckExactAndCompact(left_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(COMPARE_OP); } } @@ -4852,6 +5248,9 @@ if (!PyUnicode_CheckExact(value_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(COMPARE_OP); } } @@ -4862,6 +5261,9 @@ if (!PyUnicode_CheckExact(o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(COMPARE_OP); } } @@ -4970,6 +5372,9 @@ if (!PyDict_CheckExact(o)) { UPDATE_MISS_STATS(CONTAINS_OP); assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CONTAINS_OP); } } @@ -5028,6 +5433,9 @@ if (!PyAnySet_CheckExact(o)) { UPDATE_MISS_STATS(CONTAINS_OP); assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(CONTAINS_OP); } } @@ -5686,6 +6094,9 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(FOR_ITER); } } @@ -5696,18 +6107,27 @@ if (Py_TYPE(gen) != &PyGen_Type) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(FOR_ITER); } #ifdef Py_GIL_DISABLED if (!_PyObject_IsUniquelyReferenced((PyObject *)gen)) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(FOR_ITER); } #endif if (gen->gi_frame_state >= FRAME_EXECUTING) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(FOR_ITER); } STAT_INC(FOR_ITER, hit); @@ -5760,6 +6180,9 @@ if (Py_TYPE(iter_o) != &PyList_Type) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(FOR_ITER); } assert(PyStackRef_IsTaggedInt(null_or_index)); @@ -5767,6 +6190,9 @@ if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(FOR_ITER); } #endif @@ -5802,6 +6228,9 @@ if (result < 0) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(FOR_ITER); } if (result == 0) { @@ -5843,12 +6272,18 @@ if (Py_TYPE(r) != &PyRangeIter_Type) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(FOR_ITER); } #ifdef Py_GIL_DISABLED if (!_PyObject_IsUniquelyReferenced((PyObject *)r)) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(FOR_ITER); } #endif @@ -5913,6 +6348,9 @@ if (Py_TYPE(iter_o) != &PyTuple_Type) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(FOR_ITER); } assert(PyStackRef_IsTaggedInt(null_or_index)); @@ -7914,12 +8352,18 @@ if (!PyType_Check(owner_o)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -7972,12 +8416,18 @@ if (!PyType_Check(owner_o)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -7989,6 +8439,9 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8039,6 +8492,9 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } PyTypeObject *cls = Py_TYPE(owner_o); @@ -8046,6 +8502,9 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); @@ -8054,6 +8513,9 @@ if (f->func_version != func_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } PyCodeObject *code = (PyCodeObject *)f->func_code; @@ -8061,6 +8523,9 @@ if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } STAT_INC(LOAD_ATTR, hit); @@ -8099,6 +8564,9 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8110,6 +8578,9 @@ if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8122,6 +8593,9 @@ if (attr_o == NULL) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } #ifdef Py_GIL_DISABLED @@ -8130,6 +8604,9 @@ if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8179,6 +8656,9 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8190,6 +8670,9 @@ if (dict != NULL) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8235,6 +8718,9 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8281,6 +8767,9 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8292,6 +8781,9 @@ if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8304,6 +8796,9 @@ if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8348,6 +8843,9 @@ if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; @@ -8356,6 +8854,9 @@ if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } assert(keys->dk_kind == DICT_KEYS_UNICODE); @@ -8365,6 +8866,9 @@ if (attr_o == NULL) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } #ifdef Py_GIL_DISABLED @@ -8373,6 +8877,9 @@ if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8421,6 +8928,9 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8468,6 +8978,9 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8479,6 +8992,9 @@ if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8491,6 +9007,9 @@ if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8532,6 +9051,9 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8544,6 +9066,9 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8558,21 +9083,33 @@ if ((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } if (code->co_kwonlyargcount) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } if (code->co_argcount != 1) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } STAT_INC(LOAD_ATTR, hit); @@ -8631,6 +9168,9 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8643,6 +9183,9 @@ if (attr_o == NULL) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } #ifdef Py_GIL_DISABLED @@ -8650,6 +9193,9 @@ if (!increfed) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } #else @@ -8700,6 +9246,9 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8712,6 +9261,9 @@ if (dict == NULL) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } PyDictKeysObject *dk = FT_ATOMIC_LOAD_PTR(dict->ma_keys); @@ -8720,6 +9272,9 @@ if (!_Py_IsOwnedByCurrentThread((PyObject *)dict) && !_PyObject_GC_IS_SHARED(dict)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } #endif @@ -8728,6 +9283,9 @@ if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8736,6 +9294,9 @@ if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8744,6 +9305,9 @@ if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8752,6 +9316,9 @@ if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8762,6 +9329,9 @@ if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -9194,12 +9764,18 @@ if (!PyDict_CheckExact(dict)) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_GLOBAL); } PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_GLOBAL); } assert(DK_IS_UNICODE(keys)); @@ -9212,12 +9788,18 @@ if (!PyDict_CheckExact(dict)) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_GLOBAL); } PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_GLOBAL); } assert(DK_IS_UNICODE(keys)); @@ -9226,6 +9808,9 @@ if (res_o == NULL) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_GLOBAL); } #if Py_GIL_DISABLED @@ -9233,6 +9818,9 @@ if (!increfed) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_GLOBAL); } #else @@ -9278,12 +9866,18 @@ if (!PyDict_CheckExact(dict)) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_GLOBAL); } PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_GLOBAL); } assert(DK_IS_UNICODE(keys)); @@ -9293,6 +9887,9 @@ if (res_o == NULL) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_GLOBAL); } #if Py_GIL_DISABLED @@ -9300,6 +9897,9 @@ if (!increfed) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_GLOBAL); } #else @@ -9594,11 +10194,17 @@ if (global_super != (PyObject *)&PySuper_Type) { UPDATE_MISS_STATS(LOAD_SUPER_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); } if (!PyType_Check(class)) { UPDATE_MISS_STATS(LOAD_SUPER_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); } STAT_INC(LOAD_SUPER_ATTR, hit); @@ -9657,11 +10263,17 @@ if (global_super != (PyObject *)&PySuper_Type) { UPDATE_MISS_STATS(LOAD_SUPER_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); } if (!PyType_Check(class)) { UPDATE_MISS_STATS(LOAD_SUPER_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); } STAT_INC(LOAD_SUPER_ATTR, hit); @@ -10215,12 +10827,20 @@ } DISPATCH_GOTO_NON_TRACING(); } - tstate->interp->jit_state.do_not_specialize = false; + if ((tstate->interp->jit_state.prev_instr->op.code == CALL_LIST_APPEND && + opcode == POP_TOP) || + (tstate->interp->jit_state.prev_instr->op.code == BINARY_OP_INPLACE_ADD_UNICODE && + opcode == STORE_FAST)) { + tstate->interp->jit_state.prev_instr_is_super = true; + } + else { + tstate->interp->jit_state.prev_instr = next_instr; + } + tstate->interp->jit_state.specialize_counter = 0; PyCodeObject *prev_code = (PyCodeObject *)Py_NewRef(_PyFrame_GetCode(frame)); _PyFrame_SetStackPointer(frame, stack_pointer); Py_SETREF(tstate->interp->jit_state.prev_instr_code, prev_code); stack_pointer = _PyFrame_GetStackPointer(frame); - tstate->interp->jit_state.prev_instr = next_instr; tstate->interp->jit_state.prev_instr_frame = frame; tstate->interp->jit_state.prev_instr_oparg = oparg; tstate->interp->jit_state.prev_instr_stacklevel = STACK_LEVEL(); @@ -10362,6 +10982,9 @@ if (_Py_emscripten_signal_clock == 0) { UPDATE_MISS_STATS(RESUME); assert(_PyOpcode_Deopt[opcode] == (RESUME)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(RESUME); } _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; @@ -10372,6 +10995,9 @@ if (eval_breaker != version) { UPDATE_MISS_STATS(RESUME); assert(_PyOpcode_Deopt[opcode] == (RESUME)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(RESUME); } #ifdef Py_GIL_DISABLED @@ -10379,6 +11005,9 @@ ((_PyThreadStateImpl *)tstate)->tlbc_index) { UPDATE_MISS_STATS(RESUME); assert(_PyOpcode_Deopt[opcode] == (RESUME)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(RESUME); } #endif @@ -10592,6 +11221,9 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(SEND); assert(_PyOpcode_Deopt[opcode] == (SEND)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(SEND); } } @@ -10603,11 +11235,17 @@ if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { UPDATE_MISS_STATS(SEND); assert(_PyOpcode_Deopt[opcode] == (SEND)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(SEND); } if (gen->gi_frame_state >= FRAME_EXECUTING) { UPDATE_MISS_STATS(SEND); assert(_PyOpcode_Deopt[opcode] == (SEND)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(SEND); } STAT_INC(SEND, hit); @@ -10845,6 +11483,9 @@ if (!LOCK_OBJECT(owner_o)) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(STORE_ATTR); } PyTypeObject *tp = Py_TYPE(owner_o); @@ -10853,6 +11494,9 @@ if (true) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(STORE_ATTR); } } @@ -10868,6 +11512,9 @@ if (true) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(STORE_ATTR); } } @@ -10921,6 +11568,9 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(STORE_ATTR); } } @@ -10932,6 +11582,9 @@ if (!LOCK_OBJECT(owner_o)) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(STORE_ATTR); } char *addr = (char *)owner_o + index; @@ -10972,6 +11625,9 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(STORE_ATTR); } } @@ -10985,11 +11641,17 @@ if (dict == NULL) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(STORE_ATTR); } if (!LOCK_OBJECT(dict)) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(STORE_ATTR); } assert(PyDict_CheckExact((PyObject *)dict)); @@ -11000,6 +11662,9 @@ if (true) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(STORE_ATTR); } } @@ -11009,6 +11674,9 @@ if (true) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(STORE_ATTR); } } @@ -11018,6 +11686,9 @@ if (true) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(STORE_ATTR); } } @@ -11342,6 +12013,9 @@ if (!PyDict_CheckExact(o)) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(STORE_SUBSCR); } } @@ -11393,6 +12067,9 @@ if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(STORE_SUBSCR); } } @@ -11403,6 +12080,9 @@ if (!PyList_CheckExact(o)) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(STORE_SUBSCR); } } @@ -11419,12 +12099,18 @@ if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(STORE_SUBSCR); } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; if (!LOCK_OBJECT(list)) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(STORE_SUBSCR); } if (index >= PyList_GET_SIZE(list)) { @@ -11432,6 +12118,9 @@ if (true) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(STORE_SUBSCR); } } @@ -11548,6 +12237,9 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(TO_BOOL); } } @@ -11585,6 +12277,9 @@ if (!PyStackRef_BoolCheck(value)) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(TO_BOOL); } STAT_INC(TO_BOOL, hit); @@ -11611,6 +12306,9 @@ if (!PyLong_CheckExact(value_o)) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(TO_BOOL); } STAT_INC(TO_BOOL, hit); @@ -11652,6 +12350,9 @@ if (!PyList_CheckExact(o)) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(TO_BOOL); } } @@ -11693,6 +12394,9 @@ if (!PyStackRef_IsNone(value)) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(TO_BOOL); } STAT_INC(TO_BOOL, hit); @@ -11721,6 +12425,9 @@ if (!PyUnicode_CheckExact(value_o)) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(TO_BOOL); } } @@ -11924,6 +12631,9 @@ if (!PyList_CheckExact(o)) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(UNPACK_SEQUENCE); } } @@ -11937,6 +12647,9 @@ if (!LOCK_OBJECT(seq_o)) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(UNPACK_SEQUENCE); } if (PyList_GET_SIZE(seq_o) != oparg) { @@ -11944,6 +12657,9 @@ if (true) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(UNPACK_SEQUENCE); } } @@ -11983,6 +12699,9 @@ if (!PyTuple_CheckExact(o)) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(UNPACK_SEQUENCE); } } @@ -11996,6 +12715,9 @@ if (PyTuple_GET_SIZE(seq_o) != oparg) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(UNPACK_SEQUENCE); } STAT_INC(UNPACK_SEQUENCE, hit); @@ -12034,6 +12756,9 @@ if (!PyTuple_CheckExact(o)) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(UNPACK_SEQUENCE); } } @@ -12047,6 +12772,9 @@ if (PyTuple_GET_SIZE(seq_o) != 2) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(UNPACK_SEQUENCE); } STAT_INC(UNPACK_SEQUENCE, hit); diff --git a/Python/optimizer.c b/Python/optimizer.c index c11cd22cf9dfbc..31c9995c4ed6b4 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -593,14 +593,20 @@ _PyJit_translate_single_bytecode_to_trace( // We must point to the first EXTENDED_ARG when deopting. int oparg = tstate->interp->jit_state.prev_instr_oparg; int opcode = this_instr->op.code; + // Failed specialization twice in a row. Deopt! + if (tstate->interp->jit_state.specialize_counter >= 3) { + opcode = _PyOpcode_Deopt[opcode]; + } int rewind_oparg = oparg; while (rewind_oparg > 255) { rewind_oparg >>= 8; target--; } + int old_stack_level = tstate->interp->jit_state.prev_instr_stacklevel; + bool needs_guard_ip = _PyOpcode_NeedsGuardIp[opcode]; - DPRINTF(2, "%p %d: %s(%d) %d\n", old_code, target, _PyOpcode_OpName[opcode], oparg, needs_guard_ip); + DPRINTF(2, "%p %d: %s(%d) %d %d\n", old_code, target, _PyOpcode_OpName[opcode], oparg, needs_guard_ip, old_stack_level); #ifdef Py_DEBUG if (oparg > 255) { @@ -608,11 +614,16 @@ _PyJit_translate_single_bytecode_to_trace( } #endif + // Skip over super instructions. + if (tstate->interp->jit_state.prev_instr_is_super) { + tstate->interp->jit_state.prev_instr_is_super = false; + return 1; + } + if (!tstate->interp->jit_state.dependencies_still_valid) { goto done; } - int old_stack_level = tstate->interp->jit_state.prev_instr_oparg; // Strange control-flow, unsupported opcode, etc. if (tstate->interp->jit_state.dynamic_jump_taken) { goto unsupported; @@ -939,13 +950,14 @@ _PyJit_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_ tstate->interp->jit_state.initial_chain_depth = chain_depth % MAX_CHAIN_DEPTH; tstate->interp->jit_state.prev_instr_frame = frame; tstate->interp->jit_state.dependencies_still_valid = true; - tstate->interp->jit_state.do_not_specialize = false; + tstate->interp->jit_state.specialize_counter = 0; tstate->interp->jit_state.prev_instr_code = (PyCodeObject *)Py_NewRef(_PyFrame_GetCode(frame)); tstate->interp->jit_state.prev_instr = curr_instr; tstate->interp->jit_state.prev_instr_frame = frame; tstate->interp->jit_state.prev_instr_oparg = oparg; tstate->interp->jit_state.prev_instr_stacklevel = curr_stackdepth; tstate->interp->jit_state.dynamic_jump_taken = false; + tstate->interp->jit_state.prev_instr_is_super = false; _Py_BloomFilter_Init(&tstate->interp->jit_state.dependencies); } @@ -1740,7 +1752,7 @@ executor_to_gv(_PyExecutorObject *executor, FILE *out) #ifdef Py_STATS fprintf(out, " %s -- %" PRIu64 "\n", i, opname, inst->execution_count); #else - fprintf(out, " %s\n", i, opname); + fprintf(out, " %s op0=%" PRIu64 "\n", i, opname, inst->operand0); #endif if (inst->opcode == _EXIT_TRACE || inst->opcode == _JUMP_TO_TOP) { break; diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 4a66b7d5118064..804bd9773b164a 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -169,6 +169,9 @@ def deopt_if( family_name = inst.family.name self.emit(f"UPDATE_MISS_STATS({family_name});\n") self.emit(f"assert(_PyOpcode_Deopt[opcode] == ({family_name}));\n") + self.emit(f"#if _Py_TIER2\n") + self.emit(f"tstate->interp->jit_state.specialize_counter++;\n") + self.emit(f"#endif\n") self.emit(f"JUMP_TO_PREDICTED({self.jump_prefix}{family_name});\n") self.emit("}\n") return not always_true(first_tkn) From abecfd6f2729a39ddd070babc7a88d44a2f4d71a Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 24 Oct 2025 11:36:32 +0100 Subject: [PATCH 098/190] fix test --- Lib/test/test_generated_cases.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 09ce329bdcd14d..1c04881bca5730 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -329,6 +329,9 @@ def test_predictions(self): if (xxx) { UPDATE_MISS_STATS(OP1); assert(_PyOpcode_Deopt[opcode] == (OP1)); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif JUMP_TO_PREDICTED(OP1); } res = Py_None; From 8aeabd58430c7599ae74053fe263e82398766c1b Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 24 Oct 2025 14:36:38 +0100 Subject: [PATCH 099/190] disable tracing on FT --- Python/bytecodes.c | 2 +- Python/generated_cases.c.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index e872dabda87416..31a433721fe416 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2955,7 +2955,7 @@ dummy_func( }; tier1 op(_SPECIALIZE_JUMP_BACKWARD, (--)) { - #if ENABLE_SPECIALIZATION_FT + #if ENABLE_SPECIALIZATION if (this_instr->op.code == JUMP_BACKWARD) { uint8_t desired = tstate->interp->jit ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, desired); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 7b759811b3928d..8169a36a9b9562 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -8041,7 +8041,7 @@ /* Skip 1 cache entry */ // _SPECIALIZE_JUMP_BACKWARD { - #if ENABLE_SPECIALIZATION_FT + #if ENABLE_SPECIALIZATION if (this_instr->op.code == JUMP_BACKWARD) { uint8_t desired = tstate->interp->jit ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, desired); From 6bd1541a8ad54d08a8a49837b300cae58904179e Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 24 Oct 2025 16:40:00 +0100 Subject: [PATCH 100/190] Fix FT --- Python/bytecodes.c | 10 +++++----- Python/ceval_macros.h | 6 +++++- Python/executor_cases.c.h | 8 ++++---- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 31a433721fe416..fc12dc787bce58 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5229,7 +5229,7 @@ dummy_func( ERROR_IF(err < 0); DISPATCH_GOTO_NON_TRACING(); } - // Super instructions. Instruction deopted, There's a mismatch in what the stack expects + // Super instructions. Instruction deopted. There's a mismatch in what the stack expects // in the optimizer. So we have to reflect in the trace correctly. if ((tstate->interp->jit_state.prev_instr->op.code == CALL_LIST_APPEND && opcode == POP_TOP) || @@ -5470,10 +5470,6 @@ dummy_func( assert(exit != NULL); _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; _Py_BackoffCounter temperature = exit->temperature; - if (!backoff_counter_triggers(temperature)) { - exit->temperature = advance_backoff_counter(temperature); - GOTO_TIER_ONE(target); - } _PyExecutorObject *executor; if (target->op.code == ENTER_EXECUTOR) { PyCodeObject *code = _PyFrame_GetCode(frame); @@ -5481,6 +5477,10 @@ dummy_func( Py_INCREF(executor); } else { + if (!backoff_counter_triggers(temperature)) { + exit->temperature = advance_backoff_counter(temperature); + GOTO_TIER_ONE(target); + } _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); assert(tstate->current_executor == (PyObject *)previous_executor); int chain_depth = previous_executor->vm_data.chain_depth + 1; diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index abef480b902a5c..f85a7f9e271a50 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -355,7 +355,11 @@ GETITEM(PyObject *v, Py_ssize_t i) { #define RECORD_BRANCH_TAKEN(bitset, flag) #endif -#define RECORD_DYNAMIC_JUMP_TAKEN() tstate->interp->jit_state.dynamic_jump_taken = true; +#if _Py_TIER2 +# define RECORD_DYNAMIC_JUMP_TAKEN() tstate->interp->jit_state.dynamic_jump_taken = true; +#else +# define RECORD_DYNAMIC_JUMP_TAKEN() +#endif #define UNBOUNDLOCAL_ERROR_MSG \ "cannot access local variable '%s' where it is not associated with a value" diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 6d155cf8179661..2fa109b049d1a9 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7487,10 +7487,6 @@ assert(exit != NULL); _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; _Py_BackoffCounter temperature = exit->temperature; - if (!backoff_counter_triggers(temperature)) { - exit->temperature = advance_backoff_counter(temperature); - GOTO_TIER_ONE(target); - } _PyExecutorObject *executor; if (target->op.code == ENTER_EXECUTOR) { PyCodeObject *code = _PyFrame_GetCode(frame); @@ -7498,6 +7494,10 @@ Py_INCREF(executor); } else { + if (!backoff_counter_triggers(temperature)) { + exit->temperature = advance_backoff_counter(temperature); + GOTO_TIER_ONE(target); + } _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); assert(tstate->current_executor == (PyObject *)previous_executor); int chain_depth = previous_executor->vm_data.chain_depth + 1; From a53ca1de4df5399dec8dc1c54d0d5f266f2b599c Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 24 Oct 2025 19:07:37 +0100 Subject: [PATCH 101/190] Emit RECORD_DYNAMIC_JUMP_TAKEN automatically --- ...-10-18-21-50-44.gh-issue-139109.9QQOzN.rst | 2 +- Python/bytecodes.c | 9 ---- Python/generated_cases.c.h | 2 + Tools/cases_generator/analyzer.py | 44 ++++++++++++++++++- Tools/cases_generator/generators_common.py | 25 +++++++++++ 5 files changed, 71 insertions(+), 11 deletions(-) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-50-44.gh-issue-139109.9QQOzN.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-50-44.gh-issue-139109.9QQOzN.rst index 3d6f45b2b44781..40b9d19ee42968 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-50-44.gh-issue-139109.9QQOzN.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-18-21-50-44.gh-issue-139109.9QQOzN.rst @@ -1 +1 @@ -A new tracing frontend for the JIT compiler has been implemented. Patch by Ken Jin. Design for CPython by Mark Shannon, Ken Jin and Brandt Bucher. +A new tracing frontend for the JIT compiler has been implemented. Patch by Ken Jin. Design for CPython by Ken Jin, Mark Shannon and Brandt Bucher. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index fc12dc787bce58..ee9d41bd5194cf 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -45,10 +45,6 @@ #define USE_COMPUTED_GOTOS 0 #include "ceval_macros.h" -#include "ceval_macros.h" -#include "ceval_macros.h" -#include "../Include/internal/pycore_code.h" -#include "../Include/internal/pycore_stackref.h" /* Flow control macros */ @@ -1380,7 +1376,6 @@ dummy_func( if (err == 0) { assert(retval_o != NULL); JUMPBY(oparg); - RECORD_DYNAMIC_JUMP_TAKEN(); } else { PyStackRef_CLOSE(v); @@ -3235,7 +3230,6 @@ dummy_func( } // Jump forward by oparg and skip the following END_FOR JUMPBY(oparg + 1); - RECORD_DYNAMIC_JUMP_TAKEN(); DISPATCH(); } next = item; @@ -3297,7 +3291,6 @@ dummy_func( null_or_index = PyStackRef_TagInt(-1); /* Jump forward oparg, then skip following END_FOR instruction */ JUMPBY(oparg + 1); - RECORD_DYNAMIC_JUMP_TAKEN(); DISPATCH(); } #endif @@ -3375,7 +3368,6 @@ dummy_func( null_or_index = PyStackRef_TagInt(-1); /* Jump forward oparg, then skip following END_FOR instruction */ JUMPBY(oparg + 1); - RECORD_DYNAMIC_JUMP_TAKEN(); DISPATCH(); } } @@ -3420,7 +3412,6 @@ dummy_func( if (r->len <= 0) { // Jump over END_FOR instruction. JUMPBY(oparg + 1); - RECORD_DYNAMIC_JUMP_TAKEN(); DISPATCH(); } } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 8169a36a9b9562..6ed8bce67cc1cf 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -6236,6 +6236,7 @@ if (result == 0) { null_or_index = PyStackRef_TagInt(-1); JUMPBY(oparg + 1); + RECORD_DYNAMIC_JUMP_TAKEN(); stack_pointer[-1] = null_or_index; DISPATCH(); } @@ -7363,6 +7364,7 @@ JUMP_TO_LABEL(error); } JUMPBY(oparg + 1); + RECORD_DYNAMIC_JUMP_TAKEN(); stack_pointer[-1] = null_or_index; DISPATCH(); } diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 7a5e8786f2a804..a525a2bf705559 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -35,6 +35,7 @@ class Properties: pure: bool uses_opcode: bool needs_guard_ip: bool + unpredictable_jump: bool tier: int | None = None const_oparg: int = -1 needs_prev: bool = False @@ -76,7 +77,8 @@ def from_list(properties: list["Properties"]) -> "Properties": pure=all(p.pure for p in properties), needs_prev=any(p.needs_prev for p in properties), no_save_ip=all(p.no_save_ip for p in properties), - needs_guard_ip=any(p.needs_guard_ip for p in properties) + needs_guard_ip=any(p.needs_guard_ip for p in properties), + unpredictable_jump=any(p.unpredictable_jump for p in properties), ) @property @@ -105,6 +107,7 @@ def infallible(self) -> bool: pure=True, no_save_ip=False, needs_guard_ip=False, + unpredictable_jump=False, ) @@ -887,6 +890,42 @@ def stmt_escapes(stmt: Stmt) -> bool: else: assert False, "Unexpected statement type" +def stmt_has_jump_on_unpredictable_path_body(stmts: list[Stmt] | None, branches_seen: int) -> bool: + if not stmts: + return False, branches_seen + predict = False + seen = 0 + for st in stmts: + predict_body, seen_body = stmt_has_jump_on_unpredictable_path(st, branches_seen) + predict = predict or predict_body + seen += seen_body + return predict, seen + +def stmt_has_jump_on_unpredictable_path(stmt: Stmt, branches_seen: int) -> bool: + if isinstance(stmt, BlockStmt): + return stmt_has_jump_on_unpredictable_path_body(stmt.body, branches_seen) + elif isinstance(stmt, SimpleStmt): + for tkn in stmt.contents: + if tkn.text == "JUMPBY": + return True, branches_seen + return False, branches_seen + elif isinstance(stmt, IfStmt): + return True, branches_seen + 1 + elif isinstance(stmt, MacroIfStmt): + predict, seen = stmt_has_jump_on_unpredictable_path_body(stmt.body, branches_seen) + if stmt.else_body: + predict_else, seen_else = stmt_has_jump_on_unpredictable_path_body(stmt.else_body, branches_seen) + return predict != predict_else, seen + seen_else + return predict, seen + elif isinstance(stmt, ForStmt): + unpredictable, branches_seen = stmt_has_jump_on_unpredictable_path(stmt.body, branches_seen) + return unpredictable, branches_seen + 1 + elif isinstance(stmt, WhileStmt): + unpredictable, branches_seen = stmt_has_jump_on_unpredictable_path(stmt.body, branches_seen) + return unpredictable, branches_seen + 1 + else: + assert False, f"Unexpected statement type {stmt}" + def compute_properties(op: parser.CodeDef) -> Properties: escaping_calls = find_escaping_api_calls(op) @@ -914,6 +953,8 @@ def compute_properties(op: parser.CodeDef) -> Properties: escapes = stmt_escapes(op.block) pure = False if isinstance(op, parser.LabelDef) else "pure" in op.annotations no_save_ip = False if isinstance(op, parser.LabelDef) else "no_save_ip" in op.annotations + unpredictable, branches_seen = stmt_has_jump_on_unpredictable_path(op.block, 0) + unpredictable_jump = False if isinstance(op, parser.LabelDef) else (unpredictable and branches_seen > 0) return Properties( escaping_calls=escaping_calls, escapes=escapes, @@ -938,6 +979,7 @@ def compute_properties(op: parser.CodeDef) -> Properties: tier=tier_variable(op), needs_prev=variable_used(op, "prev_instr"), needs_guard_ip=variable_used(op, "TIER2_STORE_IP") or variable_used(op, "LLTRACE_RESUME_FRAME") or variable_used(op, "DISPATCH_INLINED"), + unpredictable_jump=unpredictable_jump, ) def expand(items: list[StackItem], oparg: int) -> list[StackItem]: diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 804bd9773b164a..fed0b4c1643a30 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -128,6 +128,7 @@ def __init__(self, out: CWriter, labels: dict[str, Label], cannot_escape: bool = "DISPATCH": self.dispatch, "INSTRUCTION_SIZE": self.instruction_size, "stack_pointer": self.stack_pointer, + "JUMPBY": self.jumpby, } self.out = out self.labels = labels @@ -402,6 +403,30 @@ def sync_sp( storage.stack.clear(self.out) return True + def jumpby( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ) -> bool: + self.out.start_line() + self.emit(tkn) + lparen = next(tkn_iter) + self.emit(lparen) + jump = next(tkn_iter) + self.emit(jump) + emit_to(self.out, tkn_iter, "RPAREN") + next(tkn_iter) + self.emit(");\n") + + + if uop.properties.unpredictable_jump and jump.text != "0": + self.out.start_line() + self.emit("RECORD_DYNAMIC_JUMP_TAKEN();\n") + return True + def stack_pointer( self, tkn: Token, From ce66f3b836b5efb61cafab6a41242ec8f7e59ab7 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 24 Oct 2025 19:27:16 +0100 Subject: [PATCH 102/190] Remove TIER2_STORE_IP --- Objects/frameobject.c | 1 - Python/bytecodes.c | 20 ++---------------- Python/ceval.c | 15 +------------- Python/ceval_macros.h | 3 ++- Python/executor_cases.c.h | 26 +++++------------------- Python/generated_cases.c.h | 25 ----------------------- Tools/cases_generator/tier2_generator.py | 15 ++++++++++++++ 7 files changed, 25 insertions(+), 80 deletions(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index abed4547ffd053..37d30f349fedfe 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -17,7 +17,6 @@ #include "frameobject.h" // PyFrameLocalsProxyObject #include "opcode.h" // EXTENDED_ARG -#include "../Include/pytypedefs.h" #include "pycore_optimizer.h" #include "clinic/frameobject.c.h" diff --git a/Python/bytecodes.c b/Python/bytecodes.c index ee9d41bd5194cf..e53621298553b8 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1249,12 +1249,7 @@ dummy_func( frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); RELOAD_STACK(); - #if TIER_ONE LOAD_IP(frame->return_offset); - #endif - #if TIER_TWO - TIER2_STORE_IP(frame->return_offset); - #endif res = temp; LLTRACE_RESUME_FRAME(); } @@ -1441,12 +1436,7 @@ dummy_func( _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); #endif RELOAD_STACK(); - #if TIER_ONE LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - #endif - #if TIER_TWO - TIER2_STORE_IP(1 + INLINE_CACHE_ENTRIES_SEND); - #endif value = PyStackRef_MakeHeapSafe(temp); LLTRACE_RESUME_FRAME(); } @@ -3236,15 +3226,14 @@ dummy_func( } op(_FOR_ITER_TIER_TWO, (iter, null_or_index -- iter, null_or_index, next)) { - TIER2_STORE_IP(1 + INLINE_CACHE_ENTRIES_FOR_ITER); _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); if (!PyStackRef_IsValid(item)) { if (PyStackRef_IsError(item)) { ERROR_NO_POP(); } /* iterator ended normally */ - /* This just sets the IP to what it expects */ - TIER2_STORE_IP(oparg + 1); + /* This just sets the IP to what it expects (see normal _FOR_ITER */ + frame->instr_ptr += (oparg + 2 + INLINE_CACHE_ENTRIES_FOR_ITER); EXIT_IF(true); } next = item; @@ -4992,12 +4981,7 @@ dummy_func( _PyInterpreterFrame *prev = frame->previous; _PyThreadState_PopFrame(tstate, frame); frame = tstate->current_frame = prev; - #if TIER_ONE LOAD_IP(frame->return_offset); - #endif - #if TIER_TWO - TIER2_STORE_IP(frame->return_offset); - #endif RELOAD_STACK(); res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); LLTRACE_RESUME_FRAME(); diff --git a/Python/ceval.c b/Python/ceval.c index 49c1c5986c0f03..4597cc5e641645 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1213,20 +1213,7 @@ _PyTier2Interpreter( for (;;) { uopcode = next_uop->opcode; #ifdef Py_DEBUG - if (frame->lltrace >= 4) { - if (next_uop->opcode != _START_EXECUTOR) { - if (next_uop->format == UOP_FORMAT_TARGET) { - _Py_CODEUNIT *aim = _PyFrame_GetBytecode(frame) + next_uop->target; - printf(" aim=[%s]\n", _PyOpcode_OpName[aim->op.code]); - } - else if (next_uop->format == UOP_FORMAT_JUMP) { - _PyUOpInstruction *aim_uop = current_executor->trace + next_uop->jump_target; - if (aim_uop->format == UOP_FORMAT_TARGET) { - _Py_CODEUNIT *aim = _PyFrame_GetBytecode(frame) + aim_uop->target; - printf(" aim=[%s]\n", _PyOpcode_OpName[aim->op.code]); - } - } - } + if (frame->lltrace >= 3) { dump_stack(frame, stack_pointer); if (next_uop->opcode == _START_EXECUTOR) { printf("%4d uop: ", 0); diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index f85a7f9e271a50..a29e5d5cbbfd32 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -210,12 +210,14 @@ do { \ JUMP_TO_LABEL(start_frame); \ } while (0) #else + #define DISPATCH_SAME_OPARG() \ { \ opcode = next_instr->op.code; \ PRE_DISPATCH_GOTO(); \ DISPATCH_GOTO(); \ } + #define DISPATCH_INLINED(NEW_FRAME) \ do { \ assert(tstate->interp->eval_frame == NULL); \ @@ -257,7 +259,6 @@ GETITEM(PyObject *v, Py_ssize_t i) { * and skipped instructions. */ #define JUMPBY(x) (next_instr += (x)) -#define TIER2_STORE_IP(x) (frame->instr_ptr += (x)) #define SKIP_OVER(x) (next_instr += (x)) #define STACK_LEVEL() ((int)(stack_pointer - _PyFrame_Stackbase(frame))) diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 2fa109b049d1a9..fafaa7bd12bbb5 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1930,12 +1930,7 @@ frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); stack_pointer = _PyFrame_GetStackPointer(frame); - #if TIER_ONE - LOAD_IP(frame->return_offset); - #endif - #if TIER_TWO - TIER2_STORE_IP(frame->return_offset); - #endif + frame->instr_ptr += (frame->return_offset); res = temp; LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; @@ -2101,12 +2096,7 @@ _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); #endif stack_pointer = _PyFrame_GetStackPointer(frame); - #if TIER_ONE - LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - #endif - #if TIER_TWO - TIER2_STORE_IP(1 + INLINE_CACHE_ENTRIES_SEND); - #endif + frame->instr_ptr += (1 + INLINE_CACHE_ENTRIES_SEND); value = PyStackRef_MakeHeapSafe(temp); LLTRACE_RESUME_FRAME(); stack_pointer[0] = value; @@ -4398,7 +4388,6 @@ oparg = CURRENT_OPARG(); null_or_index = stack_pointer[-1]; iter = stack_pointer[-2]; - TIER2_STORE_IP(1 + INLINE_CACHE_ENTRIES_FOR_ITER); _PyFrame_SetStackPointer(frame, stack_pointer); _PyStackRef item = _PyForIter_VirtualIteratorNext(tstate, frame, iter, &null_or_index); stack_pointer = _PyFrame_GetStackPointer(frame); @@ -4406,7 +4395,7 @@ if (PyStackRef_IsError(item)) { JUMP_TO_ERROR(); } - TIER2_STORE_IP(oparg + 1); + frame->instr_ptr += (oparg + 2 + INLINE_CACHE_ENTRIES_FOR_ITER); if (true) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -5358,7 +5347,7 @@ frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; LOAD_SP(); - LOAD_IP(0); + frame->instr_ptr += (0); LLTRACE_RESUME_FRAME(); break; } @@ -6763,12 +6752,7 @@ _PyInterpreterFrame *prev = frame->previous; _PyThreadState_PopFrame(tstate, frame); frame = tstate->current_frame = prev; - #if TIER_ONE - LOAD_IP(frame->return_offset); - #endif - #if TIER_TWO - TIER2_STORE_IP(frame->return_offset); - #endif + frame->instr_ptr += (frame->return_offset); stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); LLTRACE_RESUME_FRAME(); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 6ed8bce67cc1cf..4dc2599bff5297 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -7873,12 +7873,7 @@ frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); stack_pointer = _PyFrame_GetStackPointer(frame); - #if TIER_ONE LOAD_IP(frame->return_offset); - #endif - #if TIER_TWO - TIER2_STORE_IP(frame->return_offset); - #endif res = temp; LLTRACE_RESUME_FRAME(); } @@ -7946,12 +7941,7 @@ _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); #endif stack_pointer = _PyFrame_GetStackPointer(frame); - #if TIER_ONE LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - #endif - #if TIER_TWO - TIER2_STORE_IP(1 + INLINE_CACHE_ENTRIES_SEND); - #endif value = PyStackRef_MakeHeapSafe(temp); LLTRACE_RESUME_FRAME(); } @@ -11045,12 +11035,7 @@ _PyInterpreterFrame *prev = frame->previous; _PyThreadState_PopFrame(tstate, frame); frame = tstate->current_frame = prev; - #if TIER_ONE LOAD_IP(frame->return_offset); - #endif - #if TIER_TWO - TIER2_STORE_IP(frame->return_offset); - #endif stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); LLTRACE_RESUME_FRAME(); @@ -11082,12 +11067,7 @@ frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); stack_pointer = _PyFrame_GetStackPointer(frame); - #if TIER_ONE LOAD_IP(frame->return_offset); - #endif - #if TIER_TWO - TIER2_STORE_IP(frame->return_offset); - #endif res = temp; LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; @@ -12875,12 +12855,7 @@ _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); #endif stack_pointer = _PyFrame_GetStackPointer(frame); - #if TIER_ONE LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - #endif - #if TIER_TWO - TIER2_STORE_IP(1 + INLINE_CACHE_ENTRIES_SEND); - #endif value = PyStackRef_MakeHeapSafe(temp); LLTRACE_RESUME_FRAME(); stack_pointer[0] = value; diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index 87b26b1c732599..e95a070b163f4c 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -65,6 +65,7 @@ def __init__(self, out: CWriter, labels: dict[str, Label]): self._replacers["oparg"] = self.oparg self._replacers["JUMPBY"] = self.jumpby self._replacers["DISPATCH"] = self.dispatch + self._replacers["LOAD_IP"] = self.load_ip def goto_error(self, offset: int, storage: Storage) -> str: # To do: Add jump targets for popping values. @@ -169,6 +170,20 @@ def dispatch( next(tkn_iter) return False + def load_ip( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ) -> bool: + self.out.start_line() + self.emit("frame->instr_ptr += ") + emit_to(self.out, tkn_iter, "SEMI") + self.emit(";\n") + return True + def write_uop(uop: Uop, emitter: Emitter, stack: Stack) -> Stack: locals: dict[str, Local] = {} try: From 573345538a20929d8480b36e9459d0532bbc4962 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 24 Oct 2025 19:42:15 +0100 Subject: [PATCH 103/190] make mypy happy --- Tools/cases_generator/analyzer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index a525a2bf705559..b206a0a4b985d3 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -890,7 +890,7 @@ def stmt_escapes(stmt: Stmt) -> bool: else: assert False, "Unexpected statement type" -def stmt_has_jump_on_unpredictable_path_body(stmts: list[Stmt] | None, branches_seen: int) -> bool: +def stmt_has_jump_on_unpredictable_path_body(stmts: list[Stmt] | None, branches_seen: int) -> tuple[bool, int]: if not stmts: return False, branches_seen predict = False @@ -901,7 +901,7 @@ def stmt_has_jump_on_unpredictable_path_body(stmts: list[Stmt] | None, branches_ seen += seen_body return predict, seen -def stmt_has_jump_on_unpredictable_path(stmt: Stmt, branches_seen: int) -> bool: +def stmt_has_jump_on_unpredictable_path(stmt: Stmt, branches_seen: int) -> tuple[bool, int]: if isinstance(stmt, BlockStmt): return stmt_has_jump_on_unpredictable_path_body(stmt.body, branches_seen) elif isinstance(stmt, SimpleStmt): From 4aab2dfa00c3e6b78c581fc36249e0f4672517d1 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 24 Oct 2025 20:04:10 +0100 Subject: [PATCH 104/190] Move specializing ddetection to specialize inst --- Include/internal/pycore_opcode_metadata.h | 1 + Lib/test/test_generated_cases.py | 3 - Python/bytecodes.c | 2 +- Python/ceval_macros.h | 22 +- Python/generated_cases.c.h | 769 ++------------------- Python/optimizer.c | 2 +- Tools/cases_generator/generators_common.py | 25 +- 7 files changed, 73 insertions(+), 751 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 288396f34fc3d4..0156e083b7759f 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1411,6 +1411,7 @@ _PyOpcode_macro_expansion[256] = { [IMPORT_FROM] = { .nuops = 1, .uops = { { _IMPORT_FROM, OPARG_SIMPLE, 0 } } }, [IMPORT_NAME] = { .nuops = 1, .uops = { { _IMPORT_NAME, OPARG_SIMPLE, 0 } } }, [IS_OP] = { .nuops = 1, .uops = { { _IS_OP, OPARG_SIMPLE, 0 } } }, + [JUMP_BACKWARD] = { .nuops = 2, .uops = { { _CHECK_PERIODIC, OPARG_SIMPLE, 1 }, { _JUMP_BACKWARD_NO_INTERRUPT, OPARG_REPLACED, 1 } } }, [JUMP_BACKWARD_NO_INTERRUPT] = { .nuops = 1, .uops = { { _JUMP_BACKWARD_NO_INTERRUPT, OPARG_REPLACED, 0 } } }, [JUMP_BACKWARD_NO_JIT] = { .nuops = 2, .uops = { { _CHECK_PERIODIC, OPARG_SIMPLE, 1 }, { _JUMP_BACKWARD_NO_INTERRUPT, OPARG_REPLACED, 1 } } }, [LIST_APPEND] = { .nuops = 1, .uops = { { _LIST_APPEND, OPARG_SIMPLE, 0 } } }, diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 1c04881bca5730..09ce329bdcd14d 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -329,9 +329,6 @@ def test_predictions(self): if (xxx) { UPDATE_MISS_STATS(OP1); assert(_PyOpcode_Deopt[opcode] == (OP1)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(OP1); } res = Py_None; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index e53621298553b8..bcc9f220335cd1 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2939,7 +2939,7 @@ dummy_func( JUMP_BACKWARD_JIT, }; - tier1 op(_SPECIALIZE_JUMP_BACKWARD, (--)) { + specializing tier1 op(_SPECIALIZE_JUMP_BACKWARD, (--)) { #if ENABLE_SPECIALIZATION if (this_instr->op.code == JUMP_BACKWARD) { uint8_t desired = tstate->interp->jit ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index a29e5d5cbbfd32..9dd7b1dea16530 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -136,7 +136,7 @@ #if (_Py_TAIL_CALL_INTERP || USE_COMPUTED_GOTOS) && _Py_TIER2 # define IS_JIT_TRACING() (DISPATCH_TABLE_VAR == TRACING_DISPATCH_TABLE) // Required to not get stuck in infinite specialization loops due to specialization failure. -# define IS_JIT_TRACING_MAKING_PROGRESS() (IS_JIT_TRACING() && tstate->interp->jit_state.specialize_counter <= 2) +# define IS_JIT_TRACING_MAKING_PROGRESS() (IS_JIT_TRACING() && tstate->interp->jit_state.specialize_counter < 1) # define ENTER_TRACING() \ DISPATCH_TABLE_VAR = TRACING_DISPATCH_TABLE; # define LEAVE_TRACING() \ @@ -192,31 +192,12 @@ do { \ DISPATCH_GOTO_NON_TRACING(); \ } -#if _Py_TIER2 #define DISPATCH_SAME_OPARG() \ { \ - tstate->interp->jit_state.specialize_counter++; \ opcode = next_instr->op.code; \ PRE_DISPATCH_GOTO(); \ DISPATCH_GOTO_NON_TRACING(); \ } -#define DISPATCH_INLINED(NEW_FRAME) \ - do { \ - assert(tstate->interp->eval_frame == NULL); \ - _PyFrame_SetStackPointer(frame, stack_pointer); \ - assert((NEW_FRAME)->previous == frame); \ - frame = tstate->current_frame = (NEW_FRAME); \ - CALL_STAT_INC(inlined_py_calls); \ - JUMP_TO_LABEL(start_frame); \ - } while (0) -#else - -#define DISPATCH_SAME_OPARG() \ - { \ - opcode = next_instr->op.code; \ - PRE_DISPATCH_GOTO(); \ - DISPATCH_GOTO(); \ - } #define DISPATCH_INLINED(NEW_FRAME) \ do { \ @@ -227,7 +208,6 @@ do { \ CALL_STAT_INC(inlined_py_calls); \ JUMP_TO_LABEL(start_frame); \ } while (0) -#endif /* Tuple access macros */ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 4dc2599bff5297..8a81a6cdcc4929 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -44,6 +44,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); stack_pointer = _PyFrame_GetStackPointer(frame); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(BINARY_OP); @@ -103,9 +106,6 @@ if (!PyFloat_CheckExact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -116,9 +116,6 @@ if (!PyFloat_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -167,9 +164,6 @@ if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -180,9 +174,6 @@ if (!_PyLong_CheckExactAndCompact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -200,9 +191,6 @@ if (PyStackRef_IsNull(res)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(BINARY_OP); } PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); @@ -237,9 +225,6 @@ if (!PyUnicode_CheckExact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -250,9 +235,6 @@ if (!PyUnicode_CheckExact(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -311,9 +293,6 @@ if (!res) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -369,9 +348,6 @@ if (!PyUnicode_CheckExact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -382,9 +358,6 @@ if (!PyUnicode_CheckExact(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -408,9 +381,6 @@ if (PyStackRef_AsPyObjectBorrow(*target_local) != left_o) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(BINARY_OP); } STAT_INC(BINARY_OP, hit); @@ -461,9 +431,6 @@ if (!PyFloat_CheckExact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -474,9 +441,6 @@ if (!PyFloat_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -525,9 +489,6 @@ if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -538,9 +499,6 @@ if (!_PyLong_CheckExactAndCompact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -558,9 +516,6 @@ if (PyStackRef_IsNull(res)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(BINARY_OP); } PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); @@ -594,9 +549,6 @@ if (!PyDict_CheckExact(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -662,9 +614,6 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -675,9 +624,6 @@ if (!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(BINARY_OP); } PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; @@ -685,9 +631,6 @@ if (getitem_o == NULL) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(BINARY_OP); } assert(PyFunction_Check(getitem_o)); @@ -695,9 +638,6 @@ if (((PyFunctionObject *)getitem_o)->func_version != cached_version) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(BINARY_OP); } PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o); @@ -705,9 +645,6 @@ if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(BINARY_OP); } getitem = PyStackRef_FromPyObjectNew(getitem_o); @@ -763,9 +700,6 @@ if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -776,9 +710,6 @@ if (!PyList_CheckExact(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -794,9 +725,6 @@ if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(BINARY_OP); } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; @@ -807,9 +735,6 @@ if (res_o == NULL) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(BINARY_OP); } STAT_INC(BINARY_OP, hit); @@ -818,9 +743,6 @@ if (index >= PyList_GET_SIZE(list)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(BINARY_OP); } STAT_INC(BINARY_OP, hit); @@ -868,9 +790,6 @@ if (!PySlice_Check(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -881,9 +800,6 @@ if (!PyList_CheckExact(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -946,9 +862,6 @@ if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -959,9 +872,6 @@ if (!PyUnicode_CheckExact(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -977,27 +887,18 @@ if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(BINARY_OP); } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; if (PyUnicode_GET_LENGTH(str) <= index) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(BINARY_OP); } Py_UCS4 c = PyUnicode_READ_CHAR(str, index); if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(BINARY_OP); } STAT_INC(BINARY_OP, hit); @@ -1039,9 +940,6 @@ if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -1052,9 +950,6 @@ if (!PyTuple_CheckExact(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -1070,18 +965,12 @@ if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(BINARY_OP); } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; if (index >= PyTuple_GET_SIZE(tuple)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(BINARY_OP); } STAT_INC(BINARY_OP, hit); @@ -1123,9 +1012,6 @@ if (!PyFloat_CheckExact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -1136,9 +1022,6 @@ if (!PyFloat_CheckExact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -1187,9 +1070,6 @@ if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -1200,9 +1080,6 @@ if (!_PyLong_CheckExactAndCompact(left_o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(BINARY_OP); } } @@ -1220,9 +1097,6 @@ if (PyStackRef_IsNull(res)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(BINARY_OP); } PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc); @@ -1664,6 +1538,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_Call(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); stack_pointer = _PyFrame_GetStackPointer(frame); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CALL); @@ -1838,9 +1715,6 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } } @@ -1853,26 +1727,17 @@ if (!PyStackRef_IsNull(self_or_null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } if (!PyType_Check(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } PyTypeObject *tp = (PyTypeObject *)callable_o; if (FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } assert(tp->tp_new == PyBaseObject_Type.tp_new); @@ -1884,9 +1749,6 @@ if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); @@ -1972,9 +1834,6 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } } @@ -1985,17 +1844,11 @@ if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } if (Py_TYPE(PyStackRef_AsPyObjectBorrow(callable)) != &PyMethod_Type) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } } @@ -2022,18 +1875,12 @@ if (!PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } } @@ -2046,9 +1893,6 @@ if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } } @@ -2060,9 +1904,6 @@ if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } } @@ -2071,9 +1912,6 @@ if (tstate->py_recursion_remaining <= 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } } @@ -2139,9 +1977,6 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } } @@ -2154,34 +1989,22 @@ if (Py_TYPE(callable_o) != &PyMethod_Type) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } PyObject *func = ((PyMethodObject *)callable_o)->im_func; if (!PyFunction_Check(func)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } if (((PyFunctionObject *)func)->func_version != func_version) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } } @@ -2207,9 +2030,6 @@ if (tstate->py_recursion_remaining <= 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } } @@ -2289,9 +2109,6 @@ if (!PyType_Check(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } PyTypeObject *tp = (PyTypeObject *)callable_o; @@ -2304,9 +2121,6 @@ if (tp->tp_vectorcall == NULL) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); @@ -2406,17 +2220,11 @@ if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } if (PyCFunction_GET_FLAGS(callable_o) != METH_FASTCALL) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); @@ -2521,17 +2329,11 @@ if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } if (PyCFunction_GET_FLAGS(callable_o) != (METH_FASTCALL | METH_KEYWORDS)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); @@ -2635,33 +2437,21 @@ if (total_args != 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } if (!PyCFunction_CheckExact(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } if (PyCFunction_GET_FLAGS(callable_o) != METH_O) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } if (_Py_ReachedRecursionLimit(tstate)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); @@ -2960,9 +2750,6 @@ if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } } @@ -2974,9 +2761,6 @@ if (callable_o != interp->callable_cache.isinstance) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } } @@ -3047,6 +2831,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); stack_pointer = _PyFrame_GetStackPointer(frame); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CALL_KW); @@ -3223,9 +3010,6 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL_KW); } } @@ -3238,34 +3022,22 @@ if (Py_TYPE(callable_o) != &PyMethod_Type) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL_KW); } PyObject *func = ((PyMethodObject *)callable_o)->im_func; if (!PyFunction_Check(func)) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL_KW); } if (((PyFunctionObject *)func)->func_version != func_version) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL_KW); } if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL_KW); } } @@ -3371,17 +3143,11 @@ if (PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL_KW); } if (Py_TYPE(callable_o) == &PyMethod_Type) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL_KW); } } @@ -3500,9 +3266,6 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL_KW); } } @@ -3514,18 +3277,12 @@ if (!PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL_KW); } PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL_KW); } } @@ -3534,9 +3291,6 @@ if (tstate->py_recursion_remaining <= 1) { UPDATE_MISS_STATS(CALL_KW); assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL_KW); } } @@ -3623,9 +3377,6 @@ if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } } @@ -3637,9 +3388,6 @@ if (callable_o != interp->callable_cache.len) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } } @@ -3703,9 +3451,6 @@ if (callable_o != interp->callable_cache.list_append) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } } @@ -3716,9 +3461,6 @@ if (o == NULL) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } } @@ -3728,9 +3470,6 @@ if (!PyList_CheckExact(o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } } @@ -3743,17 +3482,11 @@ if (!PyList_CheckExact(self_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } if (!LOCK_OBJECT(self_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); @@ -3813,27 +3546,18 @@ if (total_args == 0) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } PyMethodDef *meth = method->d_method; if (meth->ml_flags != METH_FASTCALL) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } PyObject *self = PyStackRef_AsPyObjectBorrow(arguments[0]); @@ -3841,9 +3565,6 @@ if (!Py_IS_TYPE(self, method->d_common.d_type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); @@ -3946,27 +3667,18 @@ if (total_args == 0) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } PyMethodDef *meth = method->d_method; if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } PyTypeObject *d_type = method->d_common.d_type; @@ -3975,9 +3687,6 @@ if (!Py_IS_TYPE(self, d_type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); @@ -4081,18 +3790,12 @@ if (total_args != 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } PyMethodDescrObject *method = (PyMethodDescrObject *)callable_o; if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } PyMethodDef *meth = method->d_method; @@ -4101,25 +3804,16 @@ if (!Py_IS_TYPE(self, method->d_common.d_type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } if (meth->ml_flags != METH_NOARGS) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } if (_Py_ReachedRecursionLimit(tstate)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); @@ -4190,34 +3884,22 @@ if (total_args != 2) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } PyMethodDef *meth = method->d_method; if (meth->ml_flags != METH_O) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } if (_Py_ReachedRecursionLimit(tstate)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } _PyStackRef arg_stackref = arguments[1]; @@ -4226,9 +3908,6 @@ method->d_common.d_type)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } STAT_INC(CALL, hit); @@ -4303,17 +3982,11 @@ if (PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } if (Py_TYPE(callable_o) == &PyMethod_Type) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } } @@ -4420,9 +4093,6 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } } @@ -4434,18 +4104,12 @@ if (!PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } } @@ -4459,9 +4123,6 @@ if (code->co_argcount != oparg + (!PyStackRef_IsNull(self_or_null))) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } } @@ -4473,9 +4134,6 @@ if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } } @@ -4484,9 +4142,6 @@ if (tstate->py_recursion_remaining <= 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } } @@ -4551,9 +4206,6 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } } @@ -4565,18 +4217,12 @@ if (!PyFunction_Check(callable_o)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } PyFunctionObject *func = (PyFunctionObject *)callable_o; if (func->func_version != func_version) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } } @@ -4585,9 +4231,6 @@ if (tstate->py_recursion_remaining <= 1) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } } @@ -4665,9 +4308,6 @@ if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } } @@ -4678,9 +4318,6 @@ if (callable_o != (PyObject *)&PyUnicode_Type) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } } @@ -4743,9 +4380,6 @@ if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } } @@ -4756,9 +4390,6 @@ if (callable_o != (PyObject *)&PyTuple_Type) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } } @@ -4821,9 +4452,6 @@ if (!PyStackRef_IsNull(null)) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } } @@ -4834,9 +4462,6 @@ if (callable_o != (PyObject *)&PyType_Type) { UPDATE_MISS_STATS(CALL); assert(_PyOpcode_Deopt[opcode] == (CALL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CALL); } } @@ -5052,6 +4677,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_CompareOp(left, right, next_instr, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(COMPARE_OP); @@ -5121,9 +4749,6 @@ if (!PyFloat_CheckExact(value_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(COMPARE_OP); } } @@ -5134,9 +4759,6 @@ if (!PyFloat_CheckExact(left_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(COMPARE_OP); } } @@ -5182,9 +4804,6 @@ if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(COMPARE_OP); } } @@ -5195,9 +4814,6 @@ if (!_PyLong_CheckExactAndCompact(left_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(COMPARE_OP); } } @@ -5248,9 +4864,6 @@ if (!PyUnicode_CheckExact(value_o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(COMPARE_OP); } } @@ -5261,9 +4874,6 @@ if (!PyUnicode_CheckExact(o)) { UPDATE_MISS_STATS(COMPARE_OP); assert(_PyOpcode_Deopt[opcode] == (COMPARE_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(COMPARE_OP); } } @@ -5315,6 +4925,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_ContainsOp(right, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CONTAINS_OP); @@ -5372,9 +4985,6 @@ if (!PyDict_CheckExact(o)) { UPDATE_MISS_STATS(CONTAINS_OP); assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CONTAINS_OP); } } @@ -5433,9 +5043,6 @@ if (!PyAnySet_CheckExact(o)) { UPDATE_MISS_STATS(CONTAINS_OP); assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(CONTAINS_OP); } } @@ -6045,6 +5652,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_ForIter(iter, null_or_index, next_instr, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(FOR_ITER); @@ -6094,9 +5704,6 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(FOR_ITER); } } @@ -6107,27 +5714,18 @@ if (Py_TYPE(gen) != &PyGen_Type) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(FOR_ITER); } #ifdef Py_GIL_DISABLED if (!_PyObject_IsUniquelyReferenced((PyObject *)gen)) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(FOR_ITER); } #endif if (gen->gi_frame_state >= FRAME_EXECUTING) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(FOR_ITER); } STAT_INC(FOR_ITER, hit); @@ -6178,11 +5776,8 @@ iter = stack_pointer[-2]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); if (Py_TYPE(iter_o) != &PyList_Type) { - UPDATE_MISS_STATS(FOR_ITER); - assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif + UPDATE_MISS_STATS(FOR_ITER); + assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); JUMP_TO_PREDICTED(FOR_ITER); } assert(PyStackRef_IsTaggedInt(null_or_index)); @@ -6190,9 +5785,6 @@ if (!_Py_IsOwnedByCurrentThread(iter_o) && !_PyObject_GC_IS_SHARED(iter_o)) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(FOR_ITER); } #endif @@ -6228,9 +5820,6 @@ if (result < 0) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(FOR_ITER); } if (result == 0) { @@ -6273,18 +5862,12 @@ if (Py_TYPE(r) != &PyRangeIter_Type) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(FOR_ITER); } #ifdef Py_GIL_DISABLED if (!_PyObject_IsUniquelyReferenced((PyObject *)r)) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(FOR_ITER); } #endif @@ -6349,9 +5932,6 @@ if (Py_TYPE(iter_o) != &PyTuple_Type) { UPDATE_MISS_STATS(FOR_ITER); assert(_PyOpcode_Deopt[opcode] == (FOR_ITER)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(FOR_ITER); } assert(PyStackRef_IsTaggedInt(null_or_index)); @@ -8038,6 +7618,9 @@ uint8_t desired = tstate->interp->jit ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, desired); next_instr = this_instr; + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif DISPATCH_SAME_OPARG(); } #endif @@ -8266,6 +7849,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_LoadAttr(owner, next_instr, name); stack_pointer = _PyFrame_GetStackPointer(frame); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(LOAD_ATTR); @@ -8344,18 +7930,12 @@ if (!PyType_Check(owner_o)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8408,18 +7988,12 @@ if (!PyType_Check(owner_o)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } assert(type_version != 0); if (FT_ATOMIC_LOAD_UINT_RELAXED(((PyTypeObject *)owner_o)->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8431,9 +8005,6 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8484,9 +8055,6 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } PyTypeObject *cls = Py_TYPE(owner_o); @@ -8494,9 +8062,6 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(cls->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } assert(Py_IS_TYPE(getattribute, &PyFunction_Type)); @@ -8505,9 +8070,6 @@ if (f->func_version != func_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } PyCodeObject *code = (PyCodeObject *)f->func_code; @@ -8515,9 +8077,6 @@ if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } STAT_INC(LOAD_ATTR, hit); @@ -8556,9 +8115,6 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8570,9 +8126,6 @@ if (!FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(owner_o)->valid)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8585,9 +8138,6 @@ if (attr_o == NULL) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } #ifdef Py_GIL_DISABLED @@ -8596,9 +8146,6 @@ if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8648,9 +8195,6 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8662,9 +8206,6 @@ if (dict != NULL) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8710,9 +8251,6 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8759,9 +8297,6 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8773,9 +8308,6 @@ if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8788,9 +8320,6 @@ if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8835,9 +8364,6 @@ if (Py_TYPE(owner_o)->tp_getattro != PyModule_Type.tp_getattro) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } PyDictObject *dict = (PyDictObject *)((PyModuleObject *)owner_o)->md_dict; @@ -8846,9 +8372,6 @@ if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != dict_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } assert(keys->dk_kind == DICT_KEYS_UNICODE); @@ -8858,9 +8381,6 @@ if (attr_o == NULL) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } #ifdef Py_GIL_DISABLED @@ -8869,9 +8389,6 @@ if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8920,9 +8437,6 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8970,9 +8484,6 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8984,9 +8495,6 @@ if (!FT_ATOMIC_LOAD_UINT8(ivs->valid)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -8999,9 +8507,6 @@ if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != keys_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -9043,9 +8548,6 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -9058,9 +8560,6 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -9075,33 +8574,21 @@ if ((code->co_flags & (CO_VARKEYWORDS | CO_VARARGS | CO_OPTIMIZED)) != CO_OPTIMIZED) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } if (code->co_kwonlyargcount) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } if (code->co_argcount != 1) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } STAT_INC(LOAD_ATTR, hit); @@ -9160,9 +8647,6 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -9175,9 +8659,6 @@ if (attr_o == NULL) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } #ifdef Py_GIL_DISABLED @@ -9185,9 +8666,6 @@ if (!increfed) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } #else @@ -9238,9 +8716,6 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -9253,9 +8728,6 @@ if (dict == NULL) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } PyDictKeysObject *dk = FT_ATOMIC_LOAD_PTR(dict->ma_keys); @@ -9264,9 +8736,6 @@ if (!_Py_IsOwnedByCurrentThread((PyObject *)dict) && !_PyObject_GC_IS_SHARED(dict)) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } #endif @@ -9275,9 +8744,6 @@ if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -9286,9 +8752,6 @@ if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -9297,9 +8760,6 @@ if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -9308,9 +8768,6 @@ if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -9321,9 +8778,6 @@ if (true) { UPDATE_MISS_STATS(LOAD_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_ATTR); } } @@ -9703,6 +9157,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); stack_pointer = _PyFrame_GetStackPointer(frame); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(LOAD_GLOBAL); @@ -9756,18 +9213,12 @@ if (!PyDict_CheckExact(dict)) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_GLOBAL); } PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_GLOBAL); } assert(DK_IS_UNICODE(keys)); @@ -9780,18 +9231,12 @@ if (!PyDict_CheckExact(dict)) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_GLOBAL); } PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_GLOBAL); } assert(DK_IS_UNICODE(keys)); @@ -9800,9 +9245,6 @@ if (res_o == NULL) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_GLOBAL); } #if Py_GIL_DISABLED @@ -9810,9 +9252,6 @@ if (!increfed) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_GLOBAL); } #else @@ -9858,18 +9297,12 @@ if (!PyDict_CheckExact(dict)) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_GLOBAL); } PyDictKeysObject *keys = FT_ATOMIC_LOAD_PTR_ACQUIRE(dict->ma_keys); if (FT_ATOMIC_LOAD_UINT32_RELAXED(keys->dk_version) != version) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_GLOBAL); } assert(DK_IS_UNICODE(keys)); @@ -9879,9 +9312,6 @@ if (res_o == NULL) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_GLOBAL); } #if Py_GIL_DISABLED @@ -9889,9 +9319,6 @@ if (!increfed) { UPDATE_MISS_STATS(LOAD_GLOBAL); assert(_PyOpcode_Deopt[opcode] == (LOAD_GLOBAL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_GLOBAL); } #else @@ -10053,6 +9480,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); stack_pointer = _PyFrame_GetStackPointer(frame); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR); @@ -10186,17 +9616,11 @@ if (global_super != (PyObject *)&PySuper_Type) { UPDATE_MISS_STATS(LOAD_SUPER_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); } if (!PyType_Check(class)) { UPDATE_MISS_STATS(LOAD_SUPER_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); } STAT_INC(LOAD_SUPER_ATTR, hit); @@ -10255,17 +9679,11 @@ if (global_super != (PyObject *)&PySuper_Type) { UPDATE_MISS_STATS(LOAD_SUPER_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); } if (!PyType_Check(class)) { UPDATE_MISS_STATS(LOAD_SUPER_ATTR); assert(_PyOpcode_Deopt[opcode] == (LOAD_SUPER_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(LOAD_SUPER_ATTR); } STAT_INC(LOAD_SUPER_ATTR, hit); @@ -10974,9 +10392,6 @@ if (_Py_emscripten_signal_clock == 0) { UPDATE_MISS_STATS(RESUME); assert(_PyOpcode_Deopt[opcode] == (RESUME)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(RESUME); } _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; @@ -10987,9 +10402,6 @@ if (eval_breaker != version) { UPDATE_MISS_STATS(RESUME); assert(_PyOpcode_Deopt[opcode] == (RESUME)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(RESUME); } #ifdef Py_GIL_DISABLED @@ -10997,9 +10409,6 @@ ((_PyThreadStateImpl *)tstate)->tlbc_index) { UPDATE_MISS_STATS(RESUME); assert(_PyOpcode_Deopt[opcode] == (RESUME)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(RESUME); } #endif @@ -11101,6 +10510,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_Send(receiver, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(SEND); @@ -11203,9 +10615,6 @@ if (tstate->interp->eval_frame) { UPDATE_MISS_STATS(SEND); assert(_PyOpcode_Deopt[opcode] == (SEND)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(SEND); } } @@ -11217,17 +10626,11 @@ if (Py_TYPE(gen) != &PyGen_Type && Py_TYPE(gen) != &PyCoro_Type) { UPDATE_MISS_STATS(SEND); assert(_PyOpcode_Deopt[opcode] == (SEND)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(SEND); } if (gen->gi_frame_state >= FRAME_EXECUTING) { UPDATE_MISS_STATS(SEND); assert(_PyOpcode_Deopt[opcode] == (SEND)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(SEND); } STAT_INC(SEND, hit); @@ -11410,6 +10813,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_StoreAttr(owner, next_instr, name); stack_pointer = _PyFrame_GetStackPointer(frame); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(STORE_ATTR); @@ -11465,9 +10871,6 @@ if (!LOCK_OBJECT(owner_o)) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(STORE_ATTR); } PyTypeObject *tp = Py_TYPE(owner_o); @@ -11476,9 +10879,6 @@ if (true) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(STORE_ATTR); } } @@ -11494,9 +10894,6 @@ if (true) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(STORE_ATTR); } } @@ -11550,9 +10947,6 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(STORE_ATTR); } } @@ -11564,9 +10958,6 @@ if (!LOCK_OBJECT(owner_o)) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(STORE_ATTR); } char *addr = (char *)owner_o + index; @@ -11607,9 +10998,6 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(STORE_ATTR); } } @@ -11623,17 +11011,11 @@ if (dict == NULL) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(STORE_ATTR); } if (!LOCK_OBJECT(dict)) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(STORE_ATTR); } assert(PyDict_CheckExact((PyObject *)dict)); @@ -11644,9 +11026,6 @@ if (true) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(STORE_ATTR); } } @@ -11656,9 +11035,6 @@ if (true) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(STORE_ATTR); } } @@ -11668,9 +11044,6 @@ if (true) { UPDATE_MISS_STATS(STORE_ATTR); assert(_PyOpcode_Deopt[opcode] == (STORE_ATTR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(STORE_ATTR); } } @@ -11940,6 +11313,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_StoreSubscr(container, sub, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(STORE_SUBSCR); @@ -11995,9 +11371,6 @@ if (!PyDict_CheckExact(o)) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(STORE_SUBSCR); } } @@ -12049,9 +11422,6 @@ if (!_PyLong_CheckExactAndCompact(value_o)) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(STORE_SUBSCR); } } @@ -12062,9 +11432,6 @@ if (!PyList_CheckExact(o)) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(STORE_SUBSCR); } } @@ -12081,18 +11448,12 @@ if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(STORE_SUBSCR); } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; if (!LOCK_OBJECT(list)) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(STORE_SUBSCR); } if (index >= PyList_GET_SIZE(list)) { @@ -12100,9 +11461,6 @@ if (true) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(STORE_SUBSCR); } } @@ -12167,6 +11525,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_ToBool(value, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(TO_BOOL); @@ -12219,9 +11580,6 @@ if (FT_ATOMIC_LOAD_UINT_RELAXED(tp->tp_version_tag) != type_version) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(TO_BOOL); } } @@ -12259,9 +11617,6 @@ if (!PyStackRef_BoolCheck(value)) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(TO_BOOL); } STAT_INC(TO_BOOL, hit); @@ -12288,9 +11643,6 @@ if (!PyLong_CheckExact(value_o)) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(TO_BOOL); } STAT_INC(TO_BOOL, hit); @@ -12332,9 +11684,6 @@ if (!PyList_CheckExact(o)) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(TO_BOOL); } } @@ -12376,9 +11725,6 @@ if (!PyStackRef_IsNone(value)) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(TO_BOOL); } STAT_INC(TO_BOOL, hit); @@ -12407,9 +11753,6 @@ if (!PyUnicode_CheckExact(value_o)) { UPDATE_MISS_STATS(TO_BOOL); assert(_PyOpcode_Deopt[opcode] == (TO_BOOL)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(TO_BOOL); } } @@ -12565,6 +11908,9 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_UnpackSequence(seq, next_instr, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); + #if _Py_TIER2 + tstate->interp->jit_state.specialize_counter++; + #endif DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(UNPACK_SEQUENCE); @@ -12613,9 +11959,6 @@ if (!PyList_CheckExact(o)) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(UNPACK_SEQUENCE); } } @@ -12629,9 +11972,6 @@ if (!LOCK_OBJECT(seq_o)) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(UNPACK_SEQUENCE); } if (PyList_GET_SIZE(seq_o) != oparg) { @@ -12639,9 +11979,6 @@ if (true) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(UNPACK_SEQUENCE); } } @@ -12681,9 +12018,6 @@ if (!PyTuple_CheckExact(o)) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(UNPACK_SEQUENCE); } } @@ -12697,9 +12031,6 @@ if (PyTuple_GET_SIZE(seq_o) != oparg) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(UNPACK_SEQUENCE); } STAT_INC(UNPACK_SEQUENCE, hit); @@ -12738,9 +12069,6 @@ if (!PyTuple_CheckExact(o)) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(UNPACK_SEQUENCE); } } @@ -12754,9 +12082,6 @@ if (PyTuple_GET_SIZE(seq_o) != 2) { UPDATE_MISS_STATS(UNPACK_SEQUENCE); assert(_PyOpcode_Deopt[opcode] == (UNPACK_SEQUENCE)); - #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; - #endif JUMP_TO_PREDICTED(UNPACK_SEQUENCE); } STAT_INC(UNPACK_SEQUENCE, hit); diff --git a/Python/optimizer.c b/Python/optimizer.c index 31c9995c4ed6b4..0c8202f60f1030 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -594,7 +594,7 @@ _PyJit_translate_single_bytecode_to_trace( int oparg = tstate->interp->jit_state.prev_instr_oparg; int opcode = this_instr->op.code; // Failed specialization twice in a row. Deopt! - if (tstate->interp->jit_state.specialize_counter >= 3) { + if (tstate->interp->jit_state.specialize_counter >= 1) { opcode = _PyOpcode_Deopt[opcode]; } int rewind_oparg = oparg; diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index fed0b4c1643a30..5cba378933e9e6 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -7,6 +7,7 @@ analysis_error, Label, CodeSection, + Uop, ) from cwriter import CWriter from typing import Callable, TextIO, Iterator, Iterable @@ -129,6 +130,7 @@ def __init__(self, out: CWriter, labels: dict[str, Label], cannot_escape: bool = "INSTRUCTION_SIZE": self.instruction_size, "stack_pointer": self.stack_pointer, "JUMPBY": self.jumpby, + "DISPATCH_SAME_OPARG": self.dispatch_same_oparg, } self.out = out self.labels = labels @@ -149,6 +151,26 @@ def dispatch( self.emit(tkn) return False + def dispatch_same_oparg( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ) -> bool: + assert isinstance(uop, Uop) + assert "specializing" in uop.annotations, uop.name + self.out.start_line() + self.emit("#if _Py_TIER2\n") + self.emit("tstate->interp->jit_state.specialize_counter++;\n") + self.emit("#endif\n") + self.emit(tkn) + emit_to(self.out, tkn_iter, "SEMI") + self.emit(";\n") + self.out.start_line() + return False + def deopt_if( self, tkn: Token, @@ -170,9 +192,6 @@ def deopt_if( family_name = inst.family.name self.emit(f"UPDATE_MISS_STATS({family_name});\n") self.emit(f"assert(_PyOpcode_Deopt[opcode] == ({family_name}));\n") - self.emit(f"#if _Py_TIER2\n") - self.emit(f"tstate->interp->jit_state.specialize_counter++;\n") - self.emit(f"#endif\n") self.emit(f"JUMP_TO_PREDICTED({self.jump_prefix}{family_name});\n") self.emit("}\n") return not always_true(first_tkn) From ab7527c5a70398dececf80fda944e2fe6550f217 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 24 Oct 2025 20:20:23 +0100 Subject: [PATCH 105/190] Fix the counters --- Include/internal/pycore_backoff.h | 8 ++++++++ Python/ceval_macros.h | 3 +-- Python/optimizer.c | 19 ++++++++++--------- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/Include/internal/pycore_backoff.h b/Include/internal/pycore_backoff.h index 454c8dde031ff4..f92dfd278a1e68 100644 --- a/Include/internal/pycore_backoff.h +++ b/Include/internal/pycore_backoff.h @@ -129,6 +129,14 @@ initial_unreachable_backoff_counter(void) return make_backoff_counter(0, UNREACHABLE_BACKOFF); } +// Required to not get stuck in infinite specialization loops due to specialization failure. +// We use 2 here as tnere are a few scenarios: +// 1. Freshly specialized from unspecialized, in which case the counter will be 1. +// 2. Re-specialized from deopt, in which case the counter will be 1. +// 3. Deopt -> Specialize -> Deopt -> Specialize, in which case the counter will be 2. +// We do not want the 3rd case. +#define MAX_SPECIALIZATION_TRIES 2 + #ifdef __cplusplus } #endif diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 9dd7b1dea16530..0d6d5174204d96 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -135,8 +135,7 @@ #if (_Py_TAIL_CALL_INTERP || USE_COMPUTED_GOTOS) && _Py_TIER2 # define IS_JIT_TRACING() (DISPATCH_TABLE_VAR == TRACING_DISPATCH_TABLE) -// Required to not get stuck in infinite specialization loops due to specialization failure. -# define IS_JIT_TRACING_MAKING_PROGRESS() (IS_JIT_TRACING() && tstate->interp->jit_state.specialize_counter < 1) +# define IS_JIT_TRACING_MAKING_PROGRESS() (IS_JIT_TRACING() && tstate->interp->jit_state.specialize_counter < MAX_SPECIALIZATION_TRIES) # define ENTER_TRACING() \ DISPATCH_TABLE_VAR = TRACING_DISPATCH_TABLE; # define LEAVE_TRACING() \ diff --git a/Python/optimizer.c b/Python/optimizer.c index 0c8202f60f1030..9071265b508154 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -557,6 +557,14 @@ _PyJit_translate_single_bytecode_to_trace( _Py_CODEUNIT *next_instr) { +#ifdef Py_DEBUG + char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); + int lltrace = 0; + if (python_lltrace != NULL && *python_lltrace >= '0') { + lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that + } +#endif + PyCodeObject *old_code = tstate->interp->jit_state.prev_instr_code; // Something else finalized the trace. This can happen in multi-threaded scenarios as our trace // addition from bytecode execution to here is not atomic. @@ -576,13 +584,6 @@ _PyJit_translate_single_bytecode_to_trace( goto full; } -#ifdef Py_DEBUG - char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); - int lltrace = 0; - if (python_lltrace != NULL && *python_lltrace >= '0') { - lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that - } -#endif _Py_CODEUNIT *this_instr = tstate->interp->jit_state.prev_instr; _Py_CODEUNIT *target_instr = this_instr; uint32_t target = 0; @@ -593,8 +594,8 @@ _PyJit_translate_single_bytecode_to_trace( // We must point to the first EXTENDED_ARG when deopting. int oparg = tstate->interp->jit_state.prev_instr_oparg; int opcode = this_instr->op.code; - // Failed specialization twice in a row. Deopt! - if (tstate->interp->jit_state.specialize_counter >= 1) { + // Failed specialization many times. Deopt! + if (tstate->interp->jit_state.specialize_counter >= MAX_SPECIALIZATION_TRIES) { opcode = _PyOpcode_Deopt[opcode]; } int rewind_oparg = oparg; From 7e7b240043806d4c35c91c4699825321ca70f9c9 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 24 Oct 2025 20:22:16 +0100 Subject: [PATCH 106/190] fix windows builds --- Python/ceval_macros.h | 1 + 1 file changed, 1 insertion(+) diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 0d6d5174204d96..af992cc38ebbf2 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -128,6 +128,7 @@ #else # define TARGET(op) case op: TARGET_##op: # define DISPATCH_GOTO() goto dispatch_opcode +# define DISPATCH_GOTO_NON_TRACING() goto dispatch_opcode # define JUMP_TO_LABEL(name) goto name; # define JUMP_TO_PREDICTED(name) goto PREDICTED_##name; # define LABEL(name) name: From 4a4a31f0962734df3555200b0a9846c1f6356fbe Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 25 Oct 2025 18:45:33 +0100 Subject: [PATCH 107/190] Support underflow and yield value in the optimizer --- Include/internal/pycore_optimizer.h | 3 +- Python/optimizer.c | 3 + Python/optimizer_analysis.c | 14 +---- Python/optimizer_bytecodes.c | 85 ++++++++++++----------------- Python/optimizer_cases.c.h | 72 ++++++++++++------------ Python/optimizer_symbols.c | 36 ++++++++++-- 6 files changed, 106 insertions(+), 107 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index ca46249bde98df..7c542e322ecbc2 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -250,6 +250,7 @@ struct _Py_UOpsAbstractFrame { int stack_len; int locals_len; PyFunctionObject *func; + PyCodeObject *code; JitOptRef *stack_pointer; JitOptRef *stack; @@ -325,7 +326,7 @@ extern _Py_UOpsAbstractFrame *_Py_uop_frame_new( int curr_stackentries, JitOptRef *args, int arg_len); -extern int _Py_uop_frame_pop(JitOptContext *ctx); +extern int _Py_uop_frame_pop(JitOptContext *ctx, PyCodeObject *co, int curr_stackentries); PyAPI_FUNC(PyObject *) _Py_uop_symbols_test(PyObject *self, PyObject *ignored); diff --git a/Python/optimizer.c b/Python/optimizer.c index 9071265b508154..ef7e518faaed89 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -872,6 +872,9 @@ _PyJit_translate_single_bytecode_to_trace( else { operand = 0; } + ADD_TO_TRACE(uop, oparg, operand, target); + trace[trace_length - 1].operand1 = ((int)(frame->stackpointer - _PyFrame_Stackbase(frame))); + break; } if (uop == _BINARY_OP_INPLACE_ADD_UNICODE) { assert(i + 1 == nuops); diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 2d3f1d95d5ab14..c576f3b716bc2a 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -267,7 +267,7 @@ static PyCodeObject * get_current_code_object(JitOptContext *ctx) { - return (PyCodeObject *)ctx->frame->func->func_code; + return (PyCodeObject *)ctx->frame->code; } static PyObject * @@ -298,10 +298,6 @@ optimize_uops( JitOptContext context; JitOptContext *ctx = &context; uint32_t opcode = UINT16_MAX; - int curr_space = 0; - int max_space = 0; - _PyUOpInstruction *first_valid_check_stack = NULL; - _PyUOpInstruction *corresponding_check_stack = NULL; // Make sure that watchers are set up PyInterpreterState *interp = _PyInterpreterState_GET(); @@ -368,14 +364,6 @@ optimize_uops( /* Either reached the end or cannot optimize further, but there * would be no benefit in retrying later */ _Py_uop_abstractcontext_fini(ctx); - if (first_valid_check_stack != NULL) { - assert(first_valid_check_stack->opcode == _CHECK_STACK_SPACE); - assert(max_space > 0); - assert(max_space <= INT_MAX); - assert(max_space <= INT32_MAX); - first_valid_check_stack->opcode = _CHECK_STACK_SPACE_OPERAND; - first_valid_check_stack->operand0 = max_space; - } return trace_len; error: diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 09c5ae764e5ac3..7e4a9778a89efe 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -773,50 +773,55 @@ dummy_func(void) { JitOptRef temp = PyJitRef_StripReferenceInfo(retval); DEAD(retval); SAVE_STACK(); - PyCodeObject *co = get_current_code_object(ctx); ctx->frame->stack_pointer = stack_pointer; - if (frame_pop(ctx)) { + PyCodeObject *returning_code = get_code_with_logging(this_instr); + if (returning_code == NULL) { + ctx->done = true; + break; + } + int returning_stacklevel = this_instr->operand1; + if (frame_pop(ctx, returning_code, returning_stacklevel)) { break; } stack_pointer = ctx->frame->stack_pointer; - /* Stack space handling */ - assert(corresponding_check_stack == NULL); - assert(co != NULL); - int framesize = co->co_framesize; - assert(framesize > 0); - assert(framesize <= curr_space); - curr_space -= framesize; - RELOAD_STACK(); res = temp; } op(_RETURN_GENERATOR, ( -- res)) { SYNC_SP(); - PyCodeObject *co = get_current_code_object(ctx); ctx->frame->stack_pointer = stack_pointer; - frame_pop(ctx); + PyCodeObject *returning_code = get_code_with_logging(this_instr); + if (returning_code == NULL) { + ctx->done = true; + break; + } + int returning_stacklevel = this_instr->operand1; + if (frame_pop(ctx, returning_code, returning_stacklevel)) { + break; + } stack_pointer = ctx->frame->stack_pointer; res = sym_new_unknown(ctx); - /* Stack space handling */ - assert(corresponding_check_stack == NULL); - assert(co != NULL); - int framesize = co->co_framesize; - assert(framesize > 0); - assert(framesize <= curr_space); - curr_space -= framesize; - } - - op(_YIELD_VALUE, (unused -- value)) { - // TODO (gh-139109): handle this properly in a future optimization. - // A possibility to handle underflows is to just restore the current frame information - // from whatever is stored in the trace we record at that point of time. - // E.g. we record at this YIELD_VALUE, func_obj=x , stack_level=4 - // We can restore it to there. - value = sym_new_unknown(ctx); - ctx->done = true; - ctx->out_of_space = true; + } + + op(_YIELD_VALUE, (retval -- value)) { + // Mimics PyStackRef_MakeHeapSafe in the interpreter. + JitOptRef temp = PyJitRef_StripReferenceInfo(retval); + DEAD(retval); + SAVE_STACK(); + PyCodeObject *returning_code = get_code_with_logging(this_instr); + if (returning_code == NULL) { + ctx->done = true; + break; + } + int returning_stacklevel = this_instr->operand1; + if (frame_pop(ctx, returning_code, returning_stacklevel)) { + break; + } + stack_pointer = ctx->frame->stack_pointer; + RELOAD_STACK(); + value = temp; } op(_GET_ITER, (iterable -- iter, index_or_null)) { @@ -843,8 +848,6 @@ dummy_func(void) { } op(_CHECK_STACK_SPACE, (unused, unused, unused[oparg] -- unused, unused, unused[oparg])) { - assert(corresponding_check_stack == NULL); - corresponding_check_stack = this_instr; } op (_CHECK_STACK_SPACE_OPERAND, (framesize/2 -- )) { @@ -870,24 +873,6 @@ dummy_func(void) { PyCodeObject *co = (PyCodeObject *)func->func_code; assert(PyFunction_Check(func)); ctx->frame->func = func; - /* Stack space handling */ - int framesize = co->co_framesize; - assert(framesize > 0); - curr_space += framesize; - if (curr_space < 0 || curr_space > INT32_MAX) { - // won't fit in signed 32-bit int - ctx->done = true; - break; - } - max_space = curr_space > max_space ? curr_space : max_space; - if (first_valid_check_stack == NULL) { - first_valid_check_stack = corresponding_check_stack; - } - else if (corresponding_check_stack) { - // delete all but the first valid _CHECK_STACK_SPACE - corresponding_check_stack->opcode = _NOP; - } - corresponding_check_stack = NULL; } op(_UNPACK_SEQUENCE, (seq -- values[oparg], top[0])) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 002da75ea38481..5192280e5ca20f 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1116,18 +1116,17 @@ JitOptRef temp = PyJitRef_StripReferenceInfo(retval); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); - PyCodeObject *co = get_current_code_object(ctx); ctx->frame->stack_pointer = stack_pointer; - if (frame_pop(ctx)) { + PyCodeObject *returning_code = get_code_with_logging(this_instr); + if (returning_code == NULL) { + ctx->done = true; + break; + } + int returning_stacklevel = this_instr->operand1; + if (frame_pop(ctx, returning_code, returning_stacklevel)) { break; } stack_pointer = ctx->frame->stack_pointer; - assert(corresponding_check_stack == NULL); - assert(co != NULL); - int framesize = co->co_framesize; - assert(framesize > 0); - assert(framesize <= curr_space); - curr_space -= framesize; res = temp; stack_pointer[0] = res; stack_pointer += 1; @@ -1169,11 +1168,26 @@ } case _YIELD_VALUE: { + JitOptRef retval; JitOptRef value; - value = sym_new_unknown(ctx); - ctx->done = true; - ctx->out_of_space = true; - stack_pointer[-1] = value; + retval = stack_pointer[-1]; + JitOptRef temp = PyJitRef_StripReferenceInfo(retval); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + PyCodeObject *returning_code = get_code_with_logging(this_instr); + if (returning_code == NULL) { + ctx->done = true; + break; + } + int returning_stacklevel = this_instr->operand1; + if (frame_pop(ctx, returning_code, returning_stacklevel)) { + break; + } + stack_pointer = ctx->frame->stack_pointer; + value = temp; + stack_pointer[0] = value; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -2563,8 +2577,6 @@ } case _CHECK_STACK_SPACE: { - assert(corresponding_check_stack == NULL); - corresponding_check_stack = this_instr; break; } @@ -2620,21 +2632,6 @@ PyCodeObject *co = (PyCodeObject *)func->func_code; assert(PyFunction_Check(func)); ctx->frame->func = func; - int framesize = co->co_framesize; - assert(framesize > 0); - curr_space += framesize; - if (curr_space < 0 || curr_space > INT32_MAX) { - ctx->done = true; - break; - } - max_space = curr_space > max_space ? curr_space : max_space; - if (first_valid_check_stack == NULL) { - first_valid_check_stack = corresponding_check_stack; - } - else if (corresponding_check_stack) { - corresponding_check_stack->opcode = _NOP; - } - corresponding_check_stack = NULL; break; } @@ -3011,17 +3008,18 @@ case _RETURN_GENERATOR: { JitOptRef res; - PyCodeObject *co = get_current_code_object(ctx); ctx->frame->stack_pointer = stack_pointer; - frame_pop(ctx); + PyCodeObject *returning_code = get_code_with_logging(this_instr); + if (returning_code == NULL) { + ctx->done = true; + break; + } + int returning_stacklevel = this_instr->operand1; + if (frame_pop(ctx, returning_code, returning_stacklevel)) { + break; + } stack_pointer = ctx->frame->stack_pointer; res = sym_new_unknown(ctx); - assert(corresponding_check_stack == NULL); - assert(co != NULL); - int framesize = co->co_framesize; - assert(framesize > 0); - assert(framesize <= curr_space); - curr_space -= framesize; stack_pointer[0] = res; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index b0997f56b98b94..b9747b7fd8410f 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -824,6 +824,7 @@ _Py_uop_frame_new( } _Py_UOpsAbstractFrame *frame = &ctx->frames[ctx->curr_frame_depth]; + frame->code = co; frame->stack_len = co->co_stacksize; frame->locals_len = co->co_nlocalsplus; @@ -905,18 +906,41 @@ _Py_uop_abstractcontext_init(JitOptContext *ctx) } int -_Py_uop_frame_pop(JitOptContext *ctx) +_Py_uop_frame_pop(JitOptContext *ctx, PyCodeObject *co, int curr_stackentries) { _Py_UOpsAbstractFrame *frame = ctx->frame; ctx->n_consumed = frame->locals; + ctx->curr_frame_depth--; - // TODO gh-139109: Handle trace recording underflow - if (ctx->curr_frame_depth == 0) { - ctx->done = true; - ctx->out_of_space = true; + + if (ctx->curr_frame_depth >= 1) { + ctx->frame = &ctx->frames[ctx->curr_frame_depth - 1]; + + // We returned to the correct code. Nothing to do here. + if (co == ctx->frame->code) { + return 0; + } + // Else: the code we recorded doesn't match the code we *think* we're + // returning to. We could trace anything, we can't just return to the + // old frame. We have to restore what the tracer recorded + // as the traced next frame. + // Remove the current frame, and later swap it out with the right one. + else { + ctx->curr_frame_depth--; + } + } + // Else: trace stack underflow. + + // This handles swapping out frames. + assert(curr_stackentries >= 1); + // -1 to stackentries as we push to the stack our return value after this. + _Py_UOpsAbstractFrame *new_frame = _Py_uop_frame_new(ctx, co, curr_stackentries - 1, NULL, 0); + if (new_frame == NULL) { return 1; } - ctx->frame = &ctx->frames[ctx->curr_frame_depth - 1]; + + ctx->curr_frame_depth++; + ctx->frame = new_frame; return 0; } From 72e1738c45c4d4cdec63252eb9d322be9335b63a Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 25 Oct 2025 18:47:10 +0100 Subject: [PATCH 108/190] fix --- Python/optimizer_symbols.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index b9747b7fd8410f..f16c6b0659f86b 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -936,6 +936,7 @@ _Py_uop_frame_pop(JitOptContext *ctx, PyCodeObject *co, int curr_stackentries) // -1 to stackentries as we push to the stack our return value after this. _Py_UOpsAbstractFrame *new_frame = _Py_uop_frame_new(ctx, co, curr_stackentries - 1, NULL, 0); if (new_frame == NULL) { + ctx->done = true; return 1; } From 5e17707b9353592feb13e72aee72d1d56bd7b74f Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 25 Oct 2025 19:04:29 +0100 Subject: [PATCH 109/190] Fix a bug with ENTER_EXECUTOR linking --- Python/optimizer.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Python/optimizer.c b/Python/optimizer.c index ef7e518faaed89..38fccafeda4911 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -621,6 +621,10 @@ _PyJit_translate_single_bytecode_to_trace( return 1; } + if (opcode == ENTER_EXECUTOR) { + goto full; + } + if (!tstate->interp->jit_state.dependencies_still_valid) { goto done; } @@ -686,10 +690,6 @@ _PyJit_translate_single_bytecode_to_trace( // One for possible _DEOPT, one because _CHECK_VALIDITY itself might _DEOPT max_length -= 2; - if (opcode == ENTER_EXECUTOR) { - goto full; - } - const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; ADD_TO_TRACE(_CHECK_VALIDITY, 0, 0, target); From 1e132f01e32504316a449e7b6181514c925748fe Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 25 Oct 2025 23:49:49 +0100 Subject: [PATCH 110/190] up the trace length --- Include/internal/pycore_uop.h | 2 +- Python/optimizer.c | 20 ++++++++------------ 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/Include/internal/pycore_uop.h b/Include/internal/pycore_uop.h index bccc0b00a4d210..4e1b15af42caa3 100644 --- a/Include/internal/pycore_uop.h +++ b/Include/internal/pycore_uop.h @@ -36,7 +36,7 @@ typedef struct _PyUOpInstruction{ } _PyUOpInstruction; // This is the length of the trace we translate initially. -#define UOP_MAX_TRACE_LENGTH 1500 +#define UOP_MAX_TRACE_LENGTH 3000 #define UOP_BUFFER_SIZE (UOP_MAX_TRACE_LENGTH * sizeof(_PyUOpInstruction)) /* Bloom filter with m = 256 diff --git a/Python/optimizer.c b/Python/optimizer.c index 38fccafeda4911..d39156a41afdc2 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -544,9 +544,6 @@ add_to_trace( goto full; \ } -// Reserve space for N uops, plus 3 for _SET_IP, _CHECK_VALIDITY and _EXIT_TRACE -#define RESERVE(needed) RESERVE_RAW((needed) + 3, _PyUOpName(opcode)) - /* Returns 1 on success (added to trace), 0 on trace end. */ @@ -688,19 +685,13 @@ _PyJit_translate_single_bytecode_to_trace( } // One for possible _DEOPT, one because _CHECK_VALIDITY itself might _DEOPT - max_length -= 2; + max_length -= 1; const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; - ADD_TO_TRACE(_CHECK_VALIDITY, 0, 0, target); - assert(opcode != ENTER_EXECUTOR && opcode != EXTENDED_ARG); assert(!_PyErr_Occurred(tstate)); - if (!OPCODE_HAS_NO_SAVE_IP(opcode)) { - ADD_TO_TRACE(_SET_IP, 0, (uintptr_t)target_instr, target); - } - /* Special case the first instruction, * so that we can guarantee forward progress */ if (progress_needed && tstate->interp->jit_state.code_curr_size <= 2) { @@ -720,7 +711,13 @@ _PyJit_translate_single_bytecode_to_trace( max_length--; } - RESERVE_RAW(expansion->nuops + needs_guard_ip + 4, "uop and various checks"); + RESERVE_RAW(expansion->nuops + needs_guard_ip + 3 + (!OPCODE_HAS_NO_SAVE_IP(opcode)), "uop and various checks"); + + ADD_TO_TRACE(_CHECK_VALIDITY, 0, 0, target); + + if (!OPCODE_HAS_NO_SAVE_IP(opcode)) { + ADD_TO_TRACE(_SET_IP, 0, (uintptr_t)target_instr, target); + } switch (opcode) { @@ -780,7 +777,6 @@ _PyJit_translate_single_bytecode_to_trace( goto unsupported; } assert(nuops > 0); - RESERVE(nuops + 1); /* One extra for exit */ uint32_t orig_oparg = oparg; // For OPARG_TOP/BOTTOM uint32_t orig_target = target; for (int i = 0; i < nuops; i++) { From bf1753940c927faa99be64fe6a89920bfd5ca670 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 26 Oct 2025 00:56:37 +0100 Subject: [PATCH 111/190] Change the backoffs to fix nqueens --- Include/internal/pycore_backoff.h | 11 ++++++++--- Python/optimizer.c | 5 +++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/Include/internal/pycore_backoff.h b/Include/internal/pycore_backoff.h index f92dfd278a1e68..86ae7885fd1c5e 100644 --- a/Include/internal/pycore_backoff.h +++ b/Include/internal/pycore_backoff.h @@ -99,8 +99,13 @@ backoff_counter_triggers(_Py_BackoffCounter counter) // Must be larger than ADAPTIVE_COOLDOWN_VALUE, otherwise when JIT code is // invalidated we may construct a new trace before the bytecode has properly // re-specialized: -#define JUMP_BACKWARD_INITIAL_VALUE 4095 -#define JUMP_BACKWARD_INITIAL_BACKOFF 12 +// Note: this should be a prime number-1. This increases the likelihood of +// finding a "good" loop iteration to trace. +// For example, 4095 does not work for the nqueens benchmark on pyperformanc +// as we always end up tracing the loop iteration's +// exhaustion iteration. Which aborts our current tracer. +#define JUMP_BACKWARD_INITIAL_VALUE 4000 +#define JUMP_BACKWARD_INITIAL_BACKOFF 14 static inline _Py_BackoffCounter initial_jump_backoff_counter(void) { @@ -112,7 +117,7 @@ initial_jump_backoff_counter(void) * Must be larger than ADAPTIVE_COOLDOWN_VALUE, * otherwise when a side exit warms up we may construct * a new trace before the Tier 1 code has properly re-specialized. */ -#define SIDE_EXIT_INITIAL_VALUE 4095 +#define SIDE_EXIT_INITIAL_VALUE 4000 #define SIDE_EXIT_INITIAL_BACKOFF 12 static inline _Py_BackoffCounter diff --git a/Python/optimizer.c b/Python/optimizer.c index d39156a41afdc2..995ddee8605878 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -628,16 +628,19 @@ _PyJit_translate_single_bytecode_to_trace( // Strange control-flow, unsupported opcode, etc. if (tstate->interp->jit_state.dynamic_jump_taken) { + DPRINTF(2, "Unsupported: dynamic jump taken\n"); goto unsupported; } // This happens when a recursive call happens that we can't trace. Such as Python -> C -> Python calls // If we haven't guarded the IP, then it's untraceable. if (frame != tstate->interp->jit_state.prev_instr_frame && !needs_guard_ip) { + DPRINTF(2, "Unsupported: unguardable jump taken\n"); goto unsupported; } if (oparg > 0xFFFF) { + DPRINTF(2, "Unsupported: oparg too large\n"); goto unsupported; } @@ -647,10 +650,12 @@ _PyJit_translate_single_bytecode_to_trace( } if (opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO) { + DPRINTF(2, "Unsupported: strange control-flow\n"); goto unsupported; } if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) { + DPRINTF(2, "Unsupported: frame owned by interpreter\n"); unsupported: { // Rewind to previous instruction and replace with _EXIT_TRACE. From 7ab76a8fcbc8388a54cc0ca3caaaac5a56bbe680 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 26 Oct 2025 00:57:30 +0100 Subject: [PATCH 112/190] fix no-opt JIT --- Python/optimizer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/optimizer.c b/Python/optimizer.c index 995ddee8605878..f745f16fcf30b7 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -690,7 +690,7 @@ _PyJit_translate_single_bytecode_to_trace( } // One for possible _DEOPT, one because _CHECK_VALIDITY itself might _DEOPT - max_length -= 1; + max_length -= 2; const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; From 86ab7f152a55a0d333654d0b321e582973cfa91a Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 26 Oct 2025 01:00:17 +0000 Subject: [PATCH 113/190] Fix a test --- Lib/test/test_capi/test_opt.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 9adbe75b205945..b702da5320449e 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1241,14 +1241,8 @@ class Bar: pass res, ex = self._run_with_optimizer(thing, Foo()) - opnames = list(iter_opnames(ex)) - self.assertIsNotNone(ex) - self.assertEqual(res, TIER2_THRESHOLD * 6 + 1) - call = opnames.index("_CALL_BUILTIN_FAST") - load_attr_top = opnames.index("_POP_TOP_LOAD_CONST_INLINE_BORROW", 0, call) - load_attr_bottom = opnames.index("_POP_TOP_LOAD_CONST_INLINE_BORROW", call) - self.assertEqual(opnames[:load_attr_top].count("_GUARD_TYPE_VERSION"), 1) - self.assertEqual(opnames[call:load_attr_bottom].count("_CHECK_VALIDITY"), 2) + # Cleaned up by the invalidation. + self.assertIsNone(ex) def test_guard_type_version_removed_escaping(self): From 5f39672c303070dbb7a3850e6cb8fa2322c67cb3 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 26 Oct 2025 14:45:10 +0000 Subject: [PATCH 114/190] fix up gitattributes --- .gitattributes | 1 - Tools/c-analyzer/cpython/_parser.py | 1 - 2 files changed, 2 deletions(-) diff --git a/.gitattributes b/.gitattributes index 58cd96c0d3290d..823e3e975a23a8 100644 --- a/.gitattributes +++ b/.gitattributes @@ -97,7 +97,6 @@ Programs/test_frozenmain.h generated Python/Python-ast.c generated Python/executor_cases.c.h generated Python/generated_cases.c.h generated -Python/generated_tracer_cases.c.h generated Python/optimizer_cases.c.h generated Python/opcode_targets.h generated Python/stdlib_module_names.h generated diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index 0f944d45a4fc88..fd198d7d06c96f 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -78,7 +78,6 @@ def format_tsv_lines(lines): 'Python/deepfreeze/*.c', 'Python/frozen_modules/*.h', 'Python/generated_cases.c.h', - 'Python/generated_tracer_cases.c.h', 'Python/executor_cases.c.h', 'Python/optimizer_cases.c.h', 'Python/opcode_targets.h', From 440ad03aa4faffec9769452e60151b3f70defd15 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 26 Oct 2025 17:52:10 +0000 Subject: [PATCH 115/190] address review --- .github/workflows/jit.yml | 4 ++-- Include/internal/pycore_backoff.h | 2 +- Python/bytecodes.c | 2 +- Python/ceval_macros.h | 1 - Tools/cases_generator/tier1_generator.py | 2 +- 5 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index c91328cf01cc2d..6d0cd952442d8d 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -57,7 +57,7 @@ jobs: fail-fast: false matrix: target: -# To re-enable later when we support thesee. +# To re-enable later when we support these. # - i686-pc-windows-msvc/msvc # - x86_64-pc-windows-msvc/msvc # - aarch64-pc-windows-msvc/msvc @@ -71,7 +71,7 @@ jobs: llvm: - 19 include: -# To re-enable later when we support thesee. +# To re-enable later when we support these. # - target: i686-pc-windows-msvc/msvc # architecture: Win32 # runner: windows-2022 diff --git a/Include/internal/pycore_backoff.h b/Include/internal/pycore_backoff.h index 86ae7885fd1c5e..8ca351c30258fa 100644 --- a/Include/internal/pycore_backoff.h +++ b/Include/internal/pycore_backoff.h @@ -135,7 +135,7 @@ initial_unreachable_backoff_counter(void) } // Required to not get stuck in infinite specialization loops due to specialization failure. -// We use 2 here as tnere are a few scenarios: +// We use 2 here as there are a few scenarios: // 1. Freshly specialized from unspecialized, in which case the counter will be 1. // 2. Re-specialized from deopt, in which case the counter will be 1. // 3. Deopt -> Specialize -> Deopt -> Specialize, in which case the counter will be 2. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index bcc9f220335cd1..cd098ebe0eb2b6 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3232,7 +3232,7 @@ dummy_func( ERROR_NO_POP(); } /* iterator ended normally */ - /* This just sets the IP to what it expects (see normal _FOR_ITER */ + /* This just sets the IP to what it expects (see normal _FOR_ITER) */ frame->instr_ptr += (oparg + 2 + INLINE_CACHE_ENTRIES_FOR_ITER); EXIT_IF(true); } diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index af992cc38ebbf2..b845544beec600 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -209,7 +209,6 @@ do { \ JUMP_TO_LABEL(start_frame); \ } while (0) - /* Tuple access macros */ #ifndef Py_DEBUG diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index a7961f4fbc7cce..fb1874b1658cf9 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -288,7 +288,7 @@ def generate_tier1_cases( out.start_line() if reachable: # type: ignore[possibly-undefined] stack.flush(out) - out.emit(f"DISPATCH();\n") + out.emit("DISPATCH();\n") out.start_line() out.emit("}") out.emit("\n") From 9cc799983e9982c2e655a68dae8da275d2c71b65 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sun, 26 Oct 2025 20:55:44 +0000 Subject: [PATCH 116/190] Address Chris' review --- Python/ceval_macros.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index b845544beec600..634b949fa59c37 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -88,7 +88,6 @@ # define DISPATCH_TABLE instruction_funcptr_handler_table # define TRACING_DISPATCH_TABLE instruction_funcptr_tracing_table # define TARGET(op) Py_PRESERVE_NONE_CC PyObject *_TAIL_CALL_##op(TAIL_CALL_PARAMS) -# define TRACING_TARGET(op) Py_PRESERVE_NONE_CC PyObject *_TAIL_CALL_TRACING_##op(TAIL_CALL_PARAMS) # define DISPATCH_GOTO() \ do { \ @@ -119,7 +118,6 @@ # define DISPATCH_TABLE opcode_targets_table # define TRACING_DISPATCH_TABLE opcode_tracing_targets_table # define TARGET(op) TARGET_##op: -# define TRACING_TARGET(op) TARGET_TRACING_##op: # define DISPATCH_GOTO() goto *opcode_targets[opcode] # define DISPATCH_GOTO_NON_TRACING() goto *DISPATCH_TABLE[opcode]; # define JUMP_TO_LABEL(name) goto name; From 4b4e8575ec200c74cb5a50b86e45606fe1546c7f Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 27 Oct 2025 09:46:37 +0000 Subject: [PATCH 117/190] Address Kumar's review --- Include/internal/pycore_backoff.h | 2 +- Objects/frameobject.c | 4 ++-- Objects/funcobject.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_backoff.h b/Include/internal/pycore_backoff.h index 8ca351c30258fa..6b78322fb06232 100644 --- a/Include/internal/pycore_backoff.h +++ b/Include/internal/pycore_backoff.h @@ -101,7 +101,7 @@ backoff_counter_triggers(_Py_BackoffCounter counter) // re-specialized: // Note: this should be a prime number-1. This increases the likelihood of // finding a "good" loop iteration to trace. -// For example, 4095 does not work for the nqueens benchmark on pyperformanc +// For example, 4095 does not work for the nqueens benchmark on pyperformance // as we always end up tracing the loop iteration's // exhaustion iteration. Which aborts our current tracer. #define JUMP_BACKWARD_INITIAL_VALUE 4000 diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 37d30f349fedfe..b652973600c17d 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -262,8 +262,8 @@ framelocalsproxy_setitem(PyObject *self, PyObject *key, PyObject *value) } #if _Py_TIER2 - _Py_Executors_InvalidateDependency(PyInterpreterState_Get(), co, 1); - _PyJit_Tracer_InvalidateDependency(PyThreadState_GET(), co); + _Py_Executors_InvalidateDependency(_PyInterpreterState_GET(), co, 1); + _PyJit_Tracer_InvalidateDependency(_PyThreadState_GET(), co); #endif _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); diff --git a/Objects/funcobject.c b/Objects/funcobject.c index fcd79c7e4f4ea1..2311d8ce29a9d9 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -1152,7 +1152,7 @@ func_dealloc(PyObject *self) return; } #if _Py_TIER2 - _PyJit_Tracer_InvalidateDependency(PyThreadState_GET(), self); + _PyJit_Tracer_InvalidateDependency(_PyThreadState_GET(), self); #endif _PyObject_GC_UNTRACK(op); FT_CLEAR_WEAKREFS(self, op->func_weakreflist); From a5d918e0c52aea8cb6ad4330815491c734fcd090 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 27 Oct 2025 09:56:48 +0000 Subject: [PATCH 118/190] Change RECORD_PREVIOUS_INST to a label to save an opcode --- Include/internal/pycore_magic_number.h | 3 +- Include/internal/pycore_opcode_metadata.h | 9 +- Include/opcode_ids.h | 33 +- Lib/_opcode_metadata.py | 33 +- Programs/test_frozenmain.h | 4 +- Python/bytecodes.c | 68 +- Python/generated_cases.c.h | 83 +- Python/opcode_targets.h | 910 +++++++++++----------- Tools/cases_generator/target_generator.py | 4 +- Tools/cases_generator/tier1_generator.py | 31 +- 10 files changed, 573 insertions(+), 605 deletions(-) diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index 09c6b1f5b39274..7ec7bd1c695516 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -286,7 +286,6 @@ Known values: Python 3.15a1 3653 (Fix handling of opcodes that may leave operands on the stack when optimizing LOAD_FAST) Python 3.15a1 3654 (Fix missing exception handlers in logical expression) Python 3.15a1 3655 (Fix miscompilation of some module-level annotations) - Python 3.15a2 3656 (Add RECORD_PREVIOUS_INST for a tracing JIT) Python 3.16 will start with 3700 @@ -300,7 +299,7 @@ PC/launcher.c must also be updated. */ -#define PYC_MAGIC_NUMBER 3656 +#define PYC_MAGIC_NUMBER 3655 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 0156e083b7759f..2bb915ad62b17f 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -412,8 +412,6 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 0; case RAISE_VARARGS: return oparg; - case RECORD_PREVIOUS_INST: - return 0; case RERAISE: return 1 + oparg; case RESERVED: @@ -897,8 +895,6 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case RAISE_VARARGS: return 0; - case RECORD_PREVIOUS_INST: - return 0; case RERAISE: return oparg; case RESERVED: @@ -1253,7 +1249,6 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [PUSH_EXC_INFO] = { true, INSTR_FMT_IX, 0 }, [PUSH_NULL] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, - [RECORD_PREVIOUS_INST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG }, [RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [RESERVED] = { true, INSTR_FMT_IX, 0 }, [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, @@ -1701,7 +1696,6 @@ const char *_PyOpcode_OpName[267] = { [PUSH_EXC_INFO] = "PUSH_EXC_INFO", [PUSH_NULL] = "PUSH_NULL", [RAISE_VARARGS] = "RAISE_VARARGS", - [RECORD_PREVIOUS_INST] = "RECORD_PREVIOUS_INST", [RERAISE] = "RERAISE", [RESERVED] = "RESERVED", [RESUME] = "RESUME", @@ -1812,6 +1806,7 @@ const uint8_t _PyOpcode_NeedsGuardIp[256] = { extern const uint8_t _PyOpcode_Deopt[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Deopt[256] = { + [121] = 121, [122] = 122, [123] = 123, [124] = 124, @@ -2023,7 +2018,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [PUSH_EXC_INFO] = PUSH_EXC_INFO, [PUSH_NULL] = PUSH_NULL, [RAISE_VARARGS] = RAISE_VARARGS, - [RECORD_PREVIOUS_INST] = RECORD_PREVIOUS_INST, [RERAISE] = RERAISE, [RESERVED] = RESERVED, [RESUME] = RESUME, @@ -2073,6 +2067,7 @@ const uint8_t _PyOpcode_Deopt[256] = { #endif // NEED_OPCODE_METADATA #define EXTRA_CASES \ + case 121: \ case 122: \ case 123: \ case 124: \ diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 1c7468456dbd02..1d5c74adefcd35 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -115,23 +115,22 @@ extern "C" { #define POP_JUMP_IF_NOT_NONE 102 #define POP_JUMP_IF_TRUE 103 #define RAISE_VARARGS 104 -#define RECORD_PREVIOUS_INST 105 -#define RERAISE 106 -#define SEND 107 -#define SET_ADD 108 -#define SET_FUNCTION_ATTRIBUTE 109 -#define SET_UPDATE 110 -#define STORE_ATTR 111 -#define STORE_DEREF 112 -#define STORE_FAST 113 -#define STORE_FAST_LOAD_FAST 114 -#define STORE_FAST_STORE_FAST 115 -#define STORE_GLOBAL 116 -#define STORE_NAME 117 -#define SWAP 118 -#define UNPACK_EX 119 -#define UNPACK_SEQUENCE 120 -#define YIELD_VALUE 121 +#define RERAISE 105 +#define SEND 106 +#define SET_ADD 107 +#define SET_FUNCTION_ATTRIBUTE 108 +#define SET_UPDATE 109 +#define STORE_ATTR 110 +#define STORE_DEREF 111 +#define STORE_FAST 112 +#define STORE_FAST_LOAD_FAST 113 +#define STORE_FAST_STORE_FAST 114 +#define STORE_GLOBAL 115 +#define STORE_NAME 116 +#define SWAP 117 +#define UNPACK_EX 118 +#define UNPACK_SEQUENCE 119 +#define YIELD_VALUE 120 #define RESUME 128 #define BINARY_OP_ADD_FLOAT 129 #define BINARY_OP_ADD_INT 130 diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index b6bce074f8e579..f168d169a32948 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -312,23 +312,22 @@ 'POP_JUMP_IF_NOT_NONE': 102, 'POP_JUMP_IF_TRUE': 103, 'RAISE_VARARGS': 104, - 'RECORD_PREVIOUS_INST': 105, - 'RERAISE': 106, - 'SEND': 107, - 'SET_ADD': 108, - 'SET_FUNCTION_ATTRIBUTE': 109, - 'SET_UPDATE': 110, - 'STORE_ATTR': 111, - 'STORE_DEREF': 112, - 'STORE_FAST': 113, - 'STORE_FAST_LOAD_FAST': 114, - 'STORE_FAST_STORE_FAST': 115, - 'STORE_GLOBAL': 116, - 'STORE_NAME': 117, - 'SWAP': 118, - 'UNPACK_EX': 119, - 'UNPACK_SEQUENCE': 120, - 'YIELD_VALUE': 121, + 'RERAISE': 105, + 'SEND': 106, + 'SET_ADD': 107, + 'SET_FUNCTION_ATTRIBUTE': 108, + 'SET_UPDATE': 109, + 'STORE_ATTR': 110, + 'STORE_DEREF': 111, + 'STORE_FAST': 112, + 'STORE_FAST_LOAD_FAST': 113, + 'STORE_FAST_STORE_FAST': 114, + 'STORE_GLOBAL': 115, + 'STORE_NAME': 116, + 'SWAP': 117, + 'UNPACK_EX': 118, + 'UNPACK_SEQUENCE': 119, + 'YIELD_VALUE': 120, 'INSTRUMENTED_END_FOR': 234, 'INSTRUMENTED_POP_ITER': 235, 'INSTRUMENTED_END_SEND': 236, diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index 1794032e2b1315..dbeedb7ffe0ce6 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -2,14 +2,14 @@ unsigned char M_test_frozenmain[] = { 227,0,0,0,0,0,0,0,0,0,0,0,0,9,0,0, 0,0,0,0,0,243,184,0,0,0,128,0,94,0,82,1, - 73,0,117,0,94,0,82,1,73,1,117,1,93,2,33,0, + 73,0,116,0,94,0,82,1,73,1,116,1,93,2,33,0, 82,2,52,1,0,0,0,0,0,0,31,0,93,2,33,0, 82,3,93,0,80,6,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,52,2,0,0,0,0,0,0, 31,0,93,1,80,8,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,33,0,52,0,0,0,0,0, 0,0,82,4,44,26,0,0,0,0,0,0,0,0,0,0, - 117,5,82,7,16,0,70,24,0,0,117,6,93,2,33,0, + 116,5,82,7,16,0,70,24,0,0,116,6,93,2,33,0, 82,5,93,6,12,0,82,6,93,5,93,6,44,26,0,0, 0,0,0,0,0,0,0,0,12,0,50,4,52,1,0,0, 0,0,0,0,31,0,75,26,0,0,9,0,30,0,82,1, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index cd098ebe0eb2b6..0328cd3ef9693b 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5193,40 +5193,6 @@ dummy_func( Py_FatalError("Executing RESERVED instruction."); } - tier1 no_save_ip inst(RECORD_PREVIOUS_INST, (--)) { -#if _Py_TIER2 - assert(IS_JIT_TRACING()); - int opcode = next_instr->op.code; - int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr); - if (full) { - LEAVE_TRACING(); - int err = bail_tracing_and_jit(tstate, frame); - ERROR_IF(err < 0); - DISPATCH_GOTO_NON_TRACING(); - } - // Super instructions. Instruction deopted. There's a mismatch in what the stack expects - // in the optimizer. So we have to reflect in the trace correctly. - if ((tstate->interp->jit_state.prev_instr->op.code == CALL_LIST_APPEND && - opcode == POP_TOP) || - (tstate->interp->jit_state.prev_instr->op.code == BINARY_OP_INPLACE_ADD_UNICODE && - opcode == STORE_FAST)) { - tstate->interp->jit_state.prev_instr_is_super = true; - } - else { - tstate->interp->jit_state.prev_instr = next_instr; - } - tstate->interp->jit_state.specialize_counter = 0; - PyCodeObject *prev_code = (PyCodeObject *)Py_NewRef(_PyFrame_GetCode(frame)); - Py_SETREF(tstate->interp->jit_state.prev_instr_code, prev_code); - - tstate->interp->jit_state.prev_instr_frame = frame; - tstate->interp->jit_state.prev_instr_oparg = oparg; - tstate->interp->jit_state.prev_instr_stacklevel = STACK_LEVEL(); - DISPATCH_GOTO_NON_TRACING(); -#else - Py_FatalError("JIT instruction executed in non-jit build."); -#endif - } ///////// Tier-2 only opcodes ///////// op (_GUARD_IS_TRUE_POP, (flag -- )) { @@ -5646,6 +5612,40 @@ dummy_func( DISPATCH(); } + label(record_previous_inst) { +#if _Py_TIER2 + assert(IS_JIT_TRACING()); + int opcode = next_instr->op.code; + int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr); + if (full) { + LEAVE_TRACING(); + int err = bail_tracing_and_jit(tstate, frame); + ERROR_IF(err < 0); + DISPATCH_GOTO_NON_TRACING(); + } + // Super instructions. Instruction deopted. There's a mismatch in what the stack expects + // in the optimizer. So we have to reflect in the trace correctly. + if ((tstate->interp->jit_state.prev_instr->op.code == CALL_LIST_APPEND && + opcode == POP_TOP) || + (tstate->interp->jit_state.prev_instr->op.code == BINARY_OP_INPLACE_ADD_UNICODE && + opcode == STORE_FAST)) { + tstate->interp->jit_state.prev_instr_is_super = true; + } + else { + tstate->interp->jit_state.prev_instr = next_instr; + } + tstate->interp->jit_state.specialize_counter = 0; + PyCodeObject *prev_code = (PyCodeObject *)Py_NewRef(_PyFrame_GetCode(frame)); + Py_SETREF(tstate->interp->jit_state.prev_instr_code, prev_code); + + tstate->interp->jit_state.prev_instr_frame = frame; + tstate->interp->jit_state.prev_instr_oparg = oparg; + tstate->interp->jit_state.prev_instr_stacklevel = STACK_LEVEL(); + DISPATCH_GOTO_NON_TRACING(); +#else + Py_FatalError("JIT label executed in non-jit build."); +#endif + } // END BYTECODES // diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 8a81a6cdcc4929..d9fea0e4ae5767 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -10218,48 +10218,6 @@ JUMP_TO_LABEL(error); } - - TARGET(RECORD_PREVIOUS_INST) { - INSTRUCTION_STATS(RECORD_PREVIOUS_INST); - #if _Py_TIER2 - assert(IS_JIT_TRACING()); - int opcode = next_instr->op.code; - _PyFrame_SetStackPointer(frame, stack_pointer); - int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (full) { - LEAVE_TRACING(); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = bail_tracing_and_jit(tstate, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - JUMP_TO_LABEL(error); - } - DISPATCH_GOTO_NON_TRACING(); - } - if ((tstate->interp->jit_state.prev_instr->op.code == CALL_LIST_APPEND && - opcode == POP_TOP) || - (tstate->interp->jit_state.prev_instr->op.code == BINARY_OP_INPLACE_ADD_UNICODE && - opcode == STORE_FAST)) { - tstate->interp->jit_state.prev_instr_is_super = true; - } - else { - tstate->interp->jit_state.prev_instr = next_instr; - } - tstate->interp->jit_state.specialize_counter = 0; - PyCodeObject *prev_code = (PyCodeObject *)Py_NewRef(_PyFrame_GetCode(frame)); - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_SETREF(tstate->interp->jit_state.prev_instr_code, prev_code); - stack_pointer = _PyFrame_GetStackPointer(frame); - tstate->interp->jit_state.prev_instr_frame = frame; - tstate->interp->jit_state.prev_instr_oparg = oparg; - tstate->interp->jit_state.prev_instr_stacklevel = STACK_LEVEL(); - DISPATCH_GOTO_NON_TRACING(); - #else - Py_FatalError("JIT instruction executed in non-jit build."); - #endif - } - TARGET(RERAISE) { #if _Py_TAIL_CALL_INTERP int opcode = RERAISE; @@ -12355,5 +12313,46 @@ JUMP_TO_LABEL(error); DISPATCH(); } + LABEL(record_previous_inst) + { + #if _Py_TIER2 + assert(IS_JIT_TRACING()); + int opcode = next_instr->op.code; + _PyFrame_SetStackPointer(frame, stack_pointer); + int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (full) { + LEAVE_TRACING(); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = bail_tracing_and_jit(tstate, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + DISPATCH_GOTO_NON_TRACING(); + } + if ((tstate->interp->jit_state.prev_instr->op.code == CALL_LIST_APPEND && + opcode == POP_TOP) || + (tstate->interp->jit_state.prev_instr->op.code == BINARY_OP_INPLACE_ADD_UNICODE && + opcode == STORE_FAST)) { + tstate->interp->jit_state.prev_instr_is_super = true; + } + else { + tstate->interp->jit_state.prev_instr = next_instr; + } + tstate->interp->jit_state.specialize_counter = 0; + PyCodeObject *prev_code = (PyCodeObject *)Py_NewRef(_PyFrame_GetCode(frame)); + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_SETREF(tstate->interp->jit_state.prev_instr_code, prev_code); + stack_pointer = _PyFrame_GetStackPointer(frame); + tstate->interp->jit_state.prev_instr_frame = frame; + tstate->interp->jit_state.prev_instr_oparg = oparg; + tstate->interp->jit_state.prev_instr_stacklevel = STACK_LEVEL(); + DISPATCH_GOTO_NON_TRACING(); + #else + Py_FatalError("JIT label executed in non-jit build."); + #endif + } + /* END LABELS */ #undef TIER_ONE diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 0d28b3d4e69055..eb43bf63804a32 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -105,7 +105,6 @@ static void *opcode_targets_table[256] = { &&TARGET_POP_JUMP_IF_NOT_NONE, &&TARGET_POP_JUMP_IF_TRUE, &&TARGET_RAISE_VARARGS, - &&TARGET_RECORD_PREVIOUS_INST, &&TARGET_RERAISE, &&TARGET_SEND, &&TARGET_SET_ADD, @@ -128,6 +127,7 @@ static void *opcode_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, + &&_unknown_opcode, &&TARGET_RESUME, &&TARGET_BINARY_OP_ADD_FLOAT, &&TARGET_BINARY_OP_ADD_INT, @@ -259,216 +259,216 @@ static void *opcode_targets_table[256] = { }; #if _Py_TIER2 static void *opcode_tracing_targets_table[256] = { - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -493,28 +493,28 @@ static void *opcode_tracing_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, - &&TARGET_RECORD_PREVIOUS_INST, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, + &&TARGET_record_previous_inst, }; #endif #else /* _Py_TAIL_CALL_INTERP */ @@ -528,6 +528,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_error(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exception_unwind(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_start_frame(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_record_previous_inst(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS); @@ -710,7 +711,6 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_POP_TOP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_EXC_INFO(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_PUSH_NULL(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RAISE_VARARGS(TAIL_CALL_PARAMS); -Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RECORD_PREVIOUS_INST(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RERAISE(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESERVED(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_RESUME(TAIL_CALL_PARAMS); @@ -948,7 +948,6 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [PUSH_EXC_INFO] = _TAIL_CALL_PUSH_EXC_INFO, [PUSH_NULL] = _TAIL_CALL_PUSH_NULL, [RAISE_VARARGS] = _TAIL_CALL_RAISE_VARARGS, - [RECORD_PREVIOUS_INST] = _TAIL_CALL_RECORD_PREVIOUS_INST, [RERAISE] = _TAIL_CALL_RERAISE, [RESERVED] = _TAIL_CALL_RESERVED, [RESUME] = _TAIL_CALL_RESUME, @@ -993,6 +992,7 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_UNPACK_SEQUENCE_TWO_TUPLE, [WITH_EXCEPT_START] = _TAIL_CALL_WITH_EXCEPT_START, [YIELD_VALUE] = _TAIL_CALL_YIELD_VALUE, + [121] = _TAIL_CALL_UNKNOWN_OPCODE, [122] = _TAIL_CALL_UNKNOWN_OPCODE, [123] = _TAIL_CALL_UNKNOWN_OPCODE, [124] = _TAIL_CALL_UNKNOWN_OPCODE, @@ -1025,232 +1025,232 @@ static py_tail_call_funcptr instruction_funcptr_handler_table[256] = { [233] = _TAIL_CALL_UNKNOWN_OPCODE, }; static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = { - [BINARY_OP] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [BINARY_OP_ADD_INT] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [BINARY_OP_ADD_UNICODE] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [BINARY_OP_EXTEND] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [BINARY_OP_INPLACE_ADD_UNICODE] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [BINARY_OP_MULTIPLY_FLOAT] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [BINARY_OP_MULTIPLY_INT] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [BINARY_OP_SUBSCR_DICT] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [BINARY_OP_SUBSCR_GETITEM] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [BINARY_OP_SUBSCR_LIST_INT] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [BINARY_OP_SUBSCR_LIST_SLICE] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [BINARY_OP_SUBSCR_STR_INT] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [BINARY_OP_SUBSCR_TUPLE_INT] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [BINARY_OP_SUBTRACT_FLOAT] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [BINARY_OP_SUBTRACT_INT] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [BINARY_SLICE] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [BUILD_INTERPOLATION] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [BUILD_LIST] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [BUILD_MAP] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [BUILD_SET] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [BUILD_SLICE] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [BUILD_STRING] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [BUILD_TEMPLATE] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [BUILD_TUPLE] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [CACHE] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [CALL] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [CALL_ALLOC_AND_ENTER_INIT] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [CALL_BUILTIN_CLASS] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [CALL_BUILTIN_FAST] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [CALL_BUILTIN_O] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [CALL_FUNCTION_EX] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [CALL_INTRINSIC_1] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [CALL_INTRINSIC_2] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [CALL_ISINSTANCE] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [CALL_KW] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [CALL_KW_BOUND_METHOD] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [CALL_KW_NON_PY] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [CALL_KW_PY] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [CALL_LEN] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [CALL_LIST_APPEND] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [CALL_METHOD_DESCRIPTOR_FAST] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [CALL_METHOD_DESCRIPTOR_NOARGS] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [CALL_METHOD_DESCRIPTOR_O] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [CALL_NON_PY_GENERAL] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [CALL_PY_EXACT_ARGS] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [CALL_PY_GENERAL] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [CALL_STR_1] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [CALL_TUPLE_1] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [CALL_TYPE_1] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [CHECK_EG_MATCH] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [CHECK_EXC_MATCH] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [CLEANUP_THROW] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [COMPARE_OP] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [COMPARE_OP_FLOAT] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [COMPARE_OP_INT] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [COMPARE_OP_STR] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [CONTAINS_OP] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [CONTAINS_OP_DICT] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [CONTAINS_OP_SET] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [CONVERT_VALUE] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [COPY] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [COPY_FREE_VARS] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [DELETE_ATTR] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [DELETE_DEREF] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [DELETE_FAST] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [DELETE_GLOBAL] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [DELETE_NAME] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [DELETE_SUBSCR] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [DICT_MERGE] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [DICT_UPDATE] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [END_ASYNC_FOR] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [END_FOR] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [END_SEND] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [ENTER_EXECUTOR] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [EXIT_INIT_CHECK] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [EXTENDED_ARG] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [FORMAT_SIMPLE] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [FORMAT_WITH_SPEC] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [FOR_ITER] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [FOR_ITER_GEN] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [FOR_ITER_LIST] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [FOR_ITER_RANGE] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [FOR_ITER_TUPLE] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [GET_AITER] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [GET_ANEXT] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [GET_AWAITABLE] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [GET_ITER] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [GET_LEN] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [GET_YIELD_FROM_ITER] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [IMPORT_FROM] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [IMPORT_NAME] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [INSTRUMENTED_CALL] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [INSTRUMENTED_CALL_FUNCTION_EX] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [INSTRUMENTED_CALL_KW] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [INSTRUMENTED_END_ASYNC_FOR] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [INSTRUMENTED_END_FOR] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [INSTRUMENTED_END_SEND] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [INSTRUMENTED_FOR_ITER] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [INSTRUMENTED_INSTRUCTION] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [INSTRUMENTED_JUMP_BACKWARD] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [INSTRUMENTED_JUMP_FORWARD] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [INSTRUMENTED_LINE] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [INSTRUMENTED_LOAD_SUPER_ATTR] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [INSTRUMENTED_NOT_TAKEN] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [INSTRUMENTED_POP_ITER] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [INSTRUMENTED_POP_JUMP_IF_FALSE] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [INSTRUMENTED_POP_JUMP_IF_NONE] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [INSTRUMENTED_POP_JUMP_IF_TRUE] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [INSTRUMENTED_RESUME] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [INSTRUMENTED_RETURN_VALUE] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [INSTRUMENTED_YIELD_VALUE] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [INTERPRETER_EXIT] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [IS_OP] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [JUMP_BACKWARD] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [JUMP_BACKWARD_JIT] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [JUMP_BACKWARD_NO_INTERRUPT] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [JUMP_BACKWARD_NO_JIT] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [JUMP_FORWARD] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [LIST_APPEND] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [LIST_EXTEND] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [LOAD_ATTR] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [LOAD_ATTR_CLASS] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [LOAD_ATTR_INSTANCE_VALUE] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [LOAD_ATTR_METHOD_LAZY_DICT] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [LOAD_ATTR_METHOD_NO_DICT] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [LOAD_ATTR_METHOD_WITH_VALUES] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [LOAD_ATTR_MODULE] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [LOAD_ATTR_PROPERTY] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [LOAD_ATTR_SLOT] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [LOAD_ATTR_WITH_HINT] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [LOAD_BUILD_CLASS] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [LOAD_COMMON_CONSTANT] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [LOAD_CONST] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [LOAD_DEREF] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [LOAD_FAST] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [LOAD_FAST_AND_CLEAR] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [LOAD_FAST_BORROW] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [LOAD_FAST_BORROW_LOAD_FAST_BORROW] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [LOAD_FAST_CHECK] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [LOAD_FAST_LOAD_FAST] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [LOAD_FROM_DICT_OR_DEREF] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [LOAD_FROM_DICT_OR_GLOBALS] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [LOAD_GLOBAL] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [LOAD_GLOBAL_BUILTIN] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [LOAD_GLOBAL_MODULE] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [LOAD_LOCALS] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [LOAD_NAME] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [LOAD_SMALL_INT] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [LOAD_SPECIAL] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [LOAD_SUPER_ATTR] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [LOAD_SUPER_ATTR_ATTR] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [LOAD_SUPER_ATTR_METHOD] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [MAKE_CELL] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [MAKE_FUNCTION] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [MAP_ADD] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [MATCH_CLASS] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [MATCH_KEYS] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [MATCH_MAPPING] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [MATCH_SEQUENCE] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [NOP] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [NOT_TAKEN] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [POP_EXCEPT] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [POP_ITER] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [POP_JUMP_IF_FALSE] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [POP_JUMP_IF_NONE] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [POP_JUMP_IF_TRUE] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [POP_TOP] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [PUSH_EXC_INFO] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [PUSH_NULL] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [RAISE_VARARGS] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [RECORD_PREVIOUS_INST] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [RERAISE] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [RESERVED] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [RESUME] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [RESUME_CHECK] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [RETURN_GENERATOR] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [RETURN_VALUE] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [SEND] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [SEND_GEN] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [SETUP_ANNOTATIONS] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [SET_ADD] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [SET_FUNCTION_ATTRIBUTE] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [SET_UPDATE] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [STORE_ATTR] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [STORE_ATTR_INSTANCE_VALUE] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [STORE_ATTR_SLOT] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [STORE_ATTR_WITH_HINT] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [STORE_DEREF] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [STORE_FAST] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [STORE_FAST_LOAD_FAST] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [STORE_FAST_STORE_FAST] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [STORE_GLOBAL] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [STORE_NAME] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [STORE_SLICE] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [STORE_SUBSCR] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [STORE_SUBSCR_DICT] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [SWAP] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [TO_BOOL] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [TO_BOOL_BOOL] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [TO_BOOL_INT] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [TO_BOOL_LIST] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [TO_BOOL_NONE] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [TO_BOOL_STR] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [UNARY_INVERT] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [UNARY_NEGATIVE] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [UNARY_NOT] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [UNPACK_EX] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [UNPACK_SEQUENCE] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [UNPACK_SEQUENCE_LIST] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [UNPACK_SEQUENCE_TUPLE] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [WITH_EXCEPT_START] = _TAIL_CALL_RECORD_PREVIOUS_INST, - [YIELD_VALUE] = _TAIL_CALL_RECORD_PREVIOUS_INST, + [BINARY_OP] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_ADD_FLOAT] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_ADD_INT] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_ADD_UNICODE] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_EXTEND] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_INPLACE_ADD_UNICODE] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_MULTIPLY_FLOAT] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_MULTIPLY_INT] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_SUBSCR_DICT] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_SUBSCR_GETITEM] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_SUBSCR_LIST_INT] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_SUBSCR_LIST_SLICE] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_SUBSCR_STR_INT] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_SUBSCR_TUPLE_INT] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_SUBTRACT_FLOAT] = _TAIL_CALL_record_previous_inst, + [BINARY_OP_SUBTRACT_INT] = _TAIL_CALL_record_previous_inst, + [BINARY_SLICE] = _TAIL_CALL_record_previous_inst, + [BUILD_INTERPOLATION] = _TAIL_CALL_record_previous_inst, + [BUILD_LIST] = _TAIL_CALL_record_previous_inst, + [BUILD_MAP] = _TAIL_CALL_record_previous_inst, + [BUILD_SET] = _TAIL_CALL_record_previous_inst, + [BUILD_SLICE] = _TAIL_CALL_record_previous_inst, + [BUILD_STRING] = _TAIL_CALL_record_previous_inst, + [BUILD_TEMPLATE] = _TAIL_CALL_record_previous_inst, + [BUILD_TUPLE] = _TAIL_CALL_record_previous_inst, + [CACHE] = _TAIL_CALL_record_previous_inst, + [CALL] = _TAIL_CALL_record_previous_inst, + [CALL_ALLOC_AND_ENTER_INIT] = _TAIL_CALL_record_previous_inst, + [CALL_BOUND_METHOD_EXACT_ARGS] = _TAIL_CALL_record_previous_inst, + [CALL_BOUND_METHOD_GENERAL] = _TAIL_CALL_record_previous_inst, + [CALL_BUILTIN_CLASS] = _TAIL_CALL_record_previous_inst, + [CALL_BUILTIN_FAST] = _TAIL_CALL_record_previous_inst, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = _TAIL_CALL_record_previous_inst, + [CALL_BUILTIN_O] = _TAIL_CALL_record_previous_inst, + [CALL_FUNCTION_EX] = _TAIL_CALL_record_previous_inst, + [CALL_INTRINSIC_1] = _TAIL_CALL_record_previous_inst, + [CALL_INTRINSIC_2] = _TAIL_CALL_record_previous_inst, + [CALL_ISINSTANCE] = _TAIL_CALL_record_previous_inst, + [CALL_KW] = _TAIL_CALL_record_previous_inst, + [CALL_KW_BOUND_METHOD] = _TAIL_CALL_record_previous_inst, + [CALL_KW_NON_PY] = _TAIL_CALL_record_previous_inst, + [CALL_KW_PY] = _TAIL_CALL_record_previous_inst, + [CALL_LEN] = _TAIL_CALL_record_previous_inst, + [CALL_LIST_APPEND] = _TAIL_CALL_record_previous_inst, + [CALL_METHOD_DESCRIPTOR_FAST] = _TAIL_CALL_record_previous_inst, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = _TAIL_CALL_record_previous_inst, + [CALL_METHOD_DESCRIPTOR_NOARGS] = _TAIL_CALL_record_previous_inst, + [CALL_METHOD_DESCRIPTOR_O] = _TAIL_CALL_record_previous_inst, + [CALL_NON_PY_GENERAL] = _TAIL_CALL_record_previous_inst, + [CALL_PY_EXACT_ARGS] = _TAIL_CALL_record_previous_inst, + [CALL_PY_GENERAL] = _TAIL_CALL_record_previous_inst, + [CALL_STR_1] = _TAIL_CALL_record_previous_inst, + [CALL_TUPLE_1] = _TAIL_CALL_record_previous_inst, + [CALL_TYPE_1] = _TAIL_CALL_record_previous_inst, + [CHECK_EG_MATCH] = _TAIL_CALL_record_previous_inst, + [CHECK_EXC_MATCH] = _TAIL_CALL_record_previous_inst, + [CLEANUP_THROW] = _TAIL_CALL_record_previous_inst, + [COMPARE_OP] = _TAIL_CALL_record_previous_inst, + [COMPARE_OP_FLOAT] = _TAIL_CALL_record_previous_inst, + [COMPARE_OP_INT] = _TAIL_CALL_record_previous_inst, + [COMPARE_OP_STR] = _TAIL_CALL_record_previous_inst, + [CONTAINS_OP] = _TAIL_CALL_record_previous_inst, + [CONTAINS_OP_DICT] = _TAIL_CALL_record_previous_inst, + [CONTAINS_OP_SET] = _TAIL_CALL_record_previous_inst, + [CONVERT_VALUE] = _TAIL_CALL_record_previous_inst, + [COPY] = _TAIL_CALL_record_previous_inst, + [COPY_FREE_VARS] = _TAIL_CALL_record_previous_inst, + [DELETE_ATTR] = _TAIL_CALL_record_previous_inst, + [DELETE_DEREF] = _TAIL_CALL_record_previous_inst, + [DELETE_FAST] = _TAIL_CALL_record_previous_inst, + [DELETE_GLOBAL] = _TAIL_CALL_record_previous_inst, + [DELETE_NAME] = _TAIL_CALL_record_previous_inst, + [DELETE_SUBSCR] = _TAIL_CALL_record_previous_inst, + [DICT_MERGE] = _TAIL_CALL_record_previous_inst, + [DICT_UPDATE] = _TAIL_CALL_record_previous_inst, + [END_ASYNC_FOR] = _TAIL_CALL_record_previous_inst, + [END_FOR] = _TAIL_CALL_record_previous_inst, + [END_SEND] = _TAIL_CALL_record_previous_inst, + [ENTER_EXECUTOR] = _TAIL_CALL_record_previous_inst, + [EXIT_INIT_CHECK] = _TAIL_CALL_record_previous_inst, + [EXTENDED_ARG] = _TAIL_CALL_record_previous_inst, + [FORMAT_SIMPLE] = _TAIL_CALL_record_previous_inst, + [FORMAT_WITH_SPEC] = _TAIL_CALL_record_previous_inst, + [FOR_ITER] = _TAIL_CALL_record_previous_inst, + [FOR_ITER_GEN] = _TAIL_CALL_record_previous_inst, + [FOR_ITER_LIST] = _TAIL_CALL_record_previous_inst, + [FOR_ITER_RANGE] = _TAIL_CALL_record_previous_inst, + [FOR_ITER_TUPLE] = _TAIL_CALL_record_previous_inst, + [GET_AITER] = _TAIL_CALL_record_previous_inst, + [GET_ANEXT] = _TAIL_CALL_record_previous_inst, + [GET_AWAITABLE] = _TAIL_CALL_record_previous_inst, + [GET_ITER] = _TAIL_CALL_record_previous_inst, + [GET_LEN] = _TAIL_CALL_record_previous_inst, + [GET_YIELD_FROM_ITER] = _TAIL_CALL_record_previous_inst, + [IMPORT_FROM] = _TAIL_CALL_record_previous_inst, + [IMPORT_NAME] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_CALL] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_CALL_FUNCTION_EX] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_CALL_KW] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_END_ASYNC_FOR] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_END_FOR] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_END_SEND] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_FOR_ITER] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_INSTRUCTION] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_JUMP_BACKWARD] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_JUMP_FORWARD] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_LINE] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_LOAD_SUPER_ATTR] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_NOT_TAKEN] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_POP_ITER] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_POP_JUMP_IF_NONE] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_RESUME] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_RETURN_VALUE] = _TAIL_CALL_record_previous_inst, + [INSTRUMENTED_YIELD_VALUE] = _TAIL_CALL_record_previous_inst, + [INTERPRETER_EXIT] = _TAIL_CALL_record_previous_inst, + [IS_OP] = _TAIL_CALL_record_previous_inst, + [JUMP_BACKWARD] = _TAIL_CALL_record_previous_inst, + [JUMP_BACKWARD_JIT] = _TAIL_CALL_record_previous_inst, + [JUMP_BACKWARD_NO_INTERRUPT] = _TAIL_CALL_record_previous_inst, + [JUMP_BACKWARD_NO_JIT] = _TAIL_CALL_record_previous_inst, + [JUMP_FORWARD] = _TAIL_CALL_record_previous_inst, + [LIST_APPEND] = _TAIL_CALL_record_previous_inst, + [LIST_EXTEND] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_CLASS] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_INSTANCE_VALUE] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_METHOD_LAZY_DICT] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_METHOD_NO_DICT] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_METHOD_WITH_VALUES] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_MODULE] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_PROPERTY] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_SLOT] = _TAIL_CALL_record_previous_inst, + [LOAD_ATTR_WITH_HINT] = _TAIL_CALL_record_previous_inst, + [LOAD_BUILD_CLASS] = _TAIL_CALL_record_previous_inst, + [LOAD_COMMON_CONSTANT] = _TAIL_CALL_record_previous_inst, + [LOAD_CONST] = _TAIL_CALL_record_previous_inst, + [LOAD_DEREF] = _TAIL_CALL_record_previous_inst, + [LOAD_FAST] = _TAIL_CALL_record_previous_inst, + [LOAD_FAST_AND_CLEAR] = _TAIL_CALL_record_previous_inst, + [LOAD_FAST_BORROW] = _TAIL_CALL_record_previous_inst, + [LOAD_FAST_BORROW_LOAD_FAST_BORROW] = _TAIL_CALL_record_previous_inst, + [LOAD_FAST_CHECK] = _TAIL_CALL_record_previous_inst, + [LOAD_FAST_LOAD_FAST] = _TAIL_CALL_record_previous_inst, + [LOAD_FROM_DICT_OR_DEREF] = _TAIL_CALL_record_previous_inst, + [LOAD_FROM_DICT_OR_GLOBALS] = _TAIL_CALL_record_previous_inst, + [LOAD_GLOBAL] = _TAIL_CALL_record_previous_inst, + [LOAD_GLOBAL_BUILTIN] = _TAIL_CALL_record_previous_inst, + [LOAD_GLOBAL_MODULE] = _TAIL_CALL_record_previous_inst, + [LOAD_LOCALS] = _TAIL_CALL_record_previous_inst, + [LOAD_NAME] = _TAIL_CALL_record_previous_inst, + [LOAD_SMALL_INT] = _TAIL_CALL_record_previous_inst, + [LOAD_SPECIAL] = _TAIL_CALL_record_previous_inst, + [LOAD_SUPER_ATTR] = _TAIL_CALL_record_previous_inst, + [LOAD_SUPER_ATTR_ATTR] = _TAIL_CALL_record_previous_inst, + [LOAD_SUPER_ATTR_METHOD] = _TAIL_CALL_record_previous_inst, + [MAKE_CELL] = _TAIL_CALL_record_previous_inst, + [MAKE_FUNCTION] = _TAIL_CALL_record_previous_inst, + [MAP_ADD] = _TAIL_CALL_record_previous_inst, + [MATCH_CLASS] = _TAIL_CALL_record_previous_inst, + [MATCH_KEYS] = _TAIL_CALL_record_previous_inst, + [MATCH_MAPPING] = _TAIL_CALL_record_previous_inst, + [MATCH_SEQUENCE] = _TAIL_CALL_record_previous_inst, + [NOP] = _TAIL_CALL_record_previous_inst, + [NOT_TAKEN] = _TAIL_CALL_record_previous_inst, + [POP_EXCEPT] = _TAIL_CALL_record_previous_inst, + [POP_ITER] = _TAIL_CALL_record_previous_inst, + [POP_JUMP_IF_FALSE] = _TAIL_CALL_record_previous_inst, + [POP_JUMP_IF_NONE] = _TAIL_CALL_record_previous_inst, + [POP_JUMP_IF_NOT_NONE] = _TAIL_CALL_record_previous_inst, + [POP_JUMP_IF_TRUE] = _TAIL_CALL_record_previous_inst, + [POP_TOP] = _TAIL_CALL_record_previous_inst, + [PUSH_EXC_INFO] = _TAIL_CALL_record_previous_inst, + [PUSH_NULL] = _TAIL_CALL_record_previous_inst, + [RAISE_VARARGS] = _TAIL_CALL_record_previous_inst, + [RERAISE] = _TAIL_CALL_record_previous_inst, + [RESERVED] = _TAIL_CALL_record_previous_inst, + [RESUME] = _TAIL_CALL_record_previous_inst, + [RESUME_CHECK] = _TAIL_CALL_record_previous_inst, + [RETURN_GENERATOR] = _TAIL_CALL_record_previous_inst, + [RETURN_VALUE] = _TAIL_CALL_record_previous_inst, + [SEND] = _TAIL_CALL_record_previous_inst, + [SEND_GEN] = _TAIL_CALL_record_previous_inst, + [SETUP_ANNOTATIONS] = _TAIL_CALL_record_previous_inst, + [SET_ADD] = _TAIL_CALL_record_previous_inst, + [SET_FUNCTION_ATTRIBUTE] = _TAIL_CALL_record_previous_inst, + [SET_UPDATE] = _TAIL_CALL_record_previous_inst, + [STORE_ATTR] = _TAIL_CALL_record_previous_inst, + [STORE_ATTR_INSTANCE_VALUE] = _TAIL_CALL_record_previous_inst, + [STORE_ATTR_SLOT] = _TAIL_CALL_record_previous_inst, + [STORE_ATTR_WITH_HINT] = _TAIL_CALL_record_previous_inst, + [STORE_DEREF] = _TAIL_CALL_record_previous_inst, + [STORE_FAST] = _TAIL_CALL_record_previous_inst, + [STORE_FAST_LOAD_FAST] = _TAIL_CALL_record_previous_inst, + [STORE_FAST_STORE_FAST] = _TAIL_CALL_record_previous_inst, + [STORE_GLOBAL] = _TAIL_CALL_record_previous_inst, + [STORE_NAME] = _TAIL_CALL_record_previous_inst, + [STORE_SLICE] = _TAIL_CALL_record_previous_inst, + [STORE_SUBSCR] = _TAIL_CALL_record_previous_inst, + [STORE_SUBSCR_DICT] = _TAIL_CALL_record_previous_inst, + [STORE_SUBSCR_LIST_INT] = _TAIL_CALL_record_previous_inst, + [SWAP] = _TAIL_CALL_record_previous_inst, + [TO_BOOL] = _TAIL_CALL_record_previous_inst, + [TO_BOOL_ALWAYS_TRUE] = _TAIL_CALL_record_previous_inst, + [TO_BOOL_BOOL] = _TAIL_CALL_record_previous_inst, + [TO_BOOL_INT] = _TAIL_CALL_record_previous_inst, + [TO_BOOL_LIST] = _TAIL_CALL_record_previous_inst, + [TO_BOOL_NONE] = _TAIL_CALL_record_previous_inst, + [TO_BOOL_STR] = _TAIL_CALL_record_previous_inst, + [UNARY_INVERT] = _TAIL_CALL_record_previous_inst, + [UNARY_NEGATIVE] = _TAIL_CALL_record_previous_inst, + [UNARY_NOT] = _TAIL_CALL_record_previous_inst, + [UNPACK_EX] = _TAIL_CALL_record_previous_inst, + [UNPACK_SEQUENCE] = _TAIL_CALL_record_previous_inst, + [UNPACK_SEQUENCE_LIST] = _TAIL_CALL_record_previous_inst, + [UNPACK_SEQUENCE_TUPLE] = _TAIL_CALL_record_previous_inst, + [UNPACK_SEQUENCE_TWO_TUPLE] = _TAIL_CALL_record_previous_inst, + [WITH_EXCEPT_START] = _TAIL_CALL_record_previous_inst, + [YIELD_VALUE] = _TAIL_CALL_record_previous_inst, + [121] = _TAIL_CALL_UNKNOWN_OPCODE, [122] = _TAIL_CALL_UNKNOWN_OPCODE, [123] = _TAIL_CALL_UNKNOWN_OPCODE, [124] = _TAIL_CALL_UNKNOWN_OPCODE, diff --git a/Tools/cases_generator/target_generator.py b/Tools/cases_generator/target_generator.py index 209411ca6f50d9..6763e487fbdc92 100644 --- a/Tools/cases_generator/target_generator.py +++ b/Tools/cases_generator/target_generator.py @@ -34,7 +34,7 @@ def write_opcode_targets(analysis: Analysis, out: CWriter) -> None: targets = ["&&_unknown_opcode,\n"] * 256 for name, op in analysis.opmap.items(): if op < 256: - targets[op] = f"&&TARGET_RECORD_PREVIOUS_INST,\n" + targets[op] = f"&&TARGET_record_previous_inst,\n" out.emit("#if _Py_TIER2\n") out.emit("static void *opcode_tracing_targets_table[256] = {\n") for target in targets: @@ -84,7 +84,7 @@ def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: # Emit the tracing dispatch table. out.emit("static py_tail_call_funcptr instruction_funcptr_tracing_table[256] = {\n") for name in sorted(analysis.instructions.keys()): - out.emit(f"[{name}] = _TAIL_CALL_RECORD_PREVIOUS_INST,\n") + out.emit(f"[{name}] = _TAIL_CALL_record_previous_inst,\n") named_values = analysis.opmap.values() for rest in range(256): if rest not in named_values: diff --git a/Tools/cases_generator/tier1_generator.py b/Tools/cases_generator/tier1_generator.py index fb1874b1658cf9..94ffb0118f0786 100644 --- a/Tools/cases_generator/tier1_generator.py +++ b/Tools/cases_generator/tier1_generator.py @@ -167,9 +167,7 @@ def generate_tier1( {INSTRUCTION_START_MARKER} """ ) - out = CWriter(outfile, 2, lines) - emitter = Emitter(out, analysis.labels) - generate_tier1_cases(analysis, out, emitter) + generate_tier1_cases(analysis, outfile, lines) outfile.write(f""" {INSTRUCTION_END_MARKER} #if !_Py_TAIL_CALL_INTERP @@ -216,35 +214,14 @@ def get_popped(inst: Instruction, analysis: Analysis) -> str: stack = get_stack_effect(inst) return (-stack.base_offset).to_c() - -def generate_record_previous_inst(out: CWriter, emitter: Emitter, inst: Instruction) -> None: - out.emit("\n") - out.emit(f"TARGET(RECORD_PREVIOUS_INST) {{\n") - out.emit(f"INSTRUCTION_STATS(RECORD_PREVIOUS_INST);\n") - declare_variables(inst, out) - offset = 1 # The instruction itself - stack = Stack() - for part in inst.parts: - # Only emit braces if more than one uop - insert_braces = len([p for p in inst.parts if isinstance(p, Uop)]) > 1 - reachable, offset, stack = write_uop(part, emitter, offset, stack, inst, insert_braces) - out.start_line() - if reachable: # type: ignore[possibly-undefined] - stack.flush(out) - out.emit(f"DISPATCH();\n") - out.start_line() - out.emit("}") - out.emit("\n") - def generate_tier1_cases( - analysis: Analysis, out: CWriter, emitter: Emitter + analysis: Analysis, outfile: TextIO, lines: bool ) -> None: + out = CWriter(outfile, 2, lines) + emitter = Emitter(out, analysis.labels) out.emit("\n") for name, inst in sorted(analysis.instructions.items()): out.emit("\n") - if name == "RECORD_PREVIOUS_INST": - generate_record_previous_inst(out, emitter, inst) - continue out.emit(f"TARGET({name}) {{\n") popped = get_popped(inst, analysis) # We need to ifdef it because this breaks platforms From d6012561a32ac4ef15d4f0e24b7eaa815545e4f8 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 27 Oct 2025 09:57:14 +0000 Subject: [PATCH 119/190] fix some formatting --- Tools/cases_generator/target_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/cases_generator/target_generator.py b/Tools/cases_generator/target_generator.py index 6763e487fbdc92..b945c3e7203314 100644 --- a/Tools/cases_generator/target_generator.py +++ b/Tools/cases_generator/target_generator.py @@ -58,7 +58,7 @@ def write_tailcall_dispatch_table(analysis: Analysis, out: CWriter) -> None: out.emit(f"{function_proto(name)};\n") out.emit("\n") - # Emit function prototypes for opcode handlers. + # Emit function prototypes for opcode handlers. for name in sorted(analysis.instructions.keys()): out.emit(f"{function_proto(name)};\n") out.emit("\n") From 425fd519a464496c2e375411ce9509173068dd13 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 27 Oct 2025 10:03:47 +0000 Subject: [PATCH 120/190] fix cg builds, invalidate executors on function deallocation --- Objects/funcobject.c | 1 + Python/opcode_targets.h | 450 +++++++++++----------- Tools/cases_generator/target_generator.py | 2 +- 3 files changed, 227 insertions(+), 226 deletions(-) diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 2311d8ce29a9d9..b659ac8023373b 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -1152,6 +1152,7 @@ func_dealloc(PyObject *self) return; } #if _Py_TIER2 + _Py_Executors_InvalidateDependency(_PyInterpreterState_GET(), self, 1); _PyJit_Tracer_InvalidateDependency(_PyThreadState_GET(), self); #endif _PyObject_GC_UNTRACK(op); diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index eb43bf63804a32..288e45ce8bc843 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -259,127 +259,127 @@ static void *opcode_targets_table[256] = { }; #if _Py_TIER2 static void *opcode_tracing_targets_table[256] = { - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -387,88 +387,88 @@ static void *opcode_tracing_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, @@ -493,28 +493,28 @@ static void *opcode_tracing_targets_table[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, - &&TARGET_record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, + &&record_previous_inst, }; #endif #else /* _Py_TAIL_CALL_INTERP */ diff --git a/Tools/cases_generator/target_generator.py b/Tools/cases_generator/target_generator.py index b945c3e7203314..36fa1d7fa4908b 100644 --- a/Tools/cases_generator/target_generator.py +++ b/Tools/cases_generator/target_generator.py @@ -34,7 +34,7 @@ def write_opcode_targets(analysis: Analysis, out: CWriter) -> None: targets = ["&&_unknown_opcode,\n"] * 256 for name, op in analysis.opmap.items(): if op < 256: - targets[op] = f"&&TARGET_record_previous_inst,\n" + targets[op] = f"&&record_previous_inst,\n" out.emit("#if _Py_TIER2\n") out.emit("static void *opcode_tracing_targets_table[256] = {\n") for target in targets: From 1f8c3dfbee1d7e7f2f090eeb53a4d26072b1fc41 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 27 Oct 2025 10:44:10 +0000 Subject: [PATCH 121/190] Differentiate the two dependencies --- Python/optimizer.c | 17 ++++++++++++++--- Python/optimizer_bytecodes.c | 4 ++++ Python/optimizer_cases.c.h | 4 ++++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/Python/optimizer.c b/Python/optimizer.c index f745f16fcf30b7..da9247cd139ca4 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -860,6 +860,7 @@ _PyJit_translate_single_bytecode_to_trace( if (uop == _PUSH_FRAME || uop == _RETURN_VALUE || uop == _RETURN_GENERATOR || uop == _YIELD_VALUE) { PyCodeObject *new_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); PyFunctionObject *new_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + if (new_func != NULL) { operand = (uintptr_t)new_func; DPRINTF(2, "Adding %p func to op\n", (void *)operand); @@ -1277,7 +1278,17 @@ uop_optimize( _PyExecutorObject **exec_ptr, bool progress_needed) { - _PyBloomFilter *dependencies = &tstate->interp->jit_state.dependencies; + // Note: the executor has a slightly different set of dependencies than the tracer. + // For example: the tracer depends on function and code objects. + // The executor may only depend on the code object. + // Furthermore, it may decide to cut the trace early, meaning it does not depend on the rest + // of the code objects in the trace. + // It is crucial we differentiate them for performance reasons. + // This prevents endless re-tracing for nested functions. + // It is the optimizer's responsibility to add the dependencies it requires on its own. + _PyBloomFilter new_dependencies; + _Py_BloomFilter_Init(&new_dependencies); + _Py_BloomFilter_Add(&new_dependencies, tstate->interp->jit_state.initial_code); PyInterpreterState *interp = _PyInterpreterState_GET(); _PyUOpInstruction *buffer = interp->jit_state.code_buffer; OPT_STAT_INC(attempts); @@ -1298,7 +1309,7 @@ uop_optimize( if (!is_noopt) { length = _Py_uop_analyze_and_optimize(tstate->interp->jit_state.initial_func, buffer, length, - curr_stackentries, dependencies); + curr_stackentries, &new_dependencies); if (length <= 0) { return length; } @@ -1321,7 +1332,7 @@ uop_optimize( OPT_HIST(effective_trace_length(buffer, length), optimized_trace_length_hist); length = prepare_for_execution(buffer, length); assert(length <= UOP_MAX_TRACE_LENGTH); - _PyExecutorObject *executor = make_executor_from_uops(buffer, length, dependencies, tstate->interp->jit_state.initial_chain_depth); + _PyExecutorObject *executor = make_executor_from_uops(buffer, length, &new_dependencies, tstate->interp->jit_state.initial_chain_depth); if (executor == NULL) { return -1; } diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 7e4a9778a89efe..d57bea14c5ad76 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -779,6 +779,7 @@ dummy_func(void) { ctx->done = true; break; } + _Py_BloomFilter_Add(dependencies, returning_code); int returning_stacklevel = this_instr->operand1; if (frame_pop(ctx, returning_code, returning_stacklevel)) { break; @@ -797,6 +798,7 @@ dummy_func(void) { ctx->done = true; break; } + _Py_BloomFilter_Add(dependencies, returning_code); int returning_stacklevel = this_instr->operand1; if (frame_pop(ctx, returning_code, returning_stacklevel)) { break; @@ -815,6 +817,7 @@ dummy_func(void) { ctx->done = true; break; } + _Py_BloomFilter_Add(dependencies, returning_code); int returning_stacklevel = this_instr->operand1; if (frame_pop(ctx, returning_code, returning_stacklevel)) { break; @@ -871,6 +874,7 @@ dummy_func(void) { } PyFunctionObject *func = (PyFunctionObject *)operand; PyCodeObject *co = (PyCodeObject *)func->func_code; + _Py_BloomFilter_Add(dependencies, co); assert(PyFunction_Check(func)); ctx->frame->func = func; } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 5192280e5ca20f..5f7d600b3cd2a9 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1122,6 +1122,7 @@ ctx->done = true; break; } + _Py_BloomFilter_Add(dependencies, returning_code); int returning_stacklevel = this_instr->operand1; if (frame_pop(ctx, returning_code, returning_stacklevel)) { break; @@ -1179,6 +1180,7 @@ ctx->done = true; break; } + _Py_BloomFilter_Add(dependencies, returning_code); int returning_stacklevel = this_instr->operand1; if (frame_pop(ctx, returning_code, returning_stacklevel)) { break; @@ -2630,6 +2632,7 @@ } PyFunctionObject *func = (PyFunctionObject *)operand; PyCodeObject *co = (PyCodeObject *)func->func_code; + _Py_BloomFilter_Add(dependencies, co); assert(PyFunction_Check(func)); ctx->frame->func = func; break; @@ -3014,6 +3017,7 @@ ctx->done = true; break; } + _Py_BloomFilter_Add(dependencies, returning_code); int returning_stacklevel = this_instr->operand1; if (frame_pop(ctx, returning_code, returning_stacklevel)) { break; From bdd2123ea3139945d115a63b29e51790e291403d Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 27 Oct 2025 14:22:05 +0000 Subject: [PATCH 122/190] Stop recursive traces --- Include/internal/pycore_opcode_metadata.h | 2 +- Include/internal/pycore_optimizer.h | 4 ++-- Python/bytecodes.c | 26 +++++++++++++++++----- Python/executor_cases.c.h | 7 ++++-- Python/generated_cases.c.h | 27 ++++++++++++++++++----- Python/optimizer.c | 16 ++++++++++++-- Tools/cases_generator/analyzer.py | 2 +- 7 files changed, 66 insertions(+), 18 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 2bb915ad62b17f..f72c0412bc6f82 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1146,7 +1146,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [END_FOR] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG }, [END_SEND] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_PURE_FLAG }, - [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [FORMAT_SIMPLE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 7c542e322ecbc2..ff829adbc2eed4 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -361,8 +361,8 @@ extern void _Py_ClearExecutorDeletionList(PyInterpreterState *interp); int _PyJit_translate_single_bytecode_to_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr); -void -_PyJit_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, +int +_PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *curr_instr, _Py_CODEUNIT *insert_exec_instr, _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit, int oparg); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 0328cd3ef9693b..9cd32bf63e756d 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2972,8 +2972,10 @@ dummy_func( oparg >>= 8; insert_exec_at--; } - _PyJit_InitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, oparg); - ENTER_TRACING(); + int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, oparg); + if (succ) { + ENTER_TRACING(); + } } } else { @@ -3019,6 +3021,15 @@ dummy_func( tier1 inst(ENTER_EXECUTOR, (--)) { #ifdef _Py_TIER2 + // We want to end any current trace here, before we possibly need + // to start tracing new ones due to recursive traces in any inner C functions + // in tier2 code. + if (IS_JIT_TRACING()) { + _PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr); + LEAVE_TRACING(); + int err = bail_tracing_and_jit(tstate, frame); + ERROR_IF(err < 0); + } PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; assert(executor->vm_data.index == INSTR_OFFSET() - 1); @@ -5428,9 +5439,12 @@ dummy_func( // Note: it's safe to use target->op.arg here instead of the oparg given by EXTENDED_ARG. // The invariant in the optimizer is the deopt target always points back to the first EXTENDED_ARG. // So setting it to anything else is wrong. - _PyJit_InitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, target->op.arg); + int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, target->op.arg); exit->temperature = initial_temperature_backoff_counter(); - GOTO_TIER_ONE_CONTINUE_TRACING(target); + if (succ) { + GOTO_TIER_ONE_CONTINUE_TRACING(target); + } + GOTO_TIER_ONE(target); } assert(tstate->jit_exit == exit); exit->executor = executor; @@ -5636,7 +5650,9 @@ dummy_func( } tstate->interp->jit_state.specialize_counter = 0; PyCodeObject *prev_code = (PyCodeObject *)Py_NewRef(_PyFrame_GetCode(frame)); - Py_SETREF(tstate->interp->jit_state.prev_instr_code, prev_code); + if (tstate->interp->jit_state.prev_instr_code != prev_code) { + Py_SETREF(tstate->interp->jit_state.prev_instr_code, prev_code); + } tstate->interp->jit_state.prev_instr_frame = frame; tstate->interp->jit_state.prev_instr_oparg = oparg; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index fafaa7bd12bbb5..b81b0300da05fe 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7485,9 +7485,12 @@ _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); assert(tstate->current_executor == (PyObject *)previous_executor); int chain_depth = previous_executor->vm_data.chain_depth + 1; - _PyJit_InitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, target->op.arg); + int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, target->op.arg); exit->temperature = initial_temperature_backoff_counter(); - GOTO_TIER_ONE_CONTINUE_TRACING(target); + if (succ) { + GOTO_TIER_ONE_CONTINUE_TRACING(target); + } + GOTO_TIER_ONE(target); } assert(tstate->jit_exit == exit); exit->executor = executor; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index d9fea0e4ae5767..e9d5a611ccd6e0 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5491,6 +5491,19 @@ INSTRUCTION_STATS(ENTER_EXECUTOR); opcode = ENTER_EXECUTOR; #ifdef _Py_TIER2 + + if (IS_JIT_TRACING()) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + LEAVE_TRACING(); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = bail_tracing_and_jit(tstate, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + } PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; assert(executor->vm_data.index == INSTR_OFFSET() - 1); @@ -7690,8 +7703,10 @@ oparg >>= 8; insert_exec_at--; } - _PyJit_InitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, oparg); - ENTER_TRACING(); + int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, oparg); + if (succ) { + ENTER_TRACING(); + } } } else { @@ -12342,9 +12357,11 @@ JUMP_TO_LABEL(error); } tstate->interp->jit_state.specialize_counter = 0; PyCodeObject *prev_code = (PyCodeObject *)Py_NewRef(_PyFrame_GetCode(frame)); - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_SETREF(tstate->interp->jit_state.prev_instr_code, prev_code); - stack_pointer = _PyFrame_GetStackPointer(frame); + if (tstate->interp->jit_state.prev_instr_code != prev_code) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_SETREF(tstate->interp->jit_state.prev_instr_code, prev_code); + stack_pointer = _PyFrame_GetStackPointer(frame); + } tstate->interp->jit_state.prev_instr_frame = frame; tstate->interp->jit_state.prev_instr_oparg = oparg; tstate->interp->jit_state.prev_instr_stacklevel = STACK_LEVEL(); diff --git a/Python/optimizer.c b/Python/optimizer.c index da9247cd139ca4..05ab91fd543535 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -619,6 +619,11 @@ _PyJit_translate_single_bytecode_to_trace( } if (opcode == ENTER_EXECUTOR) { + int is_first_instr = tstate->interp->jit_state.close_loop_instr == next_instr || tstate->interp->jit_state.insert_exec_instr == next_instr; + if (is_first_instr && tstate->interp->jit_state.code_curr_size > 5) { + ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0, 0); + goto done; + } goto full; } @@ -925,9 +930,15 @@ _PyJit_translate_single_bytecode_to_trace( return 0; } -void -_PyJit_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *curr_instr, _Py_CODEUNIT *insert_exec_instr, _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit, int oparg) +// Returns 0 for do not enter tracing, 1 on enter tracing. +int +_PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *curr_instr, _Py_CODEUNIT *insert_exec_instr, _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit, int oparg) { + // A recursive trace. + // Don't trace into the inner call because it will stomp on the previous trace, causing endless retraces. + if (tstate->interp->jit_state.code_curr_size > 2) { + return 0; + } PyCodeObject *code = _PyFrame_GetCode(frame); #ifdef Py_DEBUG char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); @@ -965,6 +976,7 @@ _PyJit_InitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_ tstate->interp->jit_state.dynamic_jump_taken = false; tstate->interp->jit_state.prev_instr_is_super = false; _Py_BloomFilter_Init(&tstate->interp->jit_state.dependencies); + return 1; } void diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index b206a0a4b985d3..b66e2f0ab0d203 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -699,7 +699,7 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "PyStackRef_Unwrap", "_PyLong_CheckExactAndCompact", "_PyExecutor_FromExit", - "_PyJit_InitializeTracing", + "_PyJit_TryInitializeTracing", ) From 5f4f3101798255ded284d69dddd6a0990c14347c Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 27 Oct 2025 16:51:07 +0000 Subject: [PATCH 123/190] fix backoff --- Include/internal/pycore_backoff.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/internal/pycore_backoff.h b/Include/internal/pycore_backoff.h index 6b78322fb06232..3762257f3173d9 100644 --- a/Include/internal/pycore_backoff.h +++ b/Include/internal/pycore_backoff.h @@ -105,7 +105,7 @@ backoff_counter_triggers(_Py_BackoffCounter counter) // as we always end up tracing the loop iteration's // exhaustion iteration. Which aborts our current tracer. #define JUMP_BACKWARD_INITIAL_VALUE 4000 -#define JUMP_BACKWARD_INITIAL_BACKOFF 14 +#define JUMP_BACKWARD_INITIAL_BACKOFF 12 static inline _Py_BackoffCounter initial_jump_backoff_counter(void) { From 8adaf4dc6ae5ee519c2adf184d8548670e454a1b Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 27 Oct 2025 18:30:18 +0000 Subject: [PATCH 124/190] merge from upstream --- Python/bytecodes.c | 5 +---- Python/executor_cases.c.h | 4 +--- Python/optimizer.c | 1 - 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 0fad05e9c18381..971660ace56ab0 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5392,10 +5392,7 @@ dummy_func( tier2 op(_ERROR_POP_N, (target/2 --)) { assert(oparg == 0); - _Py_CODEUNIT *current_instr = _PyFrame_GetBytecode(frame) + target; - _Py_CODEUNIT *next_instr = current_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[current_instr->op.code]]; - // gh-140104: The exception handler expects frame->instr_ptr to be pointing to next_instr, not this_instr! - frame->instr_ptr = next_instr; + frame->instr_ptr = _PyFrame_GetBytecode(frame) + target; SYNC_SP(); GOTO_TIER_ONE(NULL); } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 9e46786d050ead..5273670008c029 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7439,9 +7439,7 @@ oparg = CURRENT_OPARG(); uint32_t target = (uint32_t)CURRENT_OPERAND0(); assert(oparg == 0); - _Py_CODEUNIT *current_instr = _PyFrame_GetBytecode(frame) + target; - _Py_CODEUNIT *next_instr = current_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[current_instr->op.code]]; - frame->instr_ptr = next_instr; + frame->instr_ptr = _PyFrame_GetBytecode(frame) + target; GOTO_TIER_ONE(NULL); break; } diff --git a/Python/optimizer.c b/Python/optimizer.c index c7f7f0209318cc..4b41236b69ba2a 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1352,7 +1352,6 @@ uop_optimize( assert(length <= UOP_MAX_TRACE_LENGTH); // Check executor coldness - PyThreadState *tstate = PyThreadState_Get(); // It's okay if this ends up going negative. if (--tstate->interp->executor_creation_counter == 0) { _Py_set_eval_breaker_bit(tstate, _PY_EVAL_JIT_INVALIDATE_COLD_BIT); From e918cb26f74639ed15e2926f2db0a45a237283ac Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 27 Oct 2025 21:26:42 +0000 Subject: [PATCH 125/190] Move unpredictable jump detection to the cases generator --- Include/internal/pycore_interp_structs.h | 1 - Include/internal/pycore_opcode_metadata.h | 18 +++++++------ Python/ceval_macros.h | 6 ----- Python/generated_cases.c.h | 7 ----- Python/optimizer.c | 7 ++--- Tools/cases_generator/analyzer.py | 6 ++++- Tools/cases_generator/generators_common.py | 27 ++----------------- .../opcode_metadata_generator.py | 1 + 8 files changed, 22 insertions(+), 51 deletions(-) diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index d8b12e3e7e0b95..5c9f25eff6c951 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -759,7 +759,6 @@ typedef _Py_CODEUNIT *(*_PyJitEntryFuncPtr)(struct _PyExecutorObject *exec, _PyI typedef struct _PyJitTracerState { bool dependencies_still_valid; - bool dynamic_jump_taken; bool prev_instr_is_super; int code_max_size; int code_curr_size; diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index f72c0412bc6f82..8dda65a55b3453 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1031,6 +1031,7 @@ enum InstructionFormat { #define HAS_ERROR_NO_POP_FLAG (4096) #define HAS_NO_SAVE_IP_FLAG (8192) #define HAS_PERIODIC_FLAG (16384) +#define HAS_UNPREDICTABLE_JUMP_FLAG (32768) #define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG)) #define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_CONST_FLAG)) #define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NAME_FLAG)) @@ -1046,6 +1047,7 @@ enum InstructionFormat { #define OPCODE_HAS_ERROR_NO_POP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ERROR_NO_POP_FLAG)) #define OPCODE_HAS_NO_SAVE_IP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NO_SAVE_IP_FLAG)) #define OPCODE_HAS_PERIODIC(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PERIODIC_FLAG)) +#define OPCODE_HAS_UNPREDICTABLE_JUMP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_UNPREDICTABLE_JUMP_FLAG)) #define OPARG_SIMPLE 0 #define OPARG_CACHE_1 1 @@ -1143,7 +1145,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [DELETE_SUBSCR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, [END_FOR] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG }, [END_SEND] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_PURE_FLAG }, [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1151,11 +1153,11 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [FORMAT_SIMPLE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [FORMAT_WITH_SPEC] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG }, + [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, + [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, + [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, [GET_AITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [GET_ANEXT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [GET_AWAITABLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1167,10 +1169,10 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG }, [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, [INSTRUMENTED_INSTRUCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, @@ -1255,7 +1257,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [RESUME_CHECK] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, - [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [SETUP_ANNOTATIONS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SET_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index d4f4a83e13404c..0f7e2175480bd8 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -333,12 +333,6 @@ GETITEM(PyObject *v, Py_ssize_t i) { #define RECORD_BRANCH_TAKEN(bitset, flag) #endif -#if _Py_TIER2 -# define RECORD_DYNAMIC_JUMP_TAKEN() tstate->interp->jit_state.dynamic_jump_taken = true; -#else -# define RECORD_DYNAMIC_JUMP_TAKEN() -#endif - #define UNBOUNDLOCAL_ERROR_MSG \ "cannot access local variable '%s' where it is not associated with a value" #define UNBOUNDFREE_ERROR_MSG \ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index e9d5a611ccd6e0..b97f4987c80fdd 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5684,7 +5684,6 @@ JUMP_TO_LABEL(error); } JUMPBY(oparg + 1); - RECORD_DYNAMIC_JUMP_TAKEN(); stack_pointer[-1] = null_or_index; DISPATCH(); } @@ -5813,7 +5812,6 @@ if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyList_GET_SIZE(list_o)) { null_or_index = PyStackRef_TagInt(-1); JUMPBY(oparg + 1); - RECORD_DYNAMIC_JUMP_TAKEN(); stack_pointer[-1] = null_or_index; DISPATCH(); } @@ -5838,7 +5836,6 @@ if (result == 0) { null_or_index = PyStackRef_TagInt(-1); JUMPBY(oparg + 1); - RECORD_DYNAMIC_JUMP_TAKEN(); stack_pointer[-1] = null_or_index; DISPATCH(); } @@ -5895,7 +5892,6 @@ STAT_INC(FOR_ITER, hit); if (r->len <= 0) { JUMPBY(oparg + 1); - RECORD_DYNAMIC_JUMP_TAKEN(); DISPATCH(); } } @@ -5958,7 +5954,6 @@ if ((size_t)PyStackRef_UntagInt(null_or_index) >= (size_t)PyTuple_GET_SIZE(tuple_o)) { null_or_index = PyStackRef_TagInt(-1); JUMPBY(oparg + 1); - RECORD_DYNAMIC_JUMP_TAKEN(); stack_pointer[-1] = null_or_index; DISPATCH(); } @@ -6957,7 +6952,6 @@ JUMP_TO_LABEL(error); } JUMPBY(oparg + 1); - RECORD_DYNAMIC_JUMP_TAKEN(); stack_pointer[-1] = null_or_index; DISPATCH(); } @@ -10543,7 +10537,6 @@ if (err == 0) { assert(retval_o != NULL); JUMPBY(oparg); - RECORD_DYNAMIC_JUMP_TAKEN(); } else { stack_pointer += -1; diff --git a/Python/optimizer.c b/Python/optimizer.c index 4b41236b69ba2a..12bb803d18d87e 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -632,8 +632,10 @@ _PyJit_translate_single_bytecode_to_trace( goto done; } - // Strange control-flow, unsupported opcode, etc. - if (tstate->interp->jit_state.dynamic_jump_taken) { + // Strange control-flow + bool has_dynamic_jump_taken = OPCODE_HAS_UNPREDICTABLE_JUMP(opcode) && + (next_instr != this_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]); + if (has_dynamic_jump_taken) { DPRINTF(2, "Unsupported: dynamic jump taken\n"); goto unsupported; } @@ -974,7 +976,6 @@ _PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _ tstate->interp->jit_state.prev_instr_frame = frame; tstate->interp->jit_state.prev_instr_oparg = oparg; tstate->interp->jit_state.prev_instr_stacklevel = curr_stackdepth; - tstate->interp->jit_state.dynamic_jump_taken = false; tstate->interp->jit_state.prev_instr_is_super = false; _Py_BloomFilter_Init(&tstate->interp->jit_state.dependencies); return 1; diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index b66e2f0ab0d203..10dc7d5b31a214 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -910,7 +910,11 @@ def stmt_has_jump_on_unpredictable_path(stmt: Stmt, branches_seen: int) -> tuple return True, branches_seen return False, branches_seen elif isinstance(stmt, IfStmt): - return True, branches_seen + 1 + predict, seen = stmt_has_jump_on_unpredictable_path(stmt.body, branches_seen) + if stmt.else_body: + predict_else, seen_else = stmt_has_jump_on_unpredictable_path(stmt.else_body, branches_seen) + return predict != predict_else, seen + seen_else + 1 + return predict, seen + 1 elif isinstance(stmt, MacroIfStmt): predict, seen = stmt_has_jump_on_unpredictable_path_body(stmt.body, branches_seen) if stmt.else_body: diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 5cba378933e9e6..560996ca1b4a58 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -129,7 +129,6 @@ def __init__(self, out: CWriter, labels: dict[str, Label], cannot_escape: bool = "DISPATCH": self.dispatch, "INSTRUCTION_SIZE": self.instruction_size, "stack_pointer": self.stack_pointer, - "JUMPBY": self.jumpby, "DISPATCH_SAME_OPARG": self.dispatch_same_oparg, } self.out = out @@ -422,30 +421,6 @@ def sync_sp( storage.stack.clear(self.out) return True - def jumpby( - self, - tkn: Token, - tkn_iter: TokenIterator, - uop: CodeSection, - storage: Storage, - inst: Instruction | None, - ) -> bool: - self.out.start_line() - self.emit(tkn) - lparen = next(tkn_iter) - self.emit(lparen) - jump = next(tkn_iter) - self.emit(jump) - emit_to(self.out, tkn_iter, "RPAREN") - next(tkn_iter) - self.emit(");\n") - - - if uop.properties.unpredictable_jump and jump.text != "0": - self.out.start_line() - self.emit("RECORD_DYNAMIC_JUMP_TAKEN();\n") - return True - def stack_pointer( self, tkn: Token, @@ -780,6 +755,8 @@ def cflags(p: Properties) -> str: flags.append("HAS_PURE_FLAG") if p.no_save_ip: flags.append("HAS_NO_SAVE_IP_FLAG") + if p.unpredictable_jump: + flags.append("HAS_UNPREDICTABLE_JUMP_FLAG") if flags: return " | ".join(flags) else: diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py index d17a2431c75b37..78f19bd7aece24 100644 --- a/Tools/cases_generator/opcode_metadata_generator.py +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -56,6 +56,7 @@ "ERROR_NO_POP", "NO_SAVE_IP", "PERIODIC", + "UNPREDICTABLE_JUMP", ] From e9e2bb9acd29f7d1426baf1c055d60d739340a66 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 27 Oct 2025 22:27:46 +0000 Subject: [PATCH 126/190] Sink LOAD_IP into guards --- Include/internal/pycore_uop_ids.h | 318 ++++++++++++----------- Include/internal/pycore_uop_metadata.h | 14 +- Python/bytecodes.c | 16 +- Python/executor_cases.c.h | 34 ++- Python/optimizer.c | 21 +- Python/optimizer_cases.c.h | 10 +- Tools/cases_generator/tier2_generator.py | 14 - 7 files changed, 243 insertions(+), 184 deletions(-) diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 9f4ae71a313aa0..067da4209f0102 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -137,47 +137,49 @@ extern "C" { #define _GUARD_DORV_NO_DICT 392 #define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 393 #define _GUARD_GLOBALS_VERSION 394 -#define _GUARD_IP 395 -#define _GUARD_IS_FALSE_POP 396 -#define _GUARD_IS_NONE_POP 397 -#define _GUARD_IS_NOT_NONE_POP 398 -#define _GUARD_IS_TRUE_POP 399 -#define _GUARD_KEYS_VERSION 400 -#define _GUARD_NOS_DICT 401 -#define _GUARD_NOS_FLOAT 402 -#define _GUARD_NOS_INT 403 -#define _GUARD_NOS_LIST 404 -#define _GUARD_NOS_NOT_NULL 405 -#define _GUARD_NOS_NULL 406 -#define _GUARD_NOS_OVERFLOWED 407 -#define _GUARD_NOS_TUPLE 408 -#define _GUARD_NOS_UNICODE 409 -#define _GUARD_NOT_EXHAUSTED_LIST 410 -#define _GUARD_NOT_EXHAUSTED_RANGE 411 -#define _GUARD_NOT_EXHAUSTED_TUPLE 412 -#define _GUARD_THIRD_NULL 413 -#define _GUARD_TOS_ANY_SET 414 -#define _GUARD_TOS_DICT 415 -#define _GUARD_TOS_FLOAT 416 -#define _GUARD_TOS_INT 417 -#define _GUARD_TOS_LIST 418 -#define _GUARD_TOS_OVERFLOWED 419 -#define _GUARD_TOS_SLICE 420 -#define _GUARD_TOS_TUPLE 421 -#define _GUARD_TOS_UNICODE 422 -#define _GUARD_TYPE_VERSION 423 -#define _GUARD_TYPE_VERSION_AND_LOCK 424 -#define _HANDLE_PENDING_AND_DEOPT 425 +#define _GUARD_IP_PUSH_FRAME 395 +#define _GUARD_IP_RETURN_VALUE 396 +#define _GUARD_IP_YIELD_VALUE 397 +#define _GUARD_IS_FALSE_POP 398 +#define _GUARD_IS_NONE_POP 399 +#define _GUARD_IS_NOT_NONE_POP 400 +#define _GUARD_IS_TRUE_POP 401 +#define _GUARD_KEYS_VERSION 402 +#define _GUARD_NOS_DICT 403 +#define _GUARD_NOS_FLOAT 404 +#define _GUARD_NOS_INT 405 +#define _GUARD_NOS_LIST 406 +#define _GUARD_NOS_NOT_NULL 407 +#define _GUARD_NOS_NULL 408 +#define _GUARD_NOS_OVERFLOWED 409 +#define _GUARD_NOS_TUPLE 410 +#define _GUARD_NOS_UNICODE 411 +#define _GUARD_NOT_EXHAUSTED_LIST 412 +#define _GUARD_NOT_EXHAUSTED_RANGE 413 +#define _GUARD_NOT_EXHAUSTED_TUPLE 414 +#define _GUARD_THIRD_NULL 415 +#define _GUARD_TOS_ANY_SET 416 +#define _GUARD_TOS_DICT 417 +#define _GUARD_TOS_FLOAT 418 +#define _GUARD_TOS_INT 419 +#define _GUARD_TOS_LIST 420 +#define _GUARD_TOS_OVERFLOWED 421 +#define _GUARD_TOS_SLICE 422 +#define _GUARD_TOS_TUPLE 423 +#define _GUARD_TOS_UNICODE 424 +#define _GUARD_TYPE_VERSION 425 +#define _GUARD_TYPE_VERSION_AND_LOCK 426 +#define _HANDLE_PENDING_AND_DEOPT 427 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 426 -#define _INIT_CALL_PY_EXACT_ARGS 427 -#define _INIT_CALL_PY_EXACT_ARGS_0 428 -#define _INIT_CALL_PY_EXACT_ARGS_1 429 -#define _INIT_CALL_PY_EXACT_ARGS_2 430 -#define _INIT_CALL_PY_EXACT_ARGS_3 431 -#define _INIT_CALL_PY_EXACT_ARGS_4 432 -#define _INSERT_NULL 433 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 428 +#define _INIT_CALL_PY_EXACT_ARGS 429 +#define _INIT_CALL_PY_EXACT_ARGS_0 430 +#define _INIT_CALL_PY_EXACT_ARGS_1 431 +#define _INIT_CALL_PY_EXACT_ARGS_2 432 +#define _INIT_CALL_PY_EXACT_ARGS_3 433 +#define _INIT_CALL_PY_EXACT_ARGS_4 434 +#define _INSERT_NULL 435 #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER #define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION #define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD @@ -187,178 +189,178 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _IS_NONE 434 +#define _IS_NONE 436 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 435 -#define _ITER_CHECK_RANGE 436 -#define _ITER_CHECK_TUPLE 437 -#define _ITER_JUMP_LIST 438 -#define _ITER_JUMP_RANGE 439 -#define _ITER_JUMP_TUPLE 440 -#define _ITER_NEXT_LIST 441 -#define _ITER_NEXT_LIST_TIER_TWO 442 -#define _ITER_NEXT_RANGE 443 -#define _ITER_NEXT_TUPLE 444 +#define _ITER_CHECK_LIST 437 +#define _ITER_CHECK_RANGE 438 +#define _ITER_CHECK_TUPLE 439 +#define _ITER_JUMP_LIST 440 +#define _ITER_JUMP_RANGE 441 +#define _ITER_JUMP_TUPLE 442 +#define _ITER_NEXT_LIST 443 +#define _ITER_NEXT_LIST_TIER_TWO 444 +#define _ITER_NEXT_RANGE 445 +#define _ITER_NEXT_TUPLE 446 #define _JUMP_BACKWARD_NO_INTERRUPT JUMP_BACKWARD_NO_INTERRUPT -#define _JUMP_TO_TOP 445 +#define _JUMP_TO_TOP 447 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 446 -#define _LOAD_ATTR_CLASS 447 +#define _LOAD_ATTR 448 +#define _LOAD_ATTR_CLASS 449 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 448 -#define _LOAD_ATTR_METHOD_LAZY_DICT 449 -#define _LOAD_ATTR_METHOD_NO_DICT 450 -#define _LOAD_ATTR_METHOD_WITH_VALUES 451 -#define _LOAD_ATTR_MODULE 452 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 453 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 454 -#define _LOAD_ATTR_PROPERTY_FRAME 455 -#define _LOAD_ATTR_SLOT 456 -#define _LOAD_ATTR_WITH_HINT 457 +#define _LOAD_ATTR_INSTANCE_VALUE 450 +#define _LOAD_ATTR_METHOD_LAZY_DICT 451 +#define _LOAD_ATTR_METHOD_NO_DICT 452 +#define _LOAD_ATTR_METHOD_WITH_VALUES 453 +#define _LOAD_ATTR_MODULE 454 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 455 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 456 +#define _LOAD_ATTR_PROPERTY_FRAME 457 +#define _LOAD_ATTR_SLOT 458 +#define _LOAD_ATTR_WITH_HINT 459 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _LOAD_BYTECODE 458 +#define _LOAD_BYTECODE 460 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 459 -#define _LOAD_CONST_INLINE_BORROW 460 -#define _LOAD_CONST_UNDER_INLINE 461 -#define _LOAD_CONST_UNDER_INLINE_BORROW 462 +#define _LOAD_CONST_INLINE 461 +#define _LOAD_CONST_INLINE_BORROW 462 +#define _LOAD_CONST_UNDER_INLINE 463 +#define _LOAD_CONST_UNDER_INLINE_BORROW 464 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 463 -#define _LOAD_FAST_0 464 -#define _LOAD_FAST_1 465 -#define _LOAD_FAST_2 466 -#define _LOAD_FAST_3 467 -#define _LOAD_FAST_4 468 -#define _LOAD_FAST_5 469 -#define _LOAD_FAST_6 470 -#define _LOAD_FAST_7 471 +#define _LOAD_FAST 465 +#define _LOAD_FAST_0 466 +#define _LOAD_FAST_1 467 +#define _LOAD_FAST_2 468 +#define _LOAD_FAST_3 469 +#define _LOAD_FAST_4 470 +#define _LOAD_FAST_5 471 +#define _LOAD_FAST_6 472 +#define _LOAD_FAST_7 473 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR -#define _LOAD_FAST_BORROW 472 -#define _LOAD_FAST_BORROW_0 473 -#define _LOAD_FAST_BORROW_1 474 -#define _LOAD_FAST_BORROW_2 475 -#define _LOAD_FAST_BORROW_3 476 -#define _LOAD_FAST_BORROW_4 477 -#define _LOAD_FAST_BORROW_5 478 -#define _LOAD_FAST_BORROW_6 479 -#define _LOAD_FAST_BORROW_7 480 +#define _LOAD_FAST_BORROW 474 +#define _LOAD_FAST_BORROW_0 475 +#define _LOAD_FAST_BORROW_1 476 +#define _LOAD_FAST_BORROW_2 477 +#define _LOAD_FAST_BORROW_3 478 +#define _LOAD_FAST_BORROW_4 479 +#define _LOAD_FAST_BORROW_5 480 +#define _LOAD_FAST_BORROW_6 481 +#define _LOAD_FAST_BORROW_7 482 #define _LOAD_FAST_BORROW_LOAD_FAST_BORROW LOAD_FAST_BORROW_LOAD_FAST_BORROW #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 481 -#define _LOAD_GLOBAL_BUILTINS 482 -#define _LOAD_GLOBAL_MODULE 483 +#define _LOAD_GLOBAL 483 +#define _LOAD_GLOBAL_BUILTINS 484 +#define _LOAD_GLOBAL_MODULE 485 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 484 -#define _LOAD_SMALL_INT_0 485 -#define _LOAD_SMALL_INT_1 486 -#define _LOAD_SMALL_INT_2 487 -#define _LOAD_SMALL_INT_3 488 -#define _LOAD_SPECIAL 489 +#define _LOAD_SMALL_INT 486 +#define _LOAD_SMALL_INT_0 487 +#define _LOAD_SMALL_INT_1 488 +#define _LOAD_SMALL_INT_2 489 +#define _LOAD_SMALL_INT_3 490 +#define _LOAD_SPECIAL 491 #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _MAKE_CALLARGS_A_TUPLE 490 +#define _MAKE_CALLARGS_A_TUPLE 492 #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_WARM 491 +#define _MAKE_WARM 493 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 492 -#define _MAYBE_EXPAND_METHOD_KW 493 -#define _MONITOR_CALL 494 -#define _MONITOR_CALL_KW 495 -#define _MONITOR_JUMP_BACKWARD 496 -#define _MONITOR_RESUME 497 +#define _MAYBE_EXPAND_METHOD 494 +#define _MAYBE_EXPAND_METHOD_KW 495 +#define _MONITOR_CALL 496 +#define _MONITOR_CALL_KW 497 +#define _MONITOR_JUMP_BACKWARD 498 +#define _MONITOR_RESUME 499 #define _NOP NOP -#define _POP_CALL 498 -#define _POP_CALL_LOAD_CONST_INLINE_BORROW 499 -#define _POP_CALL_ONE 500 -#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 501 -#define _POP_CALL_TWO 502 -#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 503 +#define _POP_CALL 500 +#define _POP_CALL_LOAD_CONST_INLINE_BORROW 501 +#define _POP_CALL_ONE 502 +#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 503 +#define _POP_CALL_TWO 504 +#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 505 #define _POP_EXCEPT POP_EXCEPT #define _POP_ITER POP_ITER -#define _POP_JUMP_IF_FALSE 504 -#define _POP_JUMP_IF_TRUE 505 +#define _POP_JUMP_IF_FALSE 506 +#define _POP_JUMP_IF_TRUE 507 #define _POP_TOP POP_TOP -#define _POP_TOP_FLOAT 506 -#define _POP_TOP_INT 507 -#define _POP_TOP_LOAD_CONST_INLINE 508 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 509 -#define _POP_TOP_NOP 510 -#define _POP_TOP_UNICODE 511 -#define _POP_TWO 512 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW 513 +#define _POP_TOP_FLOAT 508 +#define _POP_TOP_INT 509 +#define _POP_TOP_LOAD_CONST_INLINE 510 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 511 +#define _POP_TOP_NOP 512 +#define _POP_TOP_UNICODE 513 +#define _POP_TWO 514 +#define _POP_TWO_LOAD_CONST_INLINE_BORROW 515 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 514 +#define _PUSH_FRAME 516 #define _PUSH_NULL PUSH_NULL -#define _PUSH_NULL_CONDITIONAL 515 -#define _PY_FRAME_GENERAL 516 -#define _PY_FRAME_KW 517 -#define _QUICKEN_RESUME 518 -#define _REPLACE_WITH_TRUE 519 +#define _PUSH_NULL_CONDITIONAL 517 +#define _PY_FRAME_GENERAL 518 +#define _PY_FRAME_KW 519 +#define _QUICKEN_RESUME 520 +#define _REPLACE_WITH_TRUE 521 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 520 -#define _SEND 521 -#define _SEND_GEN_FRAME 522 +#define _SAVE_RETURN_OFFSET 522 +#define _SEND 523 +#define _SEND_GEN_FRAME 524 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 523 -#define _STORE_ATTR 524 -#define _STORE_ATTR_INSTANCE_VALUE 525 -#define _STORE_ATTR_SLOT 526 -#define _STORE_ATTR_WITH_HINT 527 +#define _START_EXECUTOR 525 +#define _STORE_ATTR 526 +#define _STORE_ATTR_INSTANCE_VALUE 527 +#define _STORE_ATTR_SLOT 528 +#define _STORE_ATTR_WITH_HINT 529 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 528 -#define _STORE_FAST_0 529 -#define _STORE_FAST_1 530 -#define _STORE_FAST_2 531 -#define _STORE_FAST_3 532 -#define _STORE_FAST_4 533 -#define _STORE_FAST_5 534 -#define _STORE_FAST_6 535 -#define _STORE_FAST_7 536 +#define _STORE_FAST 530 +#define _STORE_FAST_0 531 +#define _STORE_FAST_1 532 +#define _STORE_FAST_2 533 +#define _STORE_FAST_3 534 +#define _STORE_FAST_4 535 +#define _STORE_FAST_5 536 +#define _STORE_FAST_6 537 +#define _STORE_FAST_7 538 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 537 -#define _STORE_SUBSCR 538 -#define _STORE_SUBSCR_DICT 539 -#define _STORE_SUBSCR_LIST_INT 540 -#define _SWAP 541 -#define _SWAP_2 542 -#define _SWAP_3 543 -#define _TIER2_RESUME_CHECK 544 -#define _TO_BOOL 545 +#define _STORE_SLICE 539 +#define _STORE_SUBSCR 540 +#define _STORE_SUBSCR_DICT 541 +#define _STORE_SUBSCR_LIST_INT 542 +#define _SWAP 543 +#define _SWAP_2 544 +#define _SWAP_3 545 +#define _TIER2_RESUME_CHECK 546 +#define _TO_BOOL 547 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT -#define _TO_BOOL_LIST 546 +#define _TO_BOOL_LIST 548 #define _TO_BOOL_NONE TO_BOOL_NONE -#define _TO_BOOL_STR 547 +#define _TO_BOOL_STR 549 #define _UNARY_INVERT UNARY_INVERT #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 548 -#define _UNPACK_SEQUENCE_LIST 549 -#define _UNPACK_SEQUENCE_TUPLE 550 -#define _UNPACK_SEQUENCE_TWO_TUPLE 551 +#define _UNPACK_SEQUENCE 550 +#define _UNPACK_SEQUENCE_LIST 551 +#define _UNPACK_SEQUENCE_TUPLE 552 +#define _UNPACK_SEQUENCE_TWO_TUPLE 553 #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 551 +#define MAX_UOP_ID 553 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 82a0fa92652524..67c5827ffd0c00 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -337,7 +337,9 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_ERROR_POP_N] = HAS_ARG_FLAG, [_TIER2_RESUME_CHECK] = HAS_PERIODIC_FLAG, [_COLD_EXIT] = 0, - [_GUARD_IP] = HAS_EXIT_FLAG, + [_GUARD_IP_PUSH_FRAME] = HAS_EXIT_FLAG, + [_GUARD_IP_YIELD_VALUE] = HAS_EXIT_FLAG, + [_GUARD_IP_RETURN_VALUE] = HAS_EXIT_FLAG, [_DYNAMIC_EXIT] = HAS_ESCAPES_FLAG, }; @@ -474,7 +476,9 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_GUARD_DORV_NO_DICT] = "_GUARD_DORV_NO_DICT", [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT", [_GUARD_GLOBALS_VERSION] = "_GUARD_GLOBALS_VERSION", - [_GUARD_IP] = "_GUARD_IP", + [_GUARD_IP_PUSH_FRAME] = "_GUARD_IP_PUSH_FRAME", + [_GUARD_IP_RETURN_VALUE] = "_GUARD_IP_RETURN_VALUE", + [_GUARD_IP_YIELD_VALUE] = "_GUARD_IP_YIELD_VALUE", [_GUARD_IS_FALSE_POP] = "_GUARD_IS_FALSE_POP", [_GUARD_IS_NONE_POP] = "_GUARD_IS_NONE_POP", [_GUARD_IS_NOT_NONE_POP] = "_GUARD_IS_NOT_NONE_POP", @@ -1309,7 +1313,11 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _COLD_EXIT: return 0; - case _GUARD_IP: + case _GUARD_IP_PUSH_FRAME: + return 0; + case _GUARD_IP_YIELD_VALUE: + return 0; + case _GUARD_IP_RETURN_VALUE: return 0; case _DYNAMIC_EXIT: return 0; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 971660ace56ab0..b3977e5613b4dc 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5443,10 +5443,24 @@ dummy_func( TIER2_TO_TIER2(exit->executor); } - tier2 op(_GUARD_IP, (ip/4 --)) { + tier2 op(_GUARD_IP_PUSH_FRAME, (ip/4 --)) { EXIT_IF(frame->instr_ptr != (_Py_CODEUNIT *)ip); } + tier2 op(_GUARD_IP_YIELD_VALUE, (ip/4 --)) { + if (frame->instr_ptr + 1 + INLINE_CACHE_ENTRIES_SEND != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += 1 + INLINE_CACHE_ENTRIES_SEND; + EXIT_IF(true); + } + } + + tier2 op(_GUARD_IP_RETURN_VALUE, (ip/4 --)) { + if (frame->instr_ptr + frame->return_offset != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += frame->return_offset; + EXIT_IF(true); + } + } + // Note: this is different than _COLD_EXIT/_EXIT_TRACE, as it may lead to multiple executors // from a single exit! tier2 op(_DYNAMIC_EXIT, (exit_p/4 --)) { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 5273670008c029..7f915e31b12108 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1930,7 +1930,7 @@ frame = tstate->current_frame = dying->previous; _PyEval_FrameClearAndPop(tstate, dying); stack_pointer = _PyFrame_GetStackPointer(frame); - frame->instr_ptr += (frame->return_offset); + LOAD_IP(frame->return_offset); res = temp; LLTRACE_RESUME_FRAME(); stack_pointer[0] = res; @@ -2096,7 +2096,7 @@ _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); #endif stack_pointer = _PyFrame_GetStackPointer(frame); - frame->instr_ptr += (1 + INLINE_CACHE_ENTRIES_SEND); + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); value = PyStackRef_MakeHeapSafe(temp); LLTRACE_RESUME_FRAME(); stack_pointer[0] = value; @@ -5347,7 +5347,7 @@ frame = tstate->current_frame = temp; tstate->py_recursion_remaining--; LOAD_SP(); - frame->instr_ptr += (0); + LOAD_IP(0); LLTRACE_RESUME_FRAME(); break; } @@ -6752,7 +6752,7 @@ _PyInterpreterFrame *prev = frame->previous; _PyThreadState_PopFrame(tstate, frame); frame = tstate->current_frame = prev; - frame->instr_ptr += (frame->return_offset); + LOAD_IP(frame->return_offset); stack_pointer = _PyFrame_GetStackPointer(frame); res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen); LLTRACE_RESUME_FRAME(); @@ -7493,7 +7493,7 @@ break; } - case _GUARD_IP: { + case _GUARD_IP_PUSH_FRAME: { PyObject *ip = (PyObject *)CURRENT_OPERAND0(); if (frame->instr_ptr != (_Py_CODEUNIT *)ip) { UOP_STAT_INC(uopcode, miss); @@ -7502,6 +7502,30 @@ break; } + case _GUARD_IP_YIELD_VALUE: { + PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + if (frame->instr_ptr + 1 + INLINE_CACHE_ENTRIES_SEND != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += 1 + INLINE_CACHE_ENTRIES_SEND; + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + break; + } + + case _GUARD_IP_RETURN_VALUE: { + PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + if (frame->instr_ptr + frame->return_offset != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += frame->return_offset; + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + break; + } + case _DYNAMIC_EXIT: { PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); _Py_CODEUNIT *target = frame->instr_ptr; diff --git a/Python/optimizer.c b/Python/optimizer.c index 12bb803d18d87e..e6955edbc194c3 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -901,7 +901,21 @@ _PyJit_translate_single_bytecode_to_trace( } // End switch (opcode) if (needs_guard_ip) { - ADD_TO_TRACE(_GUARD_IP, 0, (uintptr_t)next_instr, 0); + switch (trace[trace_length-1].opcode) { + case _PUSH_FRAME: + ADD_TO_TRACE(_GUARD_IP_PUSH_FRAME, 0, (uintptr_t)next_instr, 0); + break; + case _RETURN_GENERATOR: + case _RETURN_VALUE: + ADD_TO_TRACE(_GUARD_IP_RETURN_VALUE, 0, (uintptr_t)next_instr, 0); + break; + case _YIELD_VALUE: + ADD_TO_TRACE(_GUARD_IP_YIELD_VALUE, 0, (uintptr_t)next_instr, 0); + break; + default: + DPRINTF(1, "Unknown uop needing guard ip %s\n", _PyOpcode_uop_name[trace[trace_length-1].opcode]); + Py_UNREACHABLE(); + } } // Loop back to the start int is_first_instr = tstate->interp->jit_state.close_loop_instr == next_instr || tstate->interp->jit_state.insert_exec_instr == next_instr; @@ -1074,7 +1088,10 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) if (opcode == _FOR_ITER_TIER_TWO) { exit_op = _DYNAMIC_EXIT; } - else if (opcode == _GUARD_IP) { + else if ( + opcode == _GUARD_IP_PUSH_FRAME || + opcode == _GUARD_IP_RETURN_VALUE || + opcode == _GUARD_IP_YIELD_VALUE) { exit_op = _DYNAMIC_EXIT; unique_target = true; } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 5f7d600b3cd2a9..9fc77dd4b6ed0c 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -3426,7 +3426,15 @@ break; } - case _GUARD_IP: { + case _GUARD_IP_PUSH_FRAME: { + break; + } + + case _GUARD_IP_YIELD_VALUE: { + break; + } + + case _GUARD_IP_RETURN_VALUE: { break; } diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index e95a070b163f4c..4bf04154df1623 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -65,7 +65,6 @@ def __init__(self, out: CWriter, labels: dict[str, Label]): self._replacers["oparg"] = self.oparg self._replacers["JUMPBY"] = self.jumpby self._replacers["DISPATCH"] = self.dispatch - self._replacers["LOAD_IP"] = self.load_ip def goto_error(self, offset: int, storage: Storage) -> str: # To do: Add jump targets for popping values. @@ -170,19 +169,6 @@ def dispatch( next(tkn_iter) return False - def load_ip( - self, - tkn: Token, - tkn_iter: TokenIterator, - uop: CodeSection, - storage: Storage, - inst: Instruction | None, - ) -> bool: - self.out.start_line() - self.emit("frame->instr_ptr += ") - emit_to(self.out, tkn_iter, "SEMI") - self.emit(";\n") - return True def write_uop(uop: Uop, emitter: Emitter, stack: Stack) -> Stack: locals: dict[str, Local] = {} From 01c2d73dff3c9936d76d87454bc67af8559440b9 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 28 Oct 2025 00:12:59 +0000 Subject: [PATCH 127/190] fix backoff counters --- Python/bytecodes.c | 1 + Python/generated_cases.c.h | 1 + 2 files changed, 2 insertions(+) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index b3977e5613b4dc..96772a772f242f 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2956,6 +2956,7 @@ dummy_func( if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD_JIT && next_instr->op.code != ENTER_EXECUTOR) { + this_instr[1].counter = restart_backoff_counter(counter); if (tstate->interp->jit_state.code_buffer == NULL) { tstate->interp->jit_state.code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); if (tstate->interp->jit_state.code_buffer == NULL) { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index b97f4987c80fdd..7daab9556e74bd 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -7682,6 +7682,7 @@ if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD_JIT && next_instr->op.code != ENTER_EXECUTOR) { + this_instr[1].counter = restart_backoff_counter(counter); if (tstate->interp->jit_state.code_buffer == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); tstate->interp->jit_state.code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); From 1d3aed16382a4e5d20a9bbf2f08620a6a0fc1c10 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 28 Oct 2025 10:37:51 +0000 Subject: [PATCH 128/190] properly restore exponential backoffs --- Include/internal/pycore_interp_structs.h | 2 ++ Include/internal/pycore_optimizer.h | 3 ++- Python/bytecodes.c | 6 ++---- Python/ceval.c | 23 +++++++++++++++++++++++ Python/executor_cases.c.h | 3 +-- Python/generated_cases.c.h | 3 +-- Python/optimizer.c | 6 +++++- 7 files changed, 36 insertions(+), 10 deletions(-) diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 5c9f25eff6c951..546cc98d166c61 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -775,6 +775,8 @@ typedef struct _PyJitTracerState { _Py_CODEUNIT *prev_instr; PyCodeObject *prev_instr_code; // Strong struct _PyExitData *prev_exit; + struct _PyExecutorObject *prev_executor; // Strong + _Py_CODEUNIT *jump_backward_instr; _PyInterpreterFrame *prev_instr_frame; _PyBloomFilter dependencies; } _PyJitTracerState; diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 5208fa9aab1db7..2f51485e93fcb5 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -365,7 +365,8 @@ int _PyJit_translate_single_bytecode_to_trace(PyThreadState *tstate, _PyInterpre int _PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *curr_instr, _Py_CODEUNIT *insert_exec_instr, - _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit, int oparg); + _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit, + _PyExecutorObject *prev_exec, int oparg); void _PyJit_FinalizeTracing(PyThreadState *tstate); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 96772a772f242f..beed5e091b6f30 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2956,7 +2956,6 @@ dummy_func( if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD_JIT && next_instr->op.code != ENTER_EXECUTOR) { - this_instr[1].counter = restart_backoff_counter(counter); if (tstate->interp->jit_state.code_buffer == NULL) { tstate->interp->jit_state.code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); if (tstate->interp->jit_state.code_buffer == NULL) { @@ -2972,7 +2971,7 @@ dummy_func( oparg >>= 8; insert_exec_at--; } - int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, oparg); + int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, NULL, oparg); if (succ) { ENTER_TRACING(); } @@ -5432,8 +5431,7 @@ dummy_func( // Note: it's safe to use target->op.arg here instead of the oparg given by EXTENDED_ARG. // The invariant in the optimizer is the deopt target always points back to the first EXTENDED_ARG. // So setting it to anything else is wrong. - int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, target->op.arg); - exit->temperature = initial_temperature_backoff_counter(); + int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, previous_executor, target->op.arg); if (succ) { GOTO_TIER_ONE_CONTINUE_TRACING(target); } diff --git a/Python/ceval.c b/Python/ceval.c index 18fe74bcc82920..29a110ce28da6c 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -998,6 +998,29 @@ bail_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame) if (!_PyErr_Occurred(tstate) && !_is_sys_tracing) { err = _PyOptimizer_Optimize(frame, tstate); } + // Deal with backoffs + _PyExitData *exit = tstate->interp->jit_state.prev_exit; + if (exit == NULL) { + // We hold a strong reference to the code object, so the instruction won't be freed. + if (err <= 0) { + assert(tstate->interp->jit_state.jump_backward_instr->op.code == JUMP_BACKWARD_JIT); + _Py_BackoffCounter counter = tstate->interp->jit_state.jump_backward_instr[1].counter; + tstate->interp->jit_state.jump_backward_instr[1].counter = restart_backoff_counter(counter); + } + else { + tstate->interp->jit_state.jump_backward_instr[1].counter = initial_jump_backoff_counter(); + } + } + else { + // Likewise, we hold a strong reference to the executor containing this exit, so the exit is guaranteed + // to be valid to access. + if (err <= 0) { + exit->temperature = restart_backoff_counter(exit->temperature); + } + else { + exit->temperature = initial_temperature_backoff_counter(); + } + } _PyJit_FinalizeTracing(tstate); return err; } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 7f915e31b12108..62ddb22a4b0b52 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7480,8 +7480,7 @@ _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); assert(tstate->current_executor == (PyObject *)previous_executor); int chain_depth = previous_executor->vm_data.chain_depth + 1; - int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, target->op.arg); - exit->temperature = initial_temperature_backoff_counter(); + int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, previous_executor, target->op.arg); if (succ) { GOTO_TIER_ONE_CONTINUE_TRACING(target); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 7daab9556e74bd..c0dbf3091f45df 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -7682,7 +7682,6 @@ if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD_JIT && next_instr->op.code != ENTER_EXECUTOR) { - this_instr[1].counter = restart_backoff_counter(counter); if (tstate->interp->jit_state.code_buffer == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); tstate->interp->jit_state.code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); @@ -7698,7 +7697,7 @@ oparg >>= 8; insert_exec_at--; } - int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, oparg); + int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, NULL, oparg); if (succ) { ENTER_TRACING(); } diff --git a/Python/optimizer.c b/Python/optimizer.c index e6955edbc194c3..166d73d4409ca8 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -949,7 +949,7 @@ _PyJit_translate_single_bytecode_to_trace( // Returns 0 for do not enter tracing, 1 on enter tracing. int -_PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *curr_instr, _Py_CODEUNIT *insert_exec_instr, _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit, int oparg) +_PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *curr_instr, _Py_CODEUNIT *insert_exec_instr, _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit, _PyExecutorObject *prev_exec, int oparg) { // A recursive trace. // Don't trace into the inner call because it will stomp on the previous trace, causing endless retraces. @@ -991,6 +991,9 @@ _PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _ tstate->interp->jit_state.prev_instr_oparg = oparg; tstate->interp->jit_state.prev_instr_stacklevel = curr_stackdepth; tstate->interp->jit_state.prev_instr_is_super = false; + assert(curr_instr->op.code == JUMP_BACKWARD_JIT || (prev_exec != NULL && exit != NULL)); + tstate->interp->jit_state.jump_backward_instr = curr_instr; + tstate->interp->jit_state.prev_executor = (_PyExecutorObject *)Py_XNewRef(prev_exec); _Py_BloomFilter_Init(&tstate->interp->jit_state.dependencies); return 1; } @@ -1001,6 +1004,7 @@ _PyJit_FinalizeTracing(PyThreadState *tstate) Py_CLEAR(tstate->interp->jit_state.initial_code); Py_CLEAR(tstate->interp->jit_state.initial_func); Py_CLEAR(tstate->interp->jit_state.prev_instr_code); + Py_CLEAR(tstate->interp->jit_state.prev_executor); tstate->interp->jit_state.code_curr_size = 2; tstate->interp->jit_state.code_max_size = UOP_MAX_TRACE_LENGTH - 1; } From 7bfac269263af23dbf6180554627a7c2f0d1c7a9 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 28 Oct 2025 10:39:33 +0000 Subject: [PATCH 129/190] fix test --- Lib/test/test_capi/test_opt.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 265bb2941399be..10b308c0aafab5 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1241,8 +1241,14 @@ class Bar: pass res, ex = self._run_with_optimizer(thing, Foo()) - # Cleaned up by the invalidation. - self.assertIsNone(ex) + opnames = list(iter_opnames(ex)) + self.assertIsNotNone(ex) + self.assertEqual(res, TIER2_THRESHOLD * 6 + 1) + call = opnames.index("_CALL_BUILTIN_FAST") + load_attr_top = opnames.index("_POP_TOP_LOAD_CONST_INLINE_BORROW", 0, call) + load_attr_bottom = opnames.index("_POP_TOP_LOAD_CONST_INLINE_BORROW", call) + self.assertEqual(opnames[:load_attr_top].count("_GUARD_TYPE_VERSION"), 1) + self.assertEqual(opnames[call:load_attr_bottom].count("_CHECK_VALIDITY"), 2) def test_guard_type_version_removed_escaping(self): From ac0711dcad90a03cc1a1f1c0a0f0c877bec074ce Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 28 Oct 2025 11:10:41 +0000 Subject: [PATCH 130/190] fix backoff for previous exits --- Python/optimizer.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Python/optimizer.c b/Python/optimizer.c index 166d73d4409ca8..4a61509112ba09 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -172,10 +172,9 @@ _PyOptimizer_Optimize( else { executor->vm_data.code = NULL; } - if (chain_depth > 0) { - _PyExitData *prev_exit = tstate->interp->jit_state.prev_exit; - assert(prev_exit != NULL); - prev_exit->executor = executor;; + _PyExitData *prev_exit = tstate->interp->jit_state.prev_exit; + if (prev_exit != NULL) { + prev_exit->executor = executor; } executor->vm_data.chain_depth = chain_depth; assert(executor->vm_data.valid); @@ -981,7 +980,7 @@ _PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _ tstate->interp->jit_state.initial_func = (PyFunctionObject *)Py_NewRef(_PyFrame_GetFunction(frame)); tstate->interp->jit_state.prev_exit = exit; tstate->interp->jit_state.initial_stack_depth = curr_stackdepth; - tstate->interp->jit_state.initial_chain_depth = chain_depth % MAX_CHAIN_DEPTH; + tstate->interp->jit_state.initial_chain_depth = chain_depth; tstate->interp->jit_state.prev_instr_frame = frame; tstate->interp->jit_state.dependencies_still_valid = true; tstate->interp->jit_state.specialize_counter = 0; From da6605858b6604d3826056cce047696ed581fc3f Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 28 Oct 2025 11:11:39 +0000 Subject: [PATCH 131/190] remove faulty assertion --- Python/ceval.c | 1 - 1 file changed, 1 deletion(-) diff --git a/Python/ceval.c b/Python/ceval.c index 29a110ce28da6c..76b09488c3b854 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1003,7 +1003,6 @@ bail_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame) if (exit == NULL) { // We hold a strong reference to the code object, so the instruction won't be freed. if (err <= 0) { - assert(tstate->interp->jit_state.jump_backward_instr->op.code == JUMP_BACKWARD_JIT); _Py_BackoffCounter counter = tstate->interp->jit_state.jump_backward_instr[1].counter; tstate->interp->jit_state.jump_backward_instr[1].counter = restart_backoff_counter(counter); } From 692a992f026f33bd4284cefbe584aedac6f37212 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 28 Oct 2025 22:13:19 +0000 Subject: [PATCH 132/190] Fix INTERPRETER_EXIT tracing --- Include/internal/pycore_ceval.h | 3 + Include/internal/pycore_code.h | 6 ++ .../pycore_global_objects_fini_generated.h | 1 + Include/internal/pycore_global_strings.h | 1 + Include/internal/pycore_opcode_metadata.h | 4 +- .../internal/pycore_runtime_init_generated.h | 1 + .../internal/pycore_unicodeobject_generated.h | 4 + Python/bytecodes.c | 54 ++++++++------ Python/ceval.c | 53 ++++++++++--- Python/generated_cases.c.h | 74 +++++++++++-------- Python/optimizer.c | 31 +++----- Python/specialize.c | 6 -- 12 files changed, 148 insertions(+), 90 deletions(-) diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 102a378f8f08bc..61917b01b7cb13 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -391,6 +391,9 @@ _PyForIter_VirtualIteratorNext(PyThreadState* tstate, struct _PyInterpreterFrame #define SPECIAL___AEXIT__ 3 #define SPECIAL_MAX 3 +struct _PyCode12 _PyCode_DEF(12); +PyAPI_DATA(const struct _PyCode12) _PyEntryFrameCode; + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 2d7d81d491c157..434f47374f793f 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -672,6 +672,12 @@ PyAPI_FUNC(int) _PyCode_ReturnsOnlyNone(PyCodeObject *); * compare bytes and str which can raise a BytesWarning exception. */ extern PyObject* _PyCode_ConstantKey(PyObject *obj); +#define NO_LOC_4 (128 | (PY_CODE_LOCATION_INFO_NONE << 3) | 3) + +static const PyBytesObject no_location = { + PyVarObject_HEAD_INIT(&PyBytes_Type, 1) + .ob_sval = { NO_LOC_4 } +}; #ifdef __cplusplus } diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 92ded14891a101..4e0a2a452728da 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -1344,6 +1344,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(TextIOWrapper)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(True)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(WarningMessage)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_PyEval_EvalFrameDefault)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_WindowsConsoleIO)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__IOBase_closed)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__abc_tpflags__)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index cd21b0847b7cdd..c150948e432ca5 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -67,6 +67,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(TextIOWrapper) STRUCT_FOR_ID(True) STRUCT_FOR_ID(WarningMessage) + STRUCT_FOR_ID(_PyEval_EvalFrameDefault) STRUCT_FOR_ID(_WindowsConsoleIO) STRUCT_FOR_ID(__IOBase_closed) STRUCT_FOR_ID(__abc_tpflags__) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 8dda65a55b3453..aa99a0c50406fb 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1148,7 +1148,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, [END_FOR] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG }, [END_SEND] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_PURE_FLAG }, - [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [FORMAT_SIMPLE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1187,7 +1187,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INTERPRETER_EXIT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [INTERPRETER_EXIT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [IS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_BACKWARD_JIT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 50d82d0a365037..0212a141fe2e36 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1342,6 +1342,7 @@ extern "C" { INIT_ID(TextIOWrapper), \ INIT_ID(True), \ INIT_ID(WarningMessage), \ + INIT_ID(_PyEval_EvalFrameDefault), \ INIT_ID(_WindowsConsoleIO), \ INIT_ID(__IOBase_closed), \ INIT_ID(__abc_tpflags__), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index b4d920154b6e83..e1a5838dd07444 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -56,6 +56,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(_PyEval_EvalFrameDefault); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_WindowsConsoleIO); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index beed5e091b6f30..d665f7db9c5dda 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -170,7 +170,9 @@ dummy_func( op(_QUICKEN_RESUME, (--)) { #if ENABLE_SPECIALIZATION_FT - if (tstate->tracing == 0 && this_instr->op.code == RESUME) { + PyCodeObject *code = _PyFrame_GetCode(frame); + if (tstate->tracing == 0 && this_instr->op.code == RESUME && + code != (PyCodeObject *)&_Py_InitCleanup && code != (PyCodeObject *)&_PyEntryFrameCode) { FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); } #endif /* ENABLE_SPECIALIZATION_FT */ @@ -1218,19 +1220,33 @@ dummy_func( tstate->current_frame = frame->previous; assert(!_PyErr_Occurred(tstate)); PyObject *result = PyStackRef_AsPyObjectSteal(retval); + if (IS_JIT_TRACING()) { +#if _Py_TIER2 + _PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr); + LEAVE_TRACING(); + int err = bail_tracing_and_jit(tstate, frame); + if (err < 0) { + Py_DECREF(result); + ERROR_IF(true); + } + return result; +#endif + } + else { #if !_Py_TAIL_CALL_INTERP - assert(frame == &entry.frame); + assert(frame == &entry.frame); #endif #ifdef _Py_TIER2 - _PyStackRef executor = frame->localsplus[0]; - assert(tstate->current_executor == NULL); - if (!PyStackRef_IsNull(executor)) { - tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor); - PyStackRef_CLOSE(executor); - } + _PyStackRef executor = frame->localsplus[0]; + assert(tstate->current_executor == NULL); + if (!PyStackRef_IsNull(executor)) { + tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor); + PyStackRef_CLOSE(executor); + } #endif - LLTRACE_RESUME_FRAME(); - return result; + LLTRACE_RESUME_FRAME(); + return result; + } } // The stack effect here is a bit misleading. @@ -3020,15 +3036,6 @@ dummy_func( tier1 inst(ENTER_EXECUTOR, (--)) { #ifdef _Py_TIER2 - // We want to end any current trace here, before we possibly need - // to start tracing new ones due to recursive traces in any inner C functions - // in tier2 code. - if (IS_JIT_TRACING()) { - _PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr); - LEAVE_TRACING(); - int err = bail_tracing_and_jit(tstate, frame); - ERROR_IF(err < 0); - } PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; assert(executor->vm_data.index == INSTR_OFFSET() - 1); @@ -3038,14 +3045,19 @@ dummy_func( /* If the eval breaker is set then stay in tier 1. * This avoids any potentially infinite loops * involving _RESUME_CHECK */ - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + if (IS_JIT_TRACING() || _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { opcode = executor->vm_data.opcode; oparg = (oparg & ~255) | executor->vm_data.oparg; next_instr = this_instr; if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); } - DISPATCH_GOTO(); + if (IS_JIT_TRACING()) { + DISPATCH_GOTO_NON_TRACING(); + } + else { + DISPATCH_GOTO(); + } } assert(executor != tstate->interp->cold_executor); tstate->jit_exit = NULL; diff --git a/Python/ceval.c b/Python/ceval.c index 76b09488c3b854..f2dd8fc6e8d792 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -931,14 +931,39 @@ int _Py_CheckRecursiveCallPy( return 0; } -static const _Py_CODEUNIT _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS[] = { - /* Put a NOP at the start, so that the IP points into - * the code, rather than before it */ - { .op.code = NOP, .op.arg = 0 }, - { .op.code = INTERPRETER_EXIT, .op.arg = 0 }, /* reached on return */ - { .op.code = NOP, .op.arg = 0 }, - { .op.code = INTERPRETER_EXIT, .op.arg = 0 }, /* reached on yield */ - { .op.code = RESUME, .op.arg = RESUME_OPARG_DEPTH1_MASK | RESUME_AT_FUNC_START } + +#ifdef Py_GIL_DISABLED +static _PyCodeArray emtry_cleanup_tlbc = { + .size = 1, + .entries = {(char*) &_PyEntryFrameCode.co_code_adaptive}, +}; +#endif + +const struct _PyCode12 _PyEntryFrameCode = { + _PyVarObject_HEAD_INIT(&PyCode_Type, 5), + .co_consts = (PyObject *)&_Py_SINGLETON(tuple_empty), + .co_names = (PyObject *)&_Py_SINGLETON(tuple_empty), + .co_exceptiontable = (PyObject *)&_Py_SINGLETON(bytes_empty), + .co_flags = CO_OPTIMIZED | CO_NO_MONITORING_EVENTS, + .co_localsplusnames = (PyObject *)&_Py_SINGLETON(tuple_empty), + .co_localspluskinds = (PyObject *)&_Py_SINGLETON(bytes_empty), + .co_filename = &_Py_ID(_PyEval_EvalFrameDefault), + .co_name = &_Py_ID(_PyEval_EvalFrameDefault), + .co_qualname = &_Py_ID(_PyEval_EvalFrameDefault), + .co_linetable = (PyObject *)&no_location, + ._co_firsttraceable = 4, + .co_stacksize = 2, + .co_framesize = 2 + FRAME_SPECIALS_SIZE, +#ifdef Py_GIL_DISABLED + .co_tlbc = &emtry_cleanup_tlbc, +#endif + .co_code_adaptive = { + NOP, 0, + INTERPRETER_EXIT, 0, /* reached on return */ + NOP, 0, + INTERPRETER_EXIT, 0, /* reached on yield */ + RESUME, RESUME_OPARG_DEPTH1_MASK | RESUME_AT_FUNC_START + } }; #ifdef Py_DEBUG @@ -1014,7 +1039,13 @@ bail_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame) // Likewise, we hold a strong reference to the executor containing this exit, so the exit is guaranteed // to be valid to access. if (err <= 0) { - exit->temperature = restart_backoff_counter(exit->temperature); + // Some opcodes will forever be unchanged. Don't ever bother specializing for them ever again. + if (tstate->interp->jit_state.prev_instr->op.code == INTERPRETER_EXIT) { + exit->temperature = initial_unreachable_backoff_counter(); + } + else { + exit->temperature = restart_backoff_counter(exit->temperature); + } } else { exit->temperature = initial_temperature_backoff_counter(); @@ -1105,8 +1136,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int entry.frame.f_globals = (PyObject*)0xaaa3; entry.frame.f_builtins = (PyObject*)0xaaa4; #endif - entry.frame.f_executable = PyStackRef_None; - entry.frame.instr_ptr = (_Py_CODEUNIT *)_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS + 1; + entry.frame.f_executable = PyStackRef_FromPyObjectBorrow((PyObject *)&_PyEntryFrameCode); + entry.frame.instr_ptr = ((_Py_CODEUNIT *)_PyEntryFrameCode.co_code_adaptive) + 1; entry.frame.stackpointer = entry.stack; entry.frame.owner = FRAME_OWNED_BY_INTERPRETER; entry.frame.visited = 0; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index c0dbf3091f45df..10a1793ad284cb 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5491,33 +5491,25 @@ INSTRUCTION_STATS(ENTER_EXECUTOR); opcode = ENTER_EXECUTOR; #ifdef _Py_TIER2 - - if (IS_JIT_TRACING()) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - LEAVE_TRACING(); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = bail_tracing_and_jit(tstate, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - JUMP_TO_LABEL(error); - } - } PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; assert(executor->vm_data.index == INSTR_OFFSET() - 1); assert(executor->vm_data.code == code); assert(executor->vm_data.valid); assert(tstate->current_executor == NULL); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + if (IS_JIT_TRACING() || _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { opcode = executor->vm_data.opcode; oparg = (oparg & ~255) | executor->vm_data.oparg; next_instr = this_instr; if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); } - DISPATCH_GOTO(); + if (IS_JIT_TRACING()) { + DISPATCH_GOTO_NON_TRACING(); + } + else { + DISPATCH_GOTO(); + } } assert(executor != tstate->interp->cold_executor); tstate->jit_exit = NULL; @@ -7553,24 +7545,46 @@ tstate->current_frame = frame->previous; assert(!_PyErr_Occurred(tstate)); PyObject *result = PyStackRef_AsPyObjectSteal(retval); - #if !_Py_TAIL_CALL_INTERP - assert(frame == &entry.frame); - #endif - #ifdef _Py_TIER2 - _PyStackRef executor = frame->localsplus[0]; - assert(tstate->current_executor == NULL); - if (!PyStackRef_IsNull(executor)) { - tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor); + if (IS_JIT_TRACING()) { + #if _Py_TIER2 stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(executor); + _PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += 1; + LEAVE_TRACING(); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = bail_tracing_and_jit(tstate, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_DECREF(result); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } + return result; + #endif + } + else { + #if !_Py_TAIL_CALL_INTERP + assert(frame == &entry.frame); + #endif + #ifdef _Py_TIER2 + _PyStackRef executor = frame->localsplus[0]; + assert(tstate->current_executor == NULL); + if (!PyStackRef_IsNull(executor)) { + tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor); + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyStackRef_CLOSE(executor); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += 1; + } + #endif + LLTRACE_RESUME_FRAME(); + return result; } - #endif - LLTRACE_RESUME_FRAME(); - return result; } TARGET(IS_OP) { @@ -10325,7 +10339,9 @@ // _QUICKEN_RESUME { #if ENABLE_SPECIALIZATION_FT - if (tstate->tracing == 0 && this_instr->op.code == RESUME) { + PyCodeObject *code = _PyFrame_GetCode(frame); + if (tstate->tracing == 0 && this_instr->op.code == RESUME && + code != (PyCodeObject *)&_Py_InitCleanup && code != (PyCodeObject *)&_PyEntryFrameCode) { FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); } #endif /* ENABLE_SPECIALIZATION_FT */ diff --git a/Python/optimizer.c b/Python/optimizer.c index 4a61509112ba09..a3ae74fd944ae8 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -34,7 +34,7 @@ static bool has_space_for_executor(PyCodeObject *code, _Py_CODEUNIT *instr) { - if (code == (PyCodeObject *)&_Py_InitCleanup) { + if (code == (PyCodeObject *)&_Py_InitCleanup || code == (PyCodeObject *)&_PyEntryFrameCode) { return false; } if (instr->op.code == ENTER_EXECUTOR) { @@ -133,11 +133,6 @@ _PyOptimizer_Optimize( bool progress_needed = chain_depth == 0; PyCodeObject *code = (PyCodeObject *)tstate->interp->jit_state.initial_code; _Py_CODEUNIT *start = tstate->interp->jit_state.insert_exec_instr; - // A recursive trace might've cleared the values. In that case, bail. - if (code == NULL) { - interp->compiling = false; - return 0; - } if (progress_needed && !has_space_for_executor(code, start)) { interp->compiling = false; return 0; @@ -619,11 +614,6 @@ _PyJit_translate_single_bytecode_to_trace( } if (opcode == ENTER_EXECUTOR) { - int is_first_instr = tstate->interp->jit_state.close_loop_instr == next_instr || tstate->interp->jit_state.insert_exec_instr == next_instr; - if (is_first_instr && tstate->interp->jit_state.code_curr_size > 5) { - ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0, 0); - goto done; - } goto full; } @@ -658,11 +648,6 @@ _PyJit_translate_single_bytecode_to_trace( if (opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO) { DPRINTF(2, "Unsupported: strange control-flow\n"); - goto unsupported; - } - - if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) { - DPRINTF(2, "Unsupported: frame owned by interpreter\n"); unsupported: { // Rewind to previous instruction and replace with _EXIT_TRACE. @@ -977,7 +962,7 @@ _PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _ tstate->interp->jit_state.insert_exec_instr = insert_exec_instr; tstate->interp->jit_state.close_loop_instr = close_loop_instr; tstate->interp->jit_state.initial_code = (PyCodeObject *)Py_NewRef(code); - tstate->interp->jit_state.initial_func = (PyFunctionObject *)Py_NewRef(_PyFrame_GetFunction(frame)); + tstate->interp->jit_state.initial_func = (PyFunctionObject *)Py_XNewRef(PyStackRef_AsPyObjectBorrow(frame->f_funcobj)); tstate->interp->jit_state.prev_exit = exit; tstate->interp->jit_state.initial_stack_depth = curr_stackdepth; tstate->interp->jit_state.initial_chain_depth = chain_depth; @@ -1580,11 +1565,15 @@ _Py_ExecutorDetach(_PyExecutorObject *executor) return; } _Py_CODEUNIT *instruction = &_PyCode_CODE(code)[executor->vm_data.index]; - assert(instruction->op.code == ENTER_EXECUTOR); int index = instruction->op.arg; - assert(code->co_executors->executors[index] == executor); - instruction->op.code = executor->vm_data.opcode; - instruction->op.arg = executor->vm_data.oparg; + // Due to a combination of re-entrancy and tracing, it's possible for an + // instruction to no longer be ENTER_EXECUTOR. In which case, no-op. + if (instruction->op.code == ENTER_EXECUTOR) { + assert(instruction->op.code == ENTER_EXECUTOR); + assert(code->co_executors->executors[index] == executor); + instruction->op.code = executor->vm_data.opcode; + instruction->op.arg = executor->vm_data.oparg; + } executor->vm_data.code = NULL; code->co_executors->executors[index] = NULL; Py_DECREF(executor); diff --git a/Python/specialize.c b/Python/specialize.c index a1c5dedd61563b..0583873e09d2be 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -3196,12 +3196,6 @@ _Py_GatherStats_GetIter(_PyStackRef iterable) * be lifted. */ -#define NO_LOC_4 (128 | (PY_CODE_LOCATION_INFO_NONE << 3) | 3) - -static const PyBytesObject no_location = { - PyVarObject_HEAD_INIT(&PyBytes_Type, 1) - .ob_sval = { NO_LOC_4 } -}; #ifdef Py_GIL_DISABLED static _PyCodeArray init_cleanup_tlbc = { From 84ee07b9cd50a328509803d36fda8c6884954581 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 29 Oct 2025 14:45:05 +0000 Subject: [PATCH 133/190] Add _GUARD_IP autogenerator --- Include/internal/pycore_opcode_metadata.h | 2 +- Include/internal/pycore_uop_ids.h | 317 +++++++++++----------- Include/internal/pycore_uop_metadata.h | 10 +- Python/bytecodes.c | 58 ++-- Python/executor_cases.c.h | 73 ++--- Python/generated_cases.c.h | 40 +-- Python/optimizer.c | 22 +- Python/optimizer_cases.c.h | 6 +- Tools/cases_generator/tier2_generator.py | 31 +++ 9 files changed, 305 insertions(+), 254 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index aa99a0c50406fb..c774807b355961 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1148,7 +1148,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, [END_FOR] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG }, [END_SEND] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_PURE_FLAG }, - [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [FORMAT_SIMPLE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 067da4209f0102..52aae3b3af64bf 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -137,49 +137,50 @@ extern "C" { #define _GUARD_DORV_NO_DICT 392 #define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 393 #define _GUARD_GLOBALS_VERSION 394 -#define _GUARD_IP_PUSH_FRAME 395 +#define _GUARD_IP_RETURN_GENERATOR 395 #define _GUARD_IP_RETURN_VALUE 396 #define _GUARD_IP_YIELD_VALUE 397 -#define _GUARD_IS_FALSE_POP 398 -#define _GUARD_IS_NONE_POP 399 -#define _GUARD_IS_NOT_NONE_POP 400 -#define _GUARD_IS_TRUE_POP 401 -#define _GUARD_KEYS_VERSION 402 -#define _GUARD_NOS_DICT 403 -#define _GUARD_NOS_FLOAT 404 -#define _GUARD_NOS_INT 405 -#define _GUARD_NOS_LIST 406 -#define _GUARD_NOS_NOT_NULL 407 -#define _GUARD_NOS_NULL 408 -#define _GUARD_NOS_OVERFLOWED 409 -#define _GUARD_NOS_TUPLE 410 -#define _GUARD_NOS_UNICODE 411 -#define _GUARD_NOT_EXHAUSTED_LIST 412 -#define _GUARD_NOT_EXHAUSTED_RANGE 413 -#define _GUARD_NOT_EXHAUSTED_TUPLE 414 -#define _GUARD_THIRD_NULL 415 -#define _GUARD_TOS_ANY_SET 416 -#define _GUARD_TOS_DICT 417 -#define _GUARD_TOS_FLOAT 418 -#define _GUARD_TOS_INT 419 -#define _GUARD_TOS_LIST 420 -#define _GUARD_TOS_OVERFLOWED 421 -#define _GUARD_TOS_SLICE 422 -#define _GUARD_TOS_TUPLE 423 -#define _GUARD_TOS_UNICODE 424 -#define _GUARD_TYPE_VERSION 425 -#define _GUARD_TYPE_VERSION_AND_LOCK 426 -#define _HANDLE_PENDING_AND_DEOPT 427 +#define _GUARD_IP__PUSH_FRAME 398 +#define _GUARD_IS_FALSE_POP 399 +#define _GUARD_IS_NONE_POP 400 +#define _GUARD_IS_NOT_NONE_POP 401 +#define _GUARD_IS_TRUE_POP 402 +#define _GUARD_KEYS_VERSION 403 +#define _GUARD_NOS_DICT 404 +#define _GUARD_NOS_FLOAT 405 +#define _GUARD_NOS_INT 406 +#define _GUARD_NOS_LIST 407 +#define _GUARD_NOS_NOT_NULL 408 +#define _GUARD_NOS_NULL 409 +#define _GUARD_NOS_OVERFLOWED 410 +#define _GUARD_NOS_TUPLE 411 +#define _GUARD_NOS_UNICODE 412 +#define _GUARD_NOT_EXHAUSTED_LIST 413 +#define _GUARD_NOT_EXHAUSTED_RANGE 414 +#define _GUARD_NOT_EXHAUSTED_TUPLE 415 +#define _GUARD_THIRD_NULL 416 +#define _GUARD_TOS_ANY_SET 417 +#define _GUARD_TOS_DICT 418 +#define _GUARD_TOS_FLOAT 419 +#define _GUARD_TOS_INT 420 +#define _GUARD_TOS_LIST 421 +#define _GUARD_TOS_OVERFLOWED 422 +#define _GUARD_TOS_SLICE 423 +#define _GUARD_TOS_TUPLE 424 +#define _GUARD_TOS_UNICODE 425 +#define _GUARD_TYPE_VERSION 426 +#define _GUARD_TYPE_VERSION_AND_LOCK 427 +#define _HANDLE_PENDING_AND_DEOPT 428 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 428 -#define _INIT_CALL_PY_EXACT_ARGS 429 -#define _INIT_CALL_PY_EXACT_ARGS_0 430 -#define _INIT_CALL_PY_EXACT_ARGS_1 431 -#define _INIT_CALL_PY_EXACT_ARGS_2 432 -#define _INIT_CALL_PY_EXACT_ARGS_3 433 -#define _INIT_CALL_PY_EXACT_ARGS_4 434 -#define _INSERT_NULL 435 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 429 +#define _INIT_CALL_PY_EXACT_ARGS 430 +#define _INIT_CALL_PY_EXACT_ARGS_0 431 +#define _INIT_CALL_PY_EXACT_ARGS_1 432 +#define _INIT_CALL_PY_EXACT_ARGS_2 433 +#define _INIT_CALL_PY_EXACT_ARGS_3 434 +#define _INIT_CALL_PY_EXACT_ARGS_4 435 +#define _INSERT_NULL 436 #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER #define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION #define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD @@ -189,178 +190,178 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _IS_NONE 436 +#define _IS_NONE 437 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 437 -#define _ITER_CHECK_RANGE 438 -#define _ITER_CHECK_TUPLE 439 -#define _ITER_JUMP_LIST 440 -#define _ITER_JUMP_RANGE 441 -#define _ITER_JUMP_TUPLE 442 -#define _ITER_NEXT_LIST 443 -#define _ITER_NEXT_LIST_TIER_TWO 444 -#define _ITER_NEXT_RANGE 445 -#define _ITER_NEXT_TUPLE 446 +#define _ITER_CHECK_LIST 438 +#define _ITER_CHECK_RANGE 439 +#define _ITER_CHECK_TUPLE 440 +#define _ITER_JUMP_LIST 441 +#define _ITER_JUMP_RANGE 442 +#define _ITER_JUMP_TUPLE 443 +#define _ITER_NEXT_LIST 444 +#define _ITER_NEXT_LIST_TIER_TWO 445 +#define _ITER_NEXT_RANGE 446 +#define _ITER_NEXT_TUPLE 447 #define _JUMP_BACKWARD_NO_INTERRUPT JUMP_BACKWARD_NO_INTERRUPT -#define _JUMP_TO_TOP 447 +#define _JUMP_TO_TOP 448 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 448 -#define _LOAD_ATTR_CLASS 449 +#define _LOAD_ATTR 449 +#define _LOAD_ATTR_CLASS 450 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 450 -#define _LOAD_ATTR_METHOD_LAZY_DICT 451 -#define _LOAD_ATTR_METHOD_NO_DICT 452 -#define _LOAD_ATTR_METHOD_WITH_VALUES 453 -#define _LOAD_ATTR_MODULE 454 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 455 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 456 -#define _LOAD_ATTR_PROPERTY_FRAME 457 -#define _LOAD_ATTR_SLOT 458 -#define _LOAD_ATTR_WITH_HINT 459 +#define _LOAD_ATTR_INSTANCE_VALUE 451 +#define _LOAD_ATTR_METHOD_LAZY_DICT 452 +#define _LOAD_ATTR_METHOD_NO_DICT 453 +#define _LOAD_ATTR_METHOD_WITH_VALUES 454 +#define _LOAD_ATTR_MODULE 455 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 456 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 457 +#define _LOAD_ATTR_PROPERTY_FRAME 458 +#define _LOAD_ATTR_SLOT 459 +#define _LOAD_ATTR_WITH_HINT 460 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _LOAD_BYTECODE 460 +#define _LOAD_BYTECODE 461 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 461 -#define _LOAD_CONST_INLINE_BORROW 462 -#define _LOAD_CONST_UNDER_INLINE 463 -#define _LOAD_CONST_UNDER_INLINE_BORROW 464 +#define _LOAD_CONST_INLINE 462 +#define _LOAD_CONST_INLINE_BORROW 463 +#define _LOAD_CONST_UNDER_INLINE 464 +#define _LOAD_CONST_UNDER_INLINE_BORROW 465 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 465 -#define _LOAD_FAST_0 466 -#define _LOAD_FAST_1 467 -#define _LOAD_FAST_2 468 -#define _LOAD_FAST_3 469 -#define _LOAD_FAST_4 470 -#define _LOAD_FAST_5 471 -#define _LOAD_FAST_6 472 -#define _LOAD_FAST_7 473 +#define _LOAD_FAST 466 +#define _LOAD_FAST_0 467 +#define _LOAD_FAST_1 468 +#define _LOAD_FAST_2 469 +#define _LOAD_FAST_3 470 +#define _LOAD_FAST_4 471 +#define _LOAD_FAST_5 472 +#define _LOAD_FAST_6 473 +#define _LOAD_FAST_7 474 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR -#define _LOAD_FAST_BORROW 474 -#define _LOAD_FAST_BORROW_0 475 -#define _LOAD_FAST_BORROW_1 476 -#define _LOAD_FAST_BORROW_2 477 -#define _LOAD_FAST_BORROW_3 478 -#define _LOAD_FAST_BORROW_4 479 -#define _LOAD_FAST_BORROW_5 480 -#define _LOAD_FAST_BORROW_6 481 -#define _LOAD_FAST_BORROW_7 482 +#define _LOAD_FAST_BORROW 475 +#define _LOAD_FAST_BORROW_0 476 +#define _LOAD_FAST_BORROW_1 477 +#define _LOAD_FAST_BORROW_2 478 +#define _LOAD_FAST_BORROW_3 479 +#define _LOAD_FAST_BORROW_4 480 +#define _LOAD_FAST_BORROW_5 481 +#define _LOAD_FAST_BORROW_6 482 +#define _LOAD_FAST_BORROW_7 483 #define _LOAD_FAST_BORROW_LOAD_FAST_BORROW LOAD_FAST_BORROW_LOAD_FAST_BORROW #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 483 -#define _LOAD_GLOBAL_BUILTINS 484 -#define _LOAD_GLOBAL_MODULE 485 +#define _LOAD_GLOBAL 484 +#define _LOAD_GLOBAL_BUILTINS 485 +#define _LOAD_GLOBAL_MODULE 486 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 486 -#define _LOAD_SMALL_INT_0 487 -#define _LOAD_SMALL_INT_1 488 -#define _LOAD_SMALL_INT_2 489 -#define _LOAD_SMALL_INT_3 490 -#define _LOAD_SPECIAL 491 +#define _LOAD_SMALL_INT 487 +#define _LOAD_SMALL_INT_0 488 +#define _LOAD_SMALL_INT_1 489 +#define _LOAD_SMALL_INT_2 490 +#define _LOAD_SMALL_INT_3 491 +#define _LOAD_SPECIAL 492 #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _MAKE_CALLARGS_A_TUPLE 492 +#define _MAKE_CALLARGS_A_TUPLE 493 #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_WARM 493 +#define _MAKE_WARM 494 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 494 -#define _MAYBE_EXPAND_METHOD_KW 495 -#define _MONITOR_CALL 496 -#define _MONITOR_CALL_KW 497 -#define _MONITOR_JUMP_BACKWARD 498 -#define _MONITOR_RESUME 499 +#define _MAYBE_EXPAND_METHOD 495 +#define _MAYBE_EXPAND_METHOD_KW 496 +#define _MONITOR_CALL 497 +#define _MONITOR_CALL_KW 498 +#define _MONITOR_JUMP_BACKWARD 499 +#define _MONITOR_RESUME 500 #define _NOP NOP -#define _POP_CALL 500 -#define _POP_CALL_LOAD_CONST_INLINE_BORROW 501 -#define _POP_CALL_ONE 502 -#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 503 -#define _POP_CALL_TWO 504 -#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 505 +#define _POP_CALL 501 +#define _POP_CALL_LOAD_CONST_INLINE_BORROW 502 +#define _POP_CALL_ONE 503 +#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 504 +#define _POP_CALL_TWO 505 +#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 506 #define _POP_EXCEPT POP_EXCEPT #define _POP_ITER POP_ITER -#define _POP_JUMP_IF_FALSE 506 -#define _POP_JUMP_IF_TRUE 507 +#define _POP_JUMP_IF_FALSE 507 +#define _POP_JUMP_IF_TRUE 508 #define _POP_TOP POP_TOP -#define _POP_TOP_FLOAT 508 -#define _POP_TOP_INT 509 -#define _POP_TOP_LOAD_CONST_INLINE 510 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 511 -#define _POP_TOP_NOP 512 -#define _POP_TOP_UNICODE 513 -#define _POP_TWO 514 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW 515 +#define _POP_TOP_FLOAT 509 +#define _POP_TOP_INT 510 +#define _POP_TOP_LOAD_CONST_INLINE 511 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 512 +#define _POP_TOP_NOP 513 +#define _POP_TOP_UNICODE 514 +#define _POP_TWO 515 +#define _POP_TWO_LOAD_CONST_INLINE_BORROW 516 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 516 +#define _PUSH_FRAME 517 #define _PUSH_NULL PUSH_NULL -#define _PUSH_NULL_CONDITIONAL 517 -#define _PY_FRAME_GENERAL 518 -#define _PY_FRAME_KW 519 -#define _QUICKEN_RESUME 520 -#define _REPLACE_WITH_TRUE 521 +#define _PUSH_NULL_CONDITIONAL 518 +#define _PY_FRAME_GENERAL 519 +#define _PY_FRAME_KW 520 +#define _QUICKEN_RESUME 521 +#define _REPLACE_WITH_TRUE 522 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 522 -#define _SEND 523 -#define _SEND_GEN_FRAME 524 +#define _SAVE_RETURN_OFFSET 523 +#define _SEND 524 +#define _SEND_GEN_FRAME 525 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 525 -#define _STORE_ATTR 526 -#define _STORE_ATTR_INSTANCE_VALUE 527 -#define _STORE_ATTR_SLOT 528 -#define _STORE_ATTR_WITH_HINT 529 +#define _START_EXECUTOR 526 +#define _STORE_ATTR 527 +#define _STORE_ATTR_INSTANCE_VALUE 528 +#define _STORE_ATTR_SLOT 529 +#define _STORE_ATTR_WITH_HINT 530 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 530 -#define _STORE_FAST_0 531 -#define _STORE_FAST_1 532 -#define _STORE_FAST_2 533 -#define _STORE_FAST_3 534 -#define _STORE_FAST_4 535 -#define _STORE_FAST_5 536 -#define _STORE_FAST_6 537 -#define _STORE_FAST_7 538 +#define _STORE_FAST 531 +#define _STORE_FAST_0 532 +#define _STORE_FAST_1 533 +#define _STORE_FAST_2 534 +#define _STORE_FAST_3 535 +#define _STORE_FAST_4 536 +#define _STORE_FAST_5 537 +#define _STORE_FAST_6 538 +#define _STORE_FAST_7 539 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 539 -#define _STORE_SUBSCR 540 -#define _STORE_SUBSCR_DICT 541 -#define _STORE_SUBSCR_LIST_INT 542 -#define _SWAP 543 -#define _SWAP_2 544 -#define _SWAP_3 545 -#define _TIER2_RESUME_CHECK 546 -#define _TO_BOOL 547 +#define _STORE_SLICE 540 +#define _STORE_SUBSCR 541 +#define _STORE_SUBSCR_DICT 542 +#define _STORE_SUBSCR_LIST_INT 543 +#define _SWAP 544 +#define _SWAP_2 545 +#define _SWAP_3 546 +#define _TIER2_RESUME_CHECK 547 +#define _TO_BOOL 548 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT -#define _TO_BOOL_LIST 548 +#define _TO_BOOL_LIST 549 #define _TO_BOOL_NONE TO_BOOL_NONE -#define _TO_BOOL_STR 549 +#define _TO_BOOL_STR 550 #define _UNARY_INVERT UNARY_INVERT #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 550 -#define _UNPACK_SEQUENCE_LIST 551 -#define _UNPACK_SEQUENCE_TUPLE 552 -#define _UNPACK_SEQUENCE_TWO_TUPLE 553 +#define _UNPACK_SEQUENCE 551 +#define _UNPACK_SEQUENCE_LIST 552 +#define _UNPACK_SEQUENCE_TUPLE 553 +#define _UNPACK_SEQUENCE_TWO_TUPLE 554 #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 553 +#define MAX_UOP_ID 554 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 67c5827ffd0c00..c7fbfb0fafa2f6 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -337,9 +337,10 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_ERROR_POP_N] = HAS_ARG_FLAG, [_TIER2_RESUME_CHECK] = HAS_PERIODIC_FLAG, [_COLD_EXIT] = 0, - [_GUARD_IP_PUSH_FRAME] = HAS_EXIT_FLAG, + [_GUARD_IP__PUSH_FRAME] = HAS_EXIT_FLAG, [_GUARD_IP_YIELD_VALUE] = HAS_EXIT_FLAG, [_GUARD_IP_RETURN_VALUE] = HAS_EXIT_FLAG, + [_GUARD_IP_RETURN_GENERATOR] = HAS_EXIT_FLAG, [_DYNAMIC_EXIT] = HAS_ESCAPES_FLAG, }; @@ -476,9 +477,10 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_GUARD_DORV_NO_DICT] = "_GUARD_DORV_NO_DICT", [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT", [_GUARD_GLOBALS_VERSION] = "_GUARD_GLOBALS_VERSION", - [_GUARD_IP_PUSH_FRAME] = "_GUARD_IP_PUSH_FRAME", + [_GUARD_IP_RETURN_GENERATOR] = "_GUARD_IP_RETURN_GENERATOR", [_GUARD_IP_RETURN_VALUE] = "_GUARD_IP_RETURN_VALUE", [_GUARD_IP_YIELD_VALUE] = "_GUARD_IP_YIELD_VALUE", + [_GUARD_IP__PUSH_FRAME] = "_GUARD_IP__PUSH_FRAME", [_GUARD_IS_FALSE_POP] = "_GUARD_IS_FALSE_POP", [_GUARD_IS_NONE_POP] = "_GUARD_IS_NONE_POP", [_GUARD_IS_NOT_NONE_POP] = "_GUARD_IS_NOT_NONE_POP", @@ -1313,12 +1315,14 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _COLD_EXIT: return 0; - case _GUARD_IP_PUSH_FRAME: + case _GUARD_IP__PUSH_FRAME: return 0; case _GUARD_IP_YIELD_VALUE: return 0; case _GUARD_IP_RETURN_VALUE: return 0; + case _GUARD_IP_RETURN_GENERATOR: + return 0; case _DYNAMIC_EXIT: return 0; default: diff --git a/Python/bytecodes.c b/Python/bytecodes.c index d665f7db9c5dda..02860122959a4e 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2979,18 +2979,15 @@ dummy_func( DISPATCH(); } } - int _is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL); - if (!_is_sys_tracing) { - /* Back up over EXTENDED_ARGs so executor is inserted at the correct place */ - _Py_CODEUNIT *insert_exec_at = this_instr; - while (oparg > 255) { - oparg >>= 8; - insert_exec_at--; - } - int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, NULL, oparg); - if (succ) { - ENTER_TRACING(); - } + /* Back up over EXTENDED_ARGs so executor is inserted at the correct place */ + _Py_CODEUNIT *insert_exec_at = this_instr; + while (oparg > 255) { + oparg >>= 8; + insert_exec_at--; + } + int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, NULL, oparg); + if (succ) { + ENTER_TRACING(); } } else { @@ -3036,6 +3033,12 @@ dummy_func( tier1 inst(ENTER_EXECUTOR, (--)) { #ifdef _Py_TIER2 + if (IS_JIT_TRACING()) { + _PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr); + LEAVE_TRACING(); + int err = bail_tracing_and_jit(tstate, frame); + ERROR_IF(err < 0); + } PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; assert(executor->vm_data.index == INSTR_OFFSET() - 1); @@ -3045,19 +3048,14 @@ dummy_func( /* If the eval breaker is set then stay in tier 1. * This avoids any potentially infinite loops * involving _RESUME_CHECK */ - if (IS_JIT_TRACING() || _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { opcode = executor->vm_data.opcode; oparg = (oparg & ~255) | executor->vm_data.oparg; next_instr = this_instr; if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); } - if (IS_JIT_TRACING()) { - DISPATCH_GOTO_NON_TRACING(); - } - else { - DISPATCH_GOTO(); - } + DISPATCH_GOTO(); } assert(executor != tstate->interp->cold_executor); tstate->jit_exit = NULL; @@ -5454,22 +5452,24 @@ dummy_func( TIER2_TO_TIER2(exit->executor); } - tier2 op(_GUARD_IP_PUSH_FRAME, (ip/4 --)) { - EXIT_IF(frame->instr_ptr != (_Py_CODEUNIT *)ip); + tier2 op(_GUARD_IP__PUSH_FRAME, (ip/4 --)) { + // Implementation automatically inserted by Tools/cases/tier2_generator.py + EXIT_IF(true); } tier2 op(_GUARD_IP_YIELD_VALUE, (ip/4 --)) { - if (frame->instr_ptr + 1 + INLINE_CACHE_ENTRIES_SEND != (_Py_CODEUNIT *)ip) { - frame->instr_ptr += 1 + INLINE_CACHE_ENTRIES_SEND; - EXIT_IF(true); - } + // Implementation automatically inserted by Tools/cases/tier2_generator.py + EXIT_IF(true); } tier2 op(_GUARD_IP_RETURN_VALUE, (ip/4 --)) { - if (frame->instr_ptr + frame->return_offset != (_Py_CODEUNIT *)ip) { - frame->instr_ptr += frame->return_offset; - EXIT_IF(true); - } + // Implementation automatically inserted by Tools/cases/tier2_generator.py + EXIT_IF(true); + } + + tier2 op(_GUARD_IP_RETURN_GENERATOR, (ip/4 --)) { + // Implementation automatically inserted by Tools/cases/tier2_generator.py + EXIT_IF(true); } // Note: this is different than _COLD_EXIT/_EXIT_TRACE, as it may lead to multiple executors diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 62ddb22a4b0b52..1534dd6995f6cb 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7492,39 +7492,6 @@ break; } - case _GUARD_IP_PUSH_FRAME: { - PyObject *ip = (PyObject *)CURRENT_OPERAND0(); - if (frame->instr_ptr != (_Py_CODEUNIT *)ip) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - break; - } - - case _GUARD_IP_YIELD_VALUE: { - PyObject *ip = (PyObject *)CURRENT_OPERAND0(); - if (frame->instr_ptr + 1 + INLINE_CACHE_ENTRIES_SEND != (_Py_CODEUNIT *)ip) { - frame->instr_ptr += 1 + INLINE_CACHE_ENTRIES_SEND; - if (true) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - } - break; - } - - case _GUARD_IP_RETURN_VALUE: { - PyObject *ip = (PyObject *)CURRENT_OPERAND0(); - if (frame->instr_ptr + frame->return_offset != (_Py_CODEUNIT *)ip) { - frame->instr_ptr += frame->return_offset; - if (true) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - } - break; - } - case _DYNAMIC_EXIT: { PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); _Py_CODEUNIT *target = frame->instr_ptr; @@ -7554,4 +7521,44 @@ break; } + case _GUARD_IP_RETURN_VALUE: { + PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + if (frame->instr_ptr + (frame->return_offset) != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += (frame->return_offset); + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _GUARD_IP_YIELD_VALUE: { + PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + if (frame->instr_ptr + (1+INLINE_CACHE_ENTRIES_SEND) != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += (1+INLINE_CACHE_ENTRIES_SEND); + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _GUARD_IP__PUSH_FRAME: { + PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + if (frame->instr_ptr + (0) != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += (0); + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _GUARD_IP_RETURN_GENERATOR: { + PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + if (frame->instr_ptr + (frame->return_offset) != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += (frame->return_offset); + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + #undef TIER_TWO diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 10a1793ad284cb..5832fbf029db41 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5491,25 +5491,32 @@ INSTRUCTION_STATS(ENTER_EXECUTOR); opcode = ENTER_EXECUTOR; #ifdef _Py_TIER2 + if (IS_JIT_TRACING()) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr); + stack_pointer = _PyFrame_GetStackPointer(frame); + LEAVE_TRACING(); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = bail_tracing_and_jit(tstate, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + } PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; assert(executor->vm_data.index == INSTR_OFFSET() - 1); assert(executor->vm_data.code == code); assert(executor->vm_data.valid); assert(tstate->current_executor == NULL); - if (IS_JIT_TRACING() || _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { opcode = executor->vm_data.opcode; oparg = (oparg & ~255) | executor->vm_data.oparg; next_instr = this_instr; if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); } - if (IS_JIT_TRACING()) { - DISPATCH_GOTO_NON_TRACING(); - } - else { - DISPATCH_GOTO(); - } + DISPATCH_GOTO(); } assert(executor != tstate->interp->cold_executor); tstate->jit_exit = NULL; @@ -7704,17 +7711,14 @@ DISPATCH(); } } - int _is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL); - if (!_is_sys_tracing) { - _Py_CODEUNIT *insert_exec_at = this_instr; - while (oparg > 255) { - oparg >>= 8; - insert_exec_at--; - } - int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, NULL, oparg); - if (succ) { - ENTER_TRACING(); - } + _Py_CODEUNIT *insert_exec_at = this_instr; + while (oparg > 255) { + oparg >>= 8; + insert_exec_at--; + } + int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, NULL, oparg); + if (succ) { + ENTER_TRACING(); } } else { diff --git a/Python/optimizer.c b/Python/optimizer.c index a3ae74fd944ae8..d11108dcddfb93 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -887,9 +887,11 @@ _PyJit_translate_single_bytecode_to_trace( if (needs_guard_ip) { switch (trace[trace_length-1].opcode) { case _PUSH_FRAME: - ADD_TO_TRACE(_GUARD_IP_PUSH_FRAME, 0, (uintptr_t)next_instr, 0); + ADD_TO_TRACE(_GUARD_IP__PUSH_FRAME, 0, (uintptr_t)next_instr, 0); break; case _RETURN_GENERATOR: + ADD_TO_TRACE(_GUARD_IP_RETURN_GENERATOR, 0, (uintptr_t)next_instr, 0); + break; case _RETURN_VALUE: ADD_TO_TRACE(_GUARD_IP_RETURN_VALUE, 0, (uintptr_t)next_instr, 0); break; @@ -1077,9 +1079,11 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) exit_op = _DYNAMIC_EXIT; } else if ( - opcode == _GUARD_IP_PUSH_FRAME || + opcode == _GUARD_IP__PUSH_FRAME || opcode == _GUARD_IP_RETURN_VALUE || - opcode == _GUARD_IP_YIELD_VALUE) { + opcode == _GUARD_IP_YIELD_VALUE || + opcode == _GUARD_IP_RETURN_GENERATOR + ) { exit_op = _DYNAMIC_EXIT; unique_target = true; } @@ -1565,15 +1569,11 @@ _Py_ExecutorDetach(_PyExecutorObject *executor) return; } _Py_CODEUNIT *instruction = &_PyCode_CODE(code)[executor->vm_data.index]; + assert(instruction->op.code == ENTER_EXECUTOR); int index = instruction->op.arg; - // Due to a combination of re-entrancy and tracing, it's possible for an - // instruction to no longer be ENTER_EXECUTOR. In which case, no-op. - if (instruction->op.code == ENTER_EXECUTOR) { - assert(instruction->op.code == ENTER_EXECUTOR); - assert(code->co_executors->executors[index] == executor); - instruction->op.code = executor->vm_data.opcode; - instruction->op.arg = executor->vm_data.oparg; - } + assert(code->co_executors->executors[index] == executor); + instruction->op.code = executor->vm_data.opcode; + instruction->op.arg = executor->vm_data.oparg; executor->vm_data.code = NULL; code->co_executors->executors[index] = NULL; Py_DECREF(executor); diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 9fc77dd4b6ed0c..d26803e97d3257 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -3426,7 +3426,7 @@ break; } - case _GUARD_IP_PUSH_FRAME: { + case _GUARD_IP__PUSH_FRAME: { break; } @@ -3438,6 +3438,10 @@ break; } + case _GUARD_IP_RETURN_GENERATOR: { + break; + } + case _DYNAMIC_EXIT: { break; } diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index 4bf04154df1623..64f034746f6897 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -200,6 +200,33 @@ def write_uop(uop: Uop, emitter: Emitter, stack: Stack) -> Stack: SKIPS = ("_EXTENDED_ARG",) +def generate_guard_ips( + analysis: Analysis, + emitter: Tier2Emitter, +) -> None: + for name, uop in analysis.uops.items(): + for stmt in uop.body.body: + tkn_iter = iter(stmt.tokens()) + for token in tkn_iter: + if token.kind == "IDENTIFIER" and token.text == "LOAD_IP": + offset = [] + while token.kind != "SEMI": + offset.append(token.text) + token = next(tkn_iter) + # 1: to remove the LOAD_IP text + offset_str = "".join(offset[1:]) + emitter.emit(f"case _GUARD_IP_{name}: {{\n") + emitter.emit("PyObject *ip = (PyObject *)CURRENT_OPERAND0();\n") + emitter.emit(f"if (frame->instr_ptr + {offset_str} != (_Py_CODEUNIT *)ip) {{\n") + emitter.emit(f"frame->instr_ptr += {offset_str};\n") + emitter.emit(f"UOP_STAT_INC(uopcode, miss);\n") + emitter.emit("JUMP_TO_JUMP_TARGET();\n") + emitter.emit("}\n") + emitter.emit("break;\n") + emitter.emit("}\n") + emitter.emit("\n") + + def generate_tier2( filenames: list[str], analysis: Analysis, outfile: TextIO, lines: bool ) -> None: @@ -220,6 +247,8 @@ def generate_tier2( continue if uop.is_super(): continue + if name.startswith("_GUARD_IP"): + continue why_not_viable = uop.why_not_viable() if why_not_viable is not None: out.emit( @@ -236,6 +265,8 @@ def generate_tier2( out.start_line() out.emit("}") out.emit("\n\n") + + generate_guard_ips(analysis, emitter) outfile.write("#undef TIER_TWO\n") From 72368edd26c4da7460091c373305ab544c41b90e Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 29 Oct 2025 14:51:33 +0000 Subject: [PATCH 134/190] make windows happy --- Include/internal/pycore_code.h | 4 ---- Python/ceval.c | 4 ++++ Python/specialize.c | 5 +++++ 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 434f47374f793f..7970d674440c93 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -674,10 +674,6 @@ extern PyObject* _PyCode_ConstantKey(PyObject *obj); #define NO_LOC_4 (128 | (PY_CODE_LOCATION_INFO_NONE << 3) | 3) -static const PyBytesObject no_location = { - PyVarObject_HEAD_INIT(&PyBytes_Type, 1) - .ob_sval = { NO_LOC_4 } -}; #ifdef __cplusplus } diff --git a/Python/ceval.c b/Python/ceval.c index f2dd8fc6e8d792..09fd30b0c8a030 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -931,6 +931,10 @@ int _Py_CheckRecursiveCallPy( return 0; } +static const PyBytesObject no_location = { + PyVarObject_HEAD_INIT(&PyBytes_Type, 1) + .ob_sval = { NO_LOC_4 } +}; #ifdef Py_GIL_DISABLED static _PyCodeArray emtry_cleanup_tlbc = { diff --git a/Python/specialize.c b/Python/specialize.c index 0583873e09d2be..8a7ba00e2e101d 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -3204,6 +3204,11 @@ static _PyCodeArray init_cleanup_tlbc = { }; #endif +static const PyBytesObject no_location = { + PyVarObject_HEAD_INIT(&PyBytes_Type, 1) + .ob_sval = { NO_LOC_4 } +}; + const struct _PyCode8 _Py_InitCleanup = { _PyVarObject_HEAD_INIT(&PyCode_Type, 3), .co_consts = (PyObject *)&_Py_SINGLETON(tuple_empty), From 8e62fd19a78c6de15b447678ebabaea98e8d6873 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 29 Oct 2025 19:01:08 +0000 Subject: [PATCH 135/190] Remove check on RESUME --- Python/bytecodes.c | 3 +-- Python/generated_cases.c.h | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 02860122959a4e..ba3b1df3d1ca33 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -171,8 +171,7 @@ dummy_func( op(_QUICKEN_RESUME, (--)) { #if ENABLE_SPECIALIZATION_FT PyCodeObject *code = _PyFrame_GetCode(frame); - if (tstate->tracing == 0 && this_instr->op.code == RESUME && - code != (PyCodeObject *)&_Py_InitCleanup && code != (PyCodeObject *)&_PyEntryFrameCode) { + if (tstate->tracing == 0 && this_instr->op.code == RESUME) { FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); } #endif /* ENABLE_SPECIALIZATION_FT */ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 5832fbf029db41..1c7b1035a1a8ce 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -10344,8 +10344,7 @@ { #if ENABLE_SPECIALIZATION_FT PyCodeObject *code = _PyFrame_GetCode(frame); - if (tstate->tracing == 0 && this_instr->op.code == RESUME && - code != (PyCodeObject *)&_Py_InitCleanup && code != (PyCodeObject *)&_PyEntryFrameCode) { + if (tstate->tracing == 0 && this_instr->op.code == RESUME) { FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); } #endif /* ENABLE_SPECIALIZATION_FT */ From 92cc140f646dee91b8f2be9b8dcaa3b08ae276e8 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 29 Oct 2025 19:29:59 +0000 Subject: [PATCH 136/190] cleanup --- Include/internal/pycore_opcode_metadata.h | 3 ++ Tools/cases_generator/analyzer.py | 2 +- Tools/cases_generator/tier2_generator.py | 35 ----------------------- 3 files changed, 4 insertions(+), 36 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index c774807b355961..1c8c49a41060d3 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1781,12 +1781,15 @@ const uint8_t _PyOpcode_NeedsGuardIp[256] = { [RETURN_VALUE] = 1, [YIELD_VALUE] = 1, [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = 1, + [INSTRUMENTED_FOR_ITER] = 1, [RETURN_GENERATOR] = 1, [BINARY_OP_SUBSCR_GETITEM] = 1, [INSTRUMENTED_RETURN_VALUE] = 1, [SEND] = 1, [SEND_GEN] = 1, [INSTRUMENTED_YIELD_VALUE] = 1, + [INSTRUMENTED_END_ASYNC_FOR] = 1, + [END_ASYNC_FOR] = 1, [LOAD_ATTR_PROPERTY] = 1, [FOR_ITER_GEN] = 1, [CALL] = 1, diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 10dc7d5b31a214..3bd92cfa55a450 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -982,7 +982,7 @@ def compute_properties(op: parser.CodeDef) -> Properties: no_save_ip=no_save_ip, tier=tier_variable(op), needs_prev=variable_used(op, "prev_instr"), - needs_guard_ip=variable_used(op, "TIER2_STORE_IP") or variable_used(op, "LLTRACE_RESUME_FRAME") or variable_used(op, "DISPATCH_INLINED"), + needs_guard_ip=(unpredictable_jump and "replaced" not in op.annotations) or variable_used(op, "LLTRACE_RESUME_FRAME") or variable_used(op, "DISPATCH_INLINED"), unpredictable_jump=unpredictable_jump, ) diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index 64f034746f6897..8a9db068382a4f 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -63,8 +63,6 @@ class Tier2Emitter(Emitter): def __init__(self, out: CWriter, labels: dict[str, Label]): super().__init__(out, labels) self._replacers["oparg"] = self.oparg - self._replacers["JUMPBY"] = self.jumpby - self._replacers["DISPATCH"] = self.dispatch def goto_error(self, offset: int, storage: Storage) -> str: # To do: Add jump targets for popping values. @@ -136,39 +134,6 @@ def oparg( self.out.emit_at(uop.name[-1], tkn) return True - def jumpby( - self, - tkn: Token, - tkn_iter: TokenIterator, - uop: CodeSection, - storage: Storage, - inst: Instruction | None, - ) -> bool: - if storage.spilled: - raise analysis_error("stack_pointer needs reloading before dispatch", tkn) - storage.stack.flush(self.out) - self.emit("TIER2_STORE_IP") - emit_to(self.out, tkn_iter, "SEMI") - self.emit(";\n") - return True - - def dispatch( - self, - tkn: Token, - tkn_iter: TokenIterator, - uop: CodeSection, - storage: Storage, - inst: Instruction | None, - ) -> bool: - if storage.spilled: - raise analysis_error("stack_pointer needs reloading before dispatch", tkn) - storage.stack.flush(self.out) - self.emit("break;\n") - next(tkn_iter) - next(tkn_iter) - next(tkn_iter) - return False - def write_uop(uop: Uop, emitter: Emitter, stack: Stack) -> Stack: locals: dict[str, Local] = {} From af9ea5750f859442ef4d39e14017f39c31d98c5f Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 30 Oct 2025 18:26:37 +0000 Subject: [PATCH 137/190] Remove dynamic exit for _FOR_ITER_TIER_TWO --- Include/internal/pycore_uop_metadata.h | 2 +- Python/bytecodes.c | 3 +-- Python/executor_cases.c.h | 2 -- Python/optimizer.c | 5 +---- 4 files changed, 3 insertions(+), 9 deletions(-) diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index c7fbfb0fafa2f6..148b3d9c1fc94c 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -213,7 +213,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_MATCH_KEYS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GET_ITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GET_YIELD_FROM_ITER] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_FOR_ITER_TIER_TWO] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_FOR_ITER_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_ITER_CHECK_LIST] = HAS_EXIT_FLAG, [_GUARD_NOT_EXHAUSTED_LIST] = HAS_EXIT_FLAG, [_ITER_NEXT_LIST_TIER_TWO] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index ba3b1df3d1ca33..d62a7515963ff0 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3251,8 +3251,7 @@ dummy_func( ERROR_NO_POP(); } /* iterator ended normally */ - /* This just sets the IP to what it expects (see normal _FOR_ITER) */ - frame->instr_ptr += (oparg + 2 + INLINE_CACHE_ENTRIES_FOR_ITER); + /* The translator sets the deopt target just past the matching END_FOR */ EXIT_IF(true); } next = item; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 1534dd6995f6cb..88ef5a8656e009 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4385,7 +4385,6 @@ _PyStackRef null_or_index; _PyStackRef iter; _PyStackRef next; - oparg = CURRENT_OPARG(); null_or_index = stack_pointer[-1]; iter = stack_pointer[-2]; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -4395,7 +4394,6 @@ if (PyStackRef_IsError(item)) { JUMP_TO_ERROR(); } - frame->instr_ptr += (oparg + 2 + INLINE_CACHE_ENTRIES_FOR_ITER); if (true) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); diff --git a/Python/optimizer.c b/Python/optimizer.c index d11108dcddfb93..97b891ac97f381 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1075,10 +1075,7 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) } int32_t jump_target = target; bool unique_target = false; - if (opcode == _FOR_ITER_TIER_TWO) { - exit_op = _DYNAMIC_EXIT; - } - else if ( + if ( opcode == _GUARD_IP__PUSH_FRAME || opcode == _GUARD_IP_RETURN_VALUE || opcode == _GUARD_IP_YIELD_VALUE || From 3ed1402da63e7e95bc6c4e4acdf69a5f0ed15971 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 1 Nov 2025 15:59:26 +0000 Subject: [PATCH 138/190] Remove bytecode object for ceval.c --- Include/internal/pycore_ceval.h | 3 +- Python/bytecodes.c | 16 +++++++---- Python/ceval.c | 49 ++++++++------------------------- Python/executor_cases.c.h | 10 +++++-- Python/generated_cases.c.h | 5 ++-- Python/optimizer.c | 10 ++++--- Python/optimizer_bytecodes.c | 1 + Python/optimizer_cases.c.h | 1 + 8 files changed, 41 insertions(+), 54 deletions(-) diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 61917b01b7cb13..3039a141a0dc13 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -391,8 +391,7 @@ _PyForIter_VirtualIteratorNext(PyThreadState* tstate, struct _PyInterpreterFrame #define SPECIAL___AEXIT__ 3 #define SPECIAL_MAX 3 -struct _PyCode12 _PyCode_DEF(12); -PyAPI_DATA(const struct _PyCode12) _PyEntryFrameCode; +PyAPI_DATA(const _Py_CODEUNIT *) _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR; #ifdef __cplusplus } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index d62a7515963ff0..b5d4468fe94fca 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -44,6 +44,7 @@ #define USE_COMPUTED_GOTOS 0 #include "ceval_macros.h" +#include "../Include/internal/pycore_stackref.h" /* Flow control macros */ @@ -170,7 +171,6 @@ dummy_func( op(_QUICKEN_RESUME, (--)) { #if ENABLE_SPECIALIZATION_FT - PyCodeObject *code = _PyFrame_GetCode(frame); if (tstate->tracing == 0 && this_instr->op.code == RESUME) { FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); } @@ -5266,7 +5266,9 @@ dummy_func( tier2 op(_EXIT_TRACE, (exit_p/4 --)) { _PyExitData *exit = (_PyExitData *)exit_p; #if defined(Py_DEBUG) && !defined(_Py_JIT) - _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; + const _Py_CODEUNIT *target = ((frame->owner >= FRAME_OWNED_BY_INTERPRETER) + ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame)) + + exit->target; OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); if (frame->lltrace >= 2) { printf("SIDE EXIT: [UOp "); @@ -5420,7 +5422,8 @@ dummy_func( tier2 op(_COLD_EXIT, ( -- )) { _PyExitData *exit = tstate->jit_exit; assert(exit != NULL); - _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; + _Py_CODEUNIT *target = ((frame->owner >= FRAME_OWNED_BY_INTERPRETER) + ? (_Py_CODEUNIT *)_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame)) + exit->target; _Py_BackoffCounter temperature = exit->temperature; _PyExecutorObject *executor; if (target->op.code == ENTER_EXECUTOR) { @@ -5429,6 +5432,9 @@ dummy_func( Py_INCREF(executor); } else { + if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) { + GOTO_TIER_ONE(target); + } if (!backoff_counter_triggers(temperature)) { exit->temperature = advance_backoff_counter(temperature); GOTO_TIER_ONE(target); @@ -5664,14 +5670,14 @@ dummy_func( tstate->interp->jit_state.prev_instr = next_instr; } tstate->interp->jit_state.specialize_counter = 0; - PyCodeObject *prev_code = (PyCodeObject *)Py_NewRef(_PyFrame_GetCode(frame)); + PyCodeObject *prev_code = (PyCodeObject *)Py_NewRef(PyStackRef_AsPyObjectBorrow(frame->f_executable)); if (tstate->interp->jit_state.prev_instr_code != prev_code) { Py_SETREF(tstate->interp->jit_state.prev_instr_code, prev_code); } tstate->interp->jit_state.prev_instr_frame = frame; tstate->interp->jit_state.prev_instr_oparg = oparg; - tstate->interp->jit_state.prev_instr_stacklevel = STACK_LEVEL(); + tstate->interp->jit_state.prev_instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); DISPATCH_GOTO_NON_TRACING(); #else Py_FatalError("JIT label executed in non-jit build."); diff --git a/Python/ceval.c b/Python/ceval.c index 09fd30b0c8a030..6abb9872037097 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -931,44 +931,17 @@ int _Py_CheckRecursiveCallPy( return 0; } -static const PyBytesObject no_location = { - PyVarObject_HEAD_INIT(&PyBytes_Type, 1) - .ob_sval = { NO_LOC_4 } +static const _Py_CODEUNIT _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS[] = { + /* Put a NOP at the start, so that the IP points into + * the code, rather than before it */ + { .op.code = NOP, .op.arg = 0 }, + { .op.code = INTERPRETER_EXIT, .op.arg = 0 }, /* reached on return */ + { .op.code = NOP, .op.arg = 0 }, + { .op.code = INTERPRETER_EXIT, .op.arg = 0 }, /* reached on yield */ + { .op.code = RESUME, .op.arg = RESUME_OPARG_DEPTH1_MASK | RESUME_AT_FUNC_START } }; -#ifdef Py_GIL_DISABLED -static _PyCodeArray emtry_cleanup_tlbc = { - .size = 1, - .entries = {(char*) &_PyEntryFrameCode.co_code_adaptive}, -}; -#endif - -const struct _PyCode12 _PyEntryFrameCode = { - _PyVarObject_HEAD_INIT(&PyCode_Type, 5), - .co_consts = (PyObject *)&_Py_SINGLETON(tuple_empty), - .co_names = (PyObject *)&_Py_SINGLETON(tuple_empty), - .co_exceptiontable = (PyObject *)&_Py_SINGLETON(bytes_empty), - .co_flags = CO_OPTIMIZED | CO_NO_MONITORING_EVENTS, - .co_localsplusnames = (PyObject *)&_Py_SINGLETON(tuple_empty), - .co_localspluskinds = (PyObject *)&_Py_SINGLETON(bytes_empty), - .co_filename = &_Py_ID(_PyEval_EvalFrameDefault), - .co_name = &_Py_ID(_PyEval_EvalFrameDefault), - .co_qualname = &_Py_ID(_PyEval_EvalFrameDefault), - .co_linetable = (PyObject *)&no_location, - ._co_firsttraceable = 4, - .co_stacksize = 2, - .co_framesize = 2 + FRAME_SPECIALS_SIZE, -#ifdef Py_GIL_DISABLED - .co_tlbc = &emtry_cleanup_tlbc, -#endif - .co_code_adaptive = { - NOP, 0, - INTERPRETER_EXIT, 0, /* reached on return */ - NOP, 0, - INTERPRETER_EXIT, 0, /* reached on yield */ - RESUME, RESUME_OPARG_DEPTH1_MASK | RESUME_AT_FUNC_START - } -}; +const _Py_CODEUNIT *_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR = (_Py_CODEUNIT*)&_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS; #ifdef Py_DEBUG extern void _PyUOpPrint(const _PyUOpInstruction *uop); @@ -1140,8 +1113,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int entry.frame.f_globals = (PyObject*)0xaaa3; entry.frame.f_builtins = (PyObject*)0xaaa4; #endif - entry.frame.f_executable = PyStackRef_FromPyObjectBorrow((PyObject *)&_PyEntryFrameCode); - entry.frame.instr_ptr = ((_Py_CODEUNIT *)_PyEntryFrameCode.co_code_adaptive) + 1; + entry.frame.f_executable = PyStackRef_None; + entry.frame.instr_ptr = (_Py_CODEUNIT *)_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS + 1; entry.frame.stackpointer = entry.stack; entry.frame.owner = FRAME_OWNED_BY_INTERPRETER; entry.frame.visited = 0; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 88ef5a8656e009..1b52a27621af5e 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7110,7 +7110,9 @@ PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); _PyExitData *exit = (_PyExitData *)exit_p; #if defined(Py_DEBUG) && !defined(_Py_JIT) - _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; + const _Py_CODEUNIT *target = ((frame->owner >= FRAME_OWNED_BY_INTERPRETER) + ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame)) + + exit->target; OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); if (frame->lltrace >= 2) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -7462,7 +7464,8 @@ case _COLD_EXIT: { _PyExitData *exit = tstate->jit_exit; assert(exit != NULL); - _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; + _Py_CODEUNIT *target = ((frame->owner >= FRAME_OWNED_BY_INTERPRETER) + ? (_Py_CODEUNIT *)_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame)) + exit->target; _Py_BackoffCounter temperature = exit->temperature; _PyExecutorObject *executor; if (target->op.code == ENTER_EXECUTOR) { @@ -7471,6 +7474,9 @@ Py_INCREF(executor); } else { + if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) { + GOTO_TIER_ONE(target); + } if (!backoff_counter_triggers(temperature)) { exit->temperature = advance_backoff_counter(temperature); GOTO_TIER_ONE(target); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 1c7b1035a1a8ce..ff5b47496e1124 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -10343,7 +10343,6 @@ // _QUICKEN_RESUME { #if ENABLE_SPECIALIZATION_FT - PyCodeObject *code = _PyFrame_GetCode(frame); if (tstate->tracing == 0 && this_instr->op.code == RESUME) { FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); } @@ -12368,7 +12367,7 @@ JUMP_TO_LABEL(error); tstate->interp->jit_state.prev_instr = next_instr; } tstate->interp->jit_state.specialize_counter = 0; - PyCodeObject *prev_code = (PyCodeObject *)Py_NewRef(_PyFrame_GetCode(frame)); + PyCodeObject *prev_code = (PyCodeObject *)Py_NewRef(PyStackRef_AsPyObjectBorrow(frame->f_executable)); if (tstate->interp->jit_state.prev_instr_code != prev_code) { _PyFrame_SetStackPointer(frame, stack_pointer); Py_SETREF(tstate->interp->jit_state.prev_instr_code, prev_code); @@ -12376,7 +12375,7 @@ JUMP_TO_LABEL(error); } tstate->interp->jit_state.prev_instr_frame = frame; tstate->interp->jit_state.prev_instr_oparg = oparg; - tstate->interp->jit_state.prev_instr_stacklevel = STACK_LEVEL(); + tstate->interp->jit_state.prev_instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); DISPATCH_GOTO_NON_TRACING(); #else Py_FatalError("JIT label executed in non-jit build."); diff --git a/Python/optimizer.c b/Python/optimizer.c index 97b891ac97f381..f0ea40b44ef1f5 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -34,7 +34,7 @@ static bool has_space_for_executor(PyCodeObject *code, _Py_CODEUNIT *instr) { - if (code == (PyCodeObject *)&_Py_InitCleanup || code == (PyCodeObject *)&_PyEntryFrameCode) { + if (code == (PyCodeObject *)&_Py_InitCleanup) { return false; } if (instr->op.code == ENTER_EXECUTOR) { @@ -580,7 +580,9 @@ _PyJit_translate_single_bytecode_to_trace( _Py_CODEUNIT *target_instr = this_instr; uint32_t target = 0; - target = INSTR_IP(target_instr, old_code); + target = Py_IsNone((PyObject *)old_code) + ? (int)(target_instr - _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR) + : INSTR_IP(target_instr, old_code); // Rewind EXTENDED_ARG so that we see the whole thing. // We must point to the first EXTENDED_ARG when deopting. @@ -858,7 +860,7 @@ _PyJit_translate_single_bytecode_to_trace( DPRINTF(2, "Adding %p func to op\n", (void *)operand); _Py_BloomFilter_Add(dependencies, new_func); } - else if (new_code != NULL) { + else if (new_code != NULL && !Py_IsNone((PyObject*)new_code)) { operand = (uintptr_t)new_code | 1; DPRINTF(2, "Adding %p code to op\n", (void *)operand); _Py_BloomFilter_Add(dependencies, new_code); @@ -867,7 +869,7 @@ _PyJit_translate_single_bytecode_to_trace( operand = 0; } ADD_TO_TRACE(uop, oparg, operand, target); - trace[trace_length - 1].operand1 = ((int)(frame->stackpointer - _PyFrame_Stackbase(frame))); + trace[trace_length - 1].operand1 = PyStackRef_IsNone(frame->f_executable) ? 2 : ((int)(frame->stackpointer - _PyFrame_Stackbase(frame))); break; } if (uop == _BINARY_OP_INPLACE_ADD_UNICODE) { diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index d57bea14c5ad76..98dabe989aed14 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -812,6 +812,7 @@ dummy_func(void) { JitOptRef temp = PyJitRef_StripReferenceInfo(retval); DEAD(retval); SAVE_STACK(); + ctx->frame->stack_pointer = stack_pointer; PyCodeObject *returning_code = get_code_with_logging(this_instr); if (returning_code == NULL) { ctx->done = true; diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index d26803e97d3257..40bf95d3081bac 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1175,6 +1175,7 @@ JitOptRef temp = PyJitRef_StripReferenceInfo(retval); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); + ctx->frame->stack_pointer = stack_pointer; PyCodeObject *returning_code = get_code_with_logging(this_instr); if (returning_code == NULL) { ctx->done = true; From 0a1bad24d0de7275015755db197acd97f6c982ef Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 1 Nov 2025 16:02:46 +0000 Subject: [PATCH 139/190] fix mypy --- Tools/cases_generator/analyzer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 3bd92cfa55a450..b96e472d53782f 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -982,7 +982,7 @@ def compute_properties(op: parser.CodeDef) -> Properties: no_save_ip=no_save_ip, tier=tier_variable(op), needs_prev=variable_used(op, "prev_instr"), - needs_guard_ip=(unpredictable_jump and "replaced" not in op.annotations) or variable_used(op, "LLTRACE_RESUME_FRAME") or variable_used(op, "DISPATCH_INLINED"), + needs_guard_ip=(isinstance(op, parser.InstDef) and (unpredictable_jump and "replaced" not in op.annotations)) or variable_used(op, "LLTRACE_RESUME_FRAME") or variable_used(op, "DISPATCH_INLINED"), unpredictable_jump=unpredictable_jump, ) From 04985689069f0e6d90c29800d0a487fdeb3a68d5 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 1 Nov 2025 16:27:25 +0000 Subject: [PATCH 140/190] regen global objects --- Include/internal/pycore_global_objects_fini_generated.h | 1 - Include/internal/pycore_global_strings.h | 1 - Include/internal/pycore_runtime_init_generated.h | 1 - Include/internal/pycore_unicodeobject_generated.h | 4 ---- 4 files changed, 7 deletions(-) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 4e0a2a452728da..92ded14891a101 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -1344,7 +1344,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(TextIOWrapper)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(True)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(WarningMessage)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_PyEval_EvalFrameDefault)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_WindowsConsoleIO)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__IOBase_closed)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__abc_tpflags__)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index c150948e432ca5..cd21b0847b7cdd 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -67,7 +67,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(TextIOWrapper) STRUCT_FOR_ID(True) STRUCT_FOR_ID(WarningMessage) - STRUCT_FOR_ID(_PyEval_EvalFrameDefault) STRUCT_FOR_ID(_WindowsConsoleIO) STRUCT_FOR_ID(__IOBase_closed) STRUCT_FOR_ID(__abc_tpflags__) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 0212a141fe2e36..50d82d0a365037 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1342,7 +1342,6 @@ extern "C" { INIT_ID(TextIOWrapper), \ INIT_ID(True), \ INIT_ID(WarningMessage), \ - INIT_ID(_PyEval_EvalFrameDefault), \ INIT_ID(_WindowsConsoleIO), \ INIT_ID(__IOBase_closed), \ INIT_ID(__abc_tpflags__), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index e1a5838dd07444..b4d920154b6e83 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -56,10 +56,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(_PyEval_EvalFrameDefault); - _PyUnicode_InternStatic(interp, &string); - assert(_PyUnicode_CheckConsistency(string, 1)); - assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_WindowsConsoleIO); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); From 2dbc2915e8305ebad7e9135e5f1e2a130fdcbcc9 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 1 Nov 2025 17:56:36 +0000 Subject: [PATCH 141/190] fix C analyzer --- Tools/c-analyzer/cpython/ignored.tsv | 1 + 1 file changed, 1 insertion(+) diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 8b73189fb07dc5..c3c711c3a62114 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -358,6 +358,7 @@ Parser/parser.c - soft_keywords - Parser/lexer/lexer.c - type_comment_prefix - Python/ceval.c - _PyEval_BinaryOps - Python/ceval.c - _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS - +Python/ceval.c - _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR - Python/codecs.c - Py_hexdigits - Python/codecs.c - codecs_builtin_error_handlers - Python/codecs.c - ucnhash_capi - From 82ec8f1c7cd1068215f3e6d2a0965ac090616264 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 1 Nov 2025 18:36:39 +0000 Subject: [PATCH 142/190] convert spaces to tabs --- Tools/c-analyzer/cpython/ignored.tsv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index c3c711c3a62114..316d11a1ddc382 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -358,7 +358,7 @@ Parser/parser.c - soft_keywords - Parser/lexer/lexer.c - type_comment_prefix - Python/ceval.c - _PyEval_BinaryOps - Python/ceval.c - _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS - -Python/ceval.c - _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR - +Python/ceval.c - _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR - Python/codecs.c - Py_hexdigits - Python/codecs.c - codecs_builtin_error_handlers - Python/codecs.c - ucnhash_capi - From f52668891a1eea1e564bb1968613ee0742ca73eb Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 5 Nov 2025 10:58:04 +0000 Subject: [PATCH 143/190] Fix a bug in not setting executors --- Python/jit.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Python/jit.c b/Python/jit.c index c3f3d686013fe4..4c412958cb764d 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -645,6 +645,7 @@ _Py_LazyJitTrampoline( } _Py_jit_entry = trampoline; } + tstate->current_executor = (PyObject *)executor; PyMutex_Unlock(&lazy_jit_mutex); return _Py_jit_entry(executor, frame, stack_pointer, tstate); } From c78a4e649fe494f8f32ba6af1daeea8ab3f273ee Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 5 Nov 2025 11:01:53 +0000 Subject: [PATCH 144/190] Revert "Fix a bug in not setting executors" This reverts commit f52668891a1eea1e564bb1968613ee0742ca73eb. --- Python/jit.c | 1 - 1 file changed, 1 deletion(-) diff --git a/Python/jit.c b/Python/jit.c index 4c412958cb764d..c3f3d686013fe4 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -645,7 +645,6 @@ _Py_LazyJitTrampoline( } _Py_jit_entry = trampoline; } - tstate->current_executor = (PyObject *)executor; PyMutex_Unlock(&lazy_jit_mutex); return _Py_jit_entry(executor, frame, stack_pointer, tstate); } From aa92d84c39aa21bd4ef9d22c5411c4334f0b096c Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 1 Nov 2025 15:36:10 +0000 Subject: [PATCH 145/190] cold dynamic executors --- Include/internal/pycore_interp_structs.h | 1 + Include/internal/pycore_optimizer.h | 4 +- Include/internal/pycore_uop_ids.h | 396 ++++++++++++----------- Include/internal/pycore_uop_metadata.h | 22 +- Python/bytecodes.c | 108 +++++-- Python/ceval.c | 12 +- Python/ceval_macros.h | 12 +- Python/executor_cases.c.h | 105 ++++-- Python/generated_cases.c.h | 2 +- Python/optimizer.c | 90 +++++- Python/optimizer_analysis.c | 7 +- Python/optimizer_cases.c.h | 24 +- Python/pystate.c | 8 + 13 files changed, 516 insertions(+), 275 deletions(-) diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 546cc98d166c61..e37cecdd1d54ff 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -962,6 +962,7 @@ struct _is { struct _PyExecutorObject *executor_list_head; struct _PyExecutorObject *executor_deletion_list_head; struct _PyExecutorObject *cold_executor; + struct _PyExecutorObject *cold_dynamic_executor; int executor_deletion_list_remaining_capacity; size_t executor_creation_counter; _rare_events rare_events; diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 2f51485e93fcb5..3ea62eab1f4693 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -37,6 +37,7 @@ typedef struct { typedef struct _PyExitData { uint32_t target; uint16_t index; + char is_dynamic; _Py_BackoffCounter temperature; struct _PyExecutorObject *executor; } _PyExitData; @@ -340,6 +341,7 @@ static inline _PyExecutorObject *_PyExecutor_FromExit(_PyExitData *exit) } extern _PyExecutorObject *_PyExecutor_GetColdExecutor(void); +extern _PyExecutorObject *_PyExecutor_GetColdDynamicExecutor(void); PyAPI_FUNC(void) _PyExecutor_ClearExit(_PyExitData *exit); @@ -366,7 +368,7 @@ int _PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *curr_instr, _Py_CODEUNIT *insert_exec_instr, _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit, - _PyExecutorObject *prev_exec, int oparg); + _PyExecutorObject *prev_exec, int oparg, bool is_dynamic_target); void _PyJit_FinalizeTracing(PyThreadState *tstate); diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 52aae3b3af64bf..a390be53e43379 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -81,106 +81,109 @@ extern "C" { #define _CHECK_STACK_SPACE 357 #define _CHECK_STACK_SPACE_OPERAND 358 #define _CHECK_VALIDITY 359 -#define _COLD_EXIT 360 -#define _COMPARE_OP 361 -#define _COMPARE_OP_FLOAT 362 -#define _COMPARE_OP_INT 363 -#define _COMPARE_OP_STR 364 -#define _CONTAINS_OP 365 -#define _CONTAINS_OP_DICT 366 -#define _CONTAINS_OP_SET 367 +#define _COLD_DYNAMIC_EXIT 360 +#define _COLD_EXIT 361 +#define _COMPARE_OP 362 +#define _COMPARE_OP_FLOAT 363 +#define _COMPARE_OP_INT 364 +#define _COMPARE_OP_STR 365 +#define _CONTAINS_OP 366 +#define _CONTAINS_OP_DICT 367 +#define _CONTAINS_OP_SET 368 #define _CONVERT_VALUE CONVERT_VALUE -#define _COPY 368 -#define _COPY_1 369 -#define _COPY_2 370 -#define _COPY_3 371 +#define _COPY 369 +#define _COPY_1 370 +#define _COPY_2 371 +#define _COPY_3 372 #define _COPY_FREE_VARS COPY_FREE_VARS -#define _CREATE_INIT_FRAME 372 +#define _CREATE_INIT_FRAME 373 #define _DELETE_ATTR DELETE_ATTR #define _DELETE_DEREF DELETE_DEREF #define _DELETE_FAST DELETE_FAST #define _DELETE_GLOBAL DELETE_GLOBAL #define _DELETE_NAME DELETE_NAME #define _DELETE_SUBSCR DELETE_SUBSCR -#define _DEOPT 373 +#define _DEOPT 374 #define _DICT_MERGE DICT_MERGE #define _DICT_UPDATE DICT_UPDATE -#define _DO_CALL 374 -#define _DO_CALL_FUNCTION_EX 375 -#define _DO_CALL_KW 376 -#define _DYNAMIC_EXIT 377 +#define _DO_CALL 375 +#define _DO_CALL_FUNCTION_EX 376 +#define _DO_CALL_KW 377 +#define _DYNAMIC_DEOPT 378 +#define _DYNAMIC_EXIT 379 #define _END_FOR END_FOR #define _END_SEND END_SEND -#define _ERROR_POP_N 378 +#define _ERROR_POP_N 380 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _EXPAND_METHOD 379 -#define _EXPAND_METHOD_KW 380 -#define _FATAL_ERROR 381 +#define _EXPAND_METHOD 381 +#define _EXPAND_METHOD_KW 382 +#define _FATAL_ERROR 383 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 382 -#define _FOR_ITER_GEN_FRAME 383 -#define _FOR_ITER_TIER_TWO 384 +#define _FOR_ITER 384 +#define _FOR_ITER_GEN_FRAME 385 +#define _FOR_ITER_TIER_TWO 386 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BINARY_OP_EXTEND 385 -#define _GUARD_CALLABLE_ISINSTANCE 386 -#define _GUARD_CALLABLE_LEN 387 -#define _GUARD_CALLABLE_LIST_APPEND 388 -#define _GUARD_CALLABLE_STR_1 389 -#define _GUARD_CALLABLE_TUPLE_1 390 -#define _GUARD_CALLABLE_TYPE_1 391 -#define _GUARD_DORV_NO_DICT 392 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 393 -#define _GUARD_GLOBALS_VERSION 394 -#define _GUARD_IP_RETURN_GENERATOR 395 -#define _GUARD_IP_RETURN_VALUE 396 -#define _GUARD_IP_YIELD_VALUE 397 -#define _GUARD_IP__PUSH_FRAME 398 -#define _GUARD_IS_FALSE_POP 399 -#define _GUARD_IS_NONE_POP 400 -#define _GUARD_IS_NOT_NONE_POP 401 -#define _GUARD_IS_TRUE_POP 402 -#define _GUARD_KEYS_VERSION 403 -#define _GUARD_NOS_DICT 404 -#define _GUARD_NOS_FLOAT 405 -#define _GUARD_NOS_INT 406 -#define _GUARD_NOS_LIST 407 -#define _GUARD_NOS_NOT_NULL 408 -#define _GUARD_NOS_NULL 409 -#define _GUARD_NOS_OVERFLOWED 410 -#define _GUARD_NOS_TUPLE 411 -#define _GUARD_NOS_UNICODE 412 -#define _GUARD_NOT_EXHAUSTED_LIST 413 -#define _GUARD_NOT_EXHAUSTED_RANGE 414 -#define _GUARD_NOT_EXHAUSTED_TUPLE 415 -#define _GUARD_THIRD_NULL 416 -#define _GUARD_TOS_ANY_SET 417 -#define _GUARD_TOS_DICT 418 -#define _GUARD_TOS_FLOAT 419 -#define _GUARD_TOS_INT 420 -#define _GUARD_TOS_LIST 421 -#define _GUARD_TOS_OVERFLOWED 422 -#define _GUARD_TOS_SLICE 423 -#define _GUARD_TOS_TUPLE 424 -#define _GUARD_TOS_UNICODE 425 -#define _GUARD_TYPE_VERSION 426 -#define _GUARD_TYPE_VERSION_AND_LOCK 427 -#define _HANDLE_PENDING_AND_DEOPT 428 +#define _GUARD_BINARY_OP_EXTEND 387 +#define _GUARD_CALLABLE_ISINSTANCE 388 +#define _GUARD_CALLABLE_LEN 389 +#define _GUARD_CALLABLE_LIST_APPEND 390 +#define _GUARD_CALLABLE_STR_1 391 +#define _GUARD_CALLABLE_TUPLE_1 392 +#define _GUARD_CALLABLE_TYPE_1 393 +#define _GUARD_DORV_NO_DICT 394 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 395 +#define _GUARD_EXECUTOR_IP 396 +#define _GUARD_GLOBALS_VERSION 397 +#define _GUARD_IP_RETURN_GENERATOR 398 +#define _GUARD_IP_RETURN_VALUE 399 +#define _GUARD_IP_YIELD_VALUE 400 +#define _GUARD_IP__PUSH_FRAME 401 +#define _GUARD_IS_FALSE_POP 402 +#define _GUARD_IS_NONE_POP 403 +#define _GUARD_IS_NOT_NONE_POP 404 +#define _GUARD_IS_TRUE_POP 405 +#define _GUARD_KEYS_VERSION 406 +#define _GUARD_NOS_DICT 407 +#define _GUARD_NOS_FLOAT 408 +#define _GUARD_NOS_INT 409 +#define _GUARD_NOS_LIST 410 +#define _GUARD_NOS_NOT_NULL 411 +#define _GUARD_NOS_NULL 412 +#define _GUARD_NOS_OVERFLOWED 413 +#define _GUARD_NOS_TUPLE 414 +#define _GUARD_NOS_UNICODE 415 +#define _GUARD_NOT_EXHAUSTED_LIST 416 +#define _GUARD_NOT_EXHAUSTED_RANGE 417 +#define _GUARD_NOT_EXHAUSTED_TUPLE 418 +#define _GUARD_THIRD_NULL 419 +#define _GUARD_TOS_ANY_SET 420 +#define _GUARD_TOS_DICT 421 +#define _GUARD_TOS_FLOAT 422 +#define _GUARD_TOS_INT 423 +#define _GUARD_TOS_LIST 424 +#define _GUARD_TOS_OVERFLOWED 425 +#define _GUARD_TOS_SLICE 426 +#define _GUARD_TOS_TUPLE 427 +#define _GUARD_TOS_UNICODE 428 +#define _GUARD_TYPE_VERSION 429 +#define _GUARD_TYPE_VERSION_AND_LOCK 430 +#define _HANDLE_PENDING_AND_DEOPT 431 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 429 -#define _INIT_CALL_PY_EXACT_ARGS 430 -#define _INIT_CALL_PY_EXACT_ARGS_0 431 -#define _INIT_CALL_PY_EXACT_ARGS_1 432 -#define _INIT_CALL_PY_EXACT_ARGS_2 433 -#define _INIT_CALL_PY_EXACT_ARGS_3 434 -#define _INIT_CALL_PY_EXACT_ARGS_4 435 -#define _INSERT_NULL 436 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 432 +#define _INIT_CALL_PY_EXACT_ARGS 433 +#define _INIT_CALL_PY_EXACT_ARGS_0 434 +#define _INIT_CALL_PY_EXACT_ARGS_1 435 +#define _INIT_CALL_PY_EXACT_ARGS_2 436 +#define _INIT_CALL_PY_EXACT_ARGS_3 437 +#define _INIT_CALL_PY_EXACT_ARGS_4 438 +#define _INSERT_NULL 439 #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER #define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION #define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD @@ -190,178 +193,179 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _IS_NONE 437 +#define _IS_NONE 440 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 438 -#define _ITER_CHECK_RANGE 439 -#define _ITER_CHECK_TUPLE 440 -#define _ITER_JUMP_LIST 441 -#define _ITER_JUMP_RANGE 442 -#define _ITER_JUMP_TUPLE 443 -#define _ITER_NEXT_LIST 444 -#define _ITER_NEXT_LIST_TIER_TWO 445 -#define _ITER_NEXT_RANGE 446 -#define _ITER_NEXT_TUPLE 447 +#define _ITER_CHECK_LIST 441 +#define _ITER_CHECK_RANGE 442 +#define _ITER_CHECK_TUPLE 443 +#define _ITER_JUMP_LIST 444 +#define _ITER_JUMP_RANGE 445 +#define _ITER_JUMP_TUPLE 446 +#define _ITER_NEXT_LIST 447 +#define _ITER_NEXT_LIST_TIER_TWO 448 +#define _ITER_NEXT_RANGE 449 +#define _ITER_NEXT_TUPLE 450 #define _JUMP_BACKWARD_NO_INTERRUPT JUMP_BACKWARD_NO_INTERRUPT -#define _JUMP_TO_TOP 448 +#define _JUMP_TO_TOP 451 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 449 -#define _LOAD_ATTR_CLASS 450 +#define _LOAD_ATTR 452 +#define _LOAD_ATTR_CLASS 453 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 451 -#define _LOAD_ATTR_METHOD_LAZY_DICT 452 -#define _LOAD_ATTR_METHOD_NO_DICT 453 -#define _LOAD_ATTR_METHOD_WITH_VALUES 454 -#define _LOAD_ATTR_MODULE 455 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 456 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 457 -#define _LOAD_ATTR_PROPERTY_FRAME 458 -#define _LOAD_ATTR_SLOT 459 -#define _LOAD_ATTR_WITH_HINT 460 +#define _LOAD_ATTR_INSTANCE_VALUE 454 +#define _LOAD_ATTR_METHOD_LAZY_DICT 455 +#define _LOAD_ATTR_METHOD_NO_DICT 456 +#define _LOAD_ATTR_METHOD_WITH_VALUES 457 +#define _LOAD_ATTR_MODULE 458 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 459 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 460 +#define _LOAD_ATTR_PROPERTY_FRAME 461 +#define _LOAD_ATTR_SLOT 462 +#define _LOAD_ATTR_WITH_HINT 463 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _LOAD_BYTECODE 461 +#define _LOAD_BYTECODE 464 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 462 -#define _LOAD_CONST_INLINE_BORROW 463 -#define _LOAD_CONST_UNDER_INLINE 464 -#define _LOAD_CONST_UNDER_INLINE_BORROW 465 +#define _LOAD_CONST_INLINE 465 +#define _LOAD_CONST_INLINE_BORROW 466 +#define _LOAD_CONST_UNDER_INLINE 467 +#define _LOAD_CONST_UNDER_INLINE_BORROW 468 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 466 -#define _LOAD_FAST_0 467 -#define _LOAD_FAST_1 468 -#define _LOAD_FAST_2 469 -#define _LOAD_FAST_3 470 -#define _LOAD_FAST_4 471 -#define _LOAD_FAST_5 472 -#define _LOAD_FAST_6 473 -#define _LOAD_FAST_7 474 +#define _LOAD_FAST 469 +#define _LOAD_FAST_0 470 +#define _LOAD_FAST_1 471 +#define _LOAD_FAST_2 472 +#define _LOAD_FAST_3 473 +#define _LOAD_FAST_4 474 +#define _LOAD_FAST_5 475 +#define _LOAD_FAST_6 476 +#define _LOAD_FAST_7 477 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR -#define _LOAD_FAST_BORROW 475 -#define _LOAD_FAST_BORROW_0 476 -#define _LOAD_FAST_BORROW_1 477 -#define _LOAD_FAST_BORROW_2 478 -#define _LOAD_FAST_BORROW_3 479 -#define _LOAD_FAST_BORROW_4 480 -#define _LOAD_FAST_BORROW_5 481 -#define _LOAD_FAST_BORROW_6 482 -#define _LOAD_FAST_BORROW_7 483 +#define _LOAD_FAST_BORROW 478 +#define _LOAD_FAST_BORROW_0 479 +#define _LOAD_FAST_BORROW_1 480 +#define _LOAD_FAST_BORROW_2 481 +#define _LOAD_FAST_BORROW_3 482 +#define _LOAD_FAST_BORROW_4 483 +#define _LOAD_FAST_BORROW_5 484 +#define _LOAD_FAST_BORROW_6 485 +#define _LOAD_FAST_BORROW_7 486 #define _LOAD_FAST_BORROW_LOAD_FAST_BORROW LOAD_FAST_BORROW_LOAD_FAST_BORROW #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 484 -#define _LOAD_GLOBAL_BUILTINS 485 -#define _LOAD_GLOBAL_MODULE 486 +#define _LOAD_GLOBAL 487 +#define _LOAD_GLOBAL_BUILTINS 488 +#define _LOAD_GLOBAL_MODULE 489 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 487 -#define _LOAD_SMALL_INT_0 488 -#define _LOAD_SMALL_INT_1 489 -#define _LOAD_SMALL_INT_2 490 -#define _LOAD_SMALL_INT_3 491 -#define _LOAD_SPECIAL 492 +#define _LOAD_SMALL_INT 490 +#define _LOAD_SMALL_INT_0 491 +#define _LOAD_SMALL_INT_1 492 +#define _LOAD_SMALL_INT_2 493 +#define _LOAD_SMALL_INT_3 494 +#define _LOAD_SPECIAL 495 #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _MAKE_CALLARGS_A_TUPLE 493 +#define _MAKE_CALLARGS_A_TUPLE 496 #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_WARM 494 +#define _MAKE_WARM 497 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 495 -#define _MAYBE_EXPAND_METHOD_KW 496 -#define _MONITOR_CALL 497 -#define _MONITOR_CALL_KW 498 -#define _MONITOR_JUMP_BACKWARD 499 -#define _MONITOR_RESUME 500 +#define _MAYBE_EXPAND_METHOD 498 +#define _MAYBE_EXPAND_METHOD_KW 499 +#define _MONITOR_CALL 500 +#define _MONITOR_CALL_KW 501 +#define _MONITOR_JUMP_BACKWARD 502 +#define _MONITOR_RESUME 503 #define _NOP NOP -#define _POP_CALL 501 -#define _POP_CALL_LOAD_CONST_INLINE_BORROW 502 -#define _POP_CALL_ONE 503 -#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 504 -#define _POP_CALL_TWO 505 -#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 506 +#define _POP_CALL 504 +#define _POP_CALL_LOAD_CONST_INLINE_BORROW 505 +#define _POP_CALL_ONE 506 +#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 507 +#define _POP_CALL_TWO 508 +#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 509 #define _POP_EXCEPT POP_EXCEPT #define _POP_ITER POP_ITER -#define _POP_JUMP_IF_FALSE 507 -#define _POP_JUMP_IF_TRUE 508 +#define _POP_JUMP_IF_FALSE 510 +#define _POP_JUMP_IF_TRUE 511 #define _POP_TOP POP_TOP -#define _POP_TOP_FLOAT 509 -#define _POP_TOP_INT 510 -#define _POP_TOP_LOAD_CONST_INLINE 511 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 512 -#define _POP_TOP_NOP 513 -#define _POP_TOP_UNICODE 514 -#define _POP_TWO 515 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW 516 +#define _POP_TOP_FLOAT 512 +#define _POP_TOP_INT 513 +#define _POP_TOP_LOAD_CONST_INLINE 514 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 515 +#define _POP_TOP_NOP 516 +#define _POP_TOP_UNICODE 517 +#define _POP_TWO 518 +#define _POP_TWO_LOAD_CONST_INLINE_BORROW 519 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 517 +#define _PUSH_FRAME 520 #define _PUSH_NULL PUSH_NULL -#define _PUSH_NULL_CONDITIONAL 518 -#define _PY_FRAME_GENERAL 519 -#define _PY_FRAME_KW 520 -#define _QUICKEN_RESUME 521 -#define _REPLACE_WITH_TRUE 522 +#define _PUSH_NULL_CONDITIONAL 521 +#define _PY_FRAME_GENERAL 522 +#define _PY_FRAME_KW 523 +#define _QUICKEN_RESUME 524 +#define _REPLACE_WITH_TRUE 525 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 523 -#define _SEND 524 -#define _SEND_GEN_FRAME 525 +#define _SAVE_RETURN_OFFSET 526 +#define _SEND 527 +#define _SEND_GEN_FRAME 528 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 526 -#define _STORE_ATTR 527 -#define _STORE_ATTR_INSTANCE_VALUE 528 -#define _STORE_ATTR_SLOT 529 -#define _STORE_ATTR_WITH_HINT 530 +#define _START_DYNAMIC_EXECUTOR 529 +#define _START_EXECUTOR 530 +#define _STORE_ATTR 531 +#define _STORE_ATTR_INSTANCE_VALUE 532 +#define _STORE_ATTR_SLOT 533 +#define _STORE_ATTR_WITH_HINT 534 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 531 -#define _STORE_FAST_0 532 -#define _STORE_FAST_1 533 -#define _STORE_FAST_2 534 -#define _STORE_FAST_3 535 -#define _STORE_FAST_4 536 -#define _STORE_FAST_5 537 -#define _STORE_FAST_6 538 -#define _STORE_FAST_7 539 +#define _STORE_FAST 535 +#define _STORE_FAST_0 536 +#define _STORE_FAST_1 537 +#define _STORE_FAST_2 538 +#define _STORE_FAST_3 539 +#define _STORE_FAST_4 540 +#define _STORE_FAST_5 541 +#define _STORE_FAST_6 542 +#define _STORE_FAST_7 543 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 540 -#define _STORE_SUBSCR 541 -#define _STORE_SUBSCR_DICT 542 -#define _STORE_SUBSCR_LIST_INT 543 -#define _SWAP 544 -#define _SWAP_2 545 -#define _SWAP_3 546 -#define _TIER2_RESUME_CHECK 547 -#define _TO_BOOL 548 +#define _STORE_SLICE 544 +#define _STORE_SUBSCR 545 +#define _STORE_SUBSCR_DICT 546 +#define _STORE_SUBSCR_LIST_INT 547 +#define _SWAP 548 +#define _SWAP_2 549 +#define _SWAP_3 550 +#define _TIER2_RESUME_CHECK 551 +#define _TO_BOOL 552 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT -#define _TO_BOOL_LIST 549 +#define _TO_BOOL_LIST 553 #define _TO_BOOL_NONE TO_BOOL_NONE -#define _TO_BOOL_STR 550 +#define _TO_BOOL_STR 554 #define _UNARY_INVERT UNARY_INVERT #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 551 -#define _UNPACK_SEQUENCE_LIST 552 -#define _UNPACK_SEQUENCE_TUPLE 553 -#define _UNPACK_SEQUENCE_TWO_TUPLE 554 +#define _UNPACK_SEQUENCE 555 +#define _UNPACK_SEQUENCE_LIST 556 +#define _UNPACK_SEQUENCE_TUPLE 557 +#define _UNPACK_SEQUENCE_TWO_TUPLE 558 #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 554 +#define MAX_UOP_ID 558 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 148b3d9c1fc94c..348ea644eee207 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -315,6 +315,8 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CHECK_STACK_SPACE_OPERAND] = HAS_DEOPT_FLAG, [_SAVE_RETURN_OFFSET] = HAS_ARG_FLAG, [_EXIT_TRACE] = HAS_ESCAPES_FLAG, + [_DYNAMIC_EXIT] = HAS_ESCAPES_FLAG, + [_DYNAMIC_DEOPT] = 0, [_CHECK_VALIDITY] = HAS_DEOPT_FLAG, [_LOAD_CONST_INLINE] = HAS_PURE_FLAG, [_POP_TOP_LOAD_CONST_INLINE] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, @@ -330,6 +332,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_CONST_UNDER_INLINE] = 0, [_LOAD_CONST_UNDER_INLINE_BORROW] = 0, [_START_EXECUTOR] = HAS_DEOPT_FLAG, + [_START_DYNAMIC_EXECUTOR] = HAS_DEOPT_FLAG, [_MAKE_WARM] = 0, [_FATAL_ERROR] = 0, [_DEOPT] = 0, @@ -337,11 +340,12 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_ERROR_POP_N] = HAS_ARG_FLAG, [_TIER2_RESUME_CHECK] = HAS_PERIODIC_FLAG, [_COLD_EXIT] = 0, + [_COLD_DYNAMIC_EXIT] = 0, + [_GUARD_EXECUTOR_IP] = HAS_EXIT_FLAG, [_GUARD_IP__PUSH_FRAME] = HAS_EXIT_FLAG, [_GUARD_IP_YIELD_VALUE] = HAS_EXIT_FLAG, [_GUARD_IP_RETURN_VALUE] = HAS_EXIT_FLAG, [_GUARD_IP_RETURN_GENERATOR] = HAS_EXIT_FLAG, - [_DYNAMIC_EXIT] = HAS_ESCAPES_FLAG, }; const ReplicationRange _PyUop_Replication[MAX_UOP_ID+1] = { @@ -424,6 +428,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_CHECK_STACK_SPACE] = "_CHECK_STACK_SPACE", [_CHECK_STACK_SPACE_OPERAND] = "_CHECK_STACK_SPACE_OPERAND", [_CHECK_VALIDITY] = "_CHECK_VALIDITY", + [_COLD_DYNAMIC_EXIT] = "_COLD_DYNAMIC_EXIT", [_COLD_EXIT] = "_COLD_EXIT", [_COMPARE_OP] = "_COMPARE_OP", [_COMPARE_OP_FLOAT] = "_COMPARE_OP_FLOAT", @@ -448,6 +453,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_DEOPT] = "_DEOPT", [_DICT_MERGE] = "_DICT_MERGE", [_DICT_UPDATE] = "_DICT_UPDATE", + [_DYNAMIC_DEOPT] = "_DYNAMIC_DEOPT", [_DYNAMIC_EXIT] = "_DYNAMIC_EXIT", [_END_FOR] = "_END_FOR", [_END_SEND] = "_END_SEND", @@ -476,6 +482,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_GUARD_CALLABLE_TYPE_1] = "_GUARD_CALLABLE_TYPE_1", [_GUARD_DORV_NO_DICT] = "_GUARD_DORV_NO_DICT", [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT", + [_GUARD_EXECUTOR_IP] = "_GUARD_EXECUTOR_IP", [_GUARD_GLOBALS_VERSION] = "_GUARD_GLOBALS_VERSION", [_GUARD_IP_RETURN_GENERATOR] = "_GUARD_IP_RETURN_GENERATOR", [_GUARD_IP_RETURN_VALUE] = "_GUARD_IP_RETURN_VALUE", @@ -634,6 +641,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_SET_FUNCTION_ATTRIBUTE] = "_SET_FUNCTION_ATTRIBUTE", [_SET_IP] = "_SET_IP", [_SET_UPDATE] = "_SET_UPDATE", + [_START_DYNAMIC_EXECUTOR] = "_START_DYNAMIC_EXECUTOR", [_START_EXECUTOR] = "_START_EXECUTOR", [_STORE_ATTR] = "_STORE_ATTR", [_STORE_ATTR_INSTANCE_VALUE] = "_STORE_ATTR_INSTANCE_VALUE", @@ -1271,6 +1279,10 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _EXIT_TRACE: return 0; + case _DYNAMIC_EXIT: + return 0; + case _DYNAMIC_DEOPT: + return 0; case _CHECK_VALIDITY: return 0; case _LOAD_CONST_INLINE: @@ -1301,6 +1313,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _START_EXECUTOR: return 0; + case _START_DYNAMIC_EXECUTOR: + return 0; case _MAKE_WARM: return 0; case _FATAL_ERROR: @@ -1315,6 +1329,10 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _COLD_EXIT: return 0; + case _COLD_DYNAMIC_EXIT: + return 0; + case _GUARD_EXECUTOR_IP: + return 0; case _GUARD_IP__PUSH_FRAME: return 0; case _GUARD_IP_YIELD_VALUE: @@ -1323,8 +1341,6 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _GUARD_IP_RETURN_GENERATOR: return 0; - case _DYNAMIC_EXIT: - return 0; default: return -1; } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index b5d4468fe94fca..8c73f35acfe4ee 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2984,7 +2984,7 @@ dummy_func( oparg >>= 8; insert_exec_at--; } - int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, NULL, oparg); + int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, NULL, oparg, false); if (succ) { ENTER_TRACING(); } @@ -5283,6 +5283,32 @@ dummy_func( TIER2_TO_TIER2(exit->executor); } + // Note: this is different than _COLD_EXIT/_EXIT_TRACE, as it may lead to multiple executors + // from a single exit! + tier2 op(_DYNAMIC_EXIT, (exit_p/4 --)) { + _PyExitData *exit = (_PyExitData *)exit_p; + #if defined(Py_DEBUG) && !defined(_Py_JIT) + _Py_CODEUNIT *target = frame->instr_ptr; + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + if (frame->lltrace >= 2) { + printf("DYNAMIC EXIT: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(", exit %tu, temp %d, target %d -> %s]\n", + exit - current_executor->exits, exit->temperature.value_and_backoff, + (int)(target - _PyFrame_GetBytecode(frame)), + _PyOpcode_OpName[target->op.code]); + } + #endif + tstate->jit_exit = exit; + assert(exit->is_dynamic); + _PyExecutorObject *exec = exit->executor; + TIER2_TO_TIER2(exec); + } + + tier2 op(_DYNAMIC_DEOPT, (--)) { + GOTO_TIER_ONE(frame->instr_ptr); + } + tier2 op(_CHECK_VALIDITY, (--)) { DEOPT_IF(!current_executor->vm_data.valid); } @@ -5382,6 +5408,21 @@ dummy_func( } } + tier2 op(_START_DYNAMIC_EXECUTOR, (executor/4 --)) { +#ifndef _Py_JIT + assert(current_executor == (_PyExecutorObject*)executor); +#endif + assert(tstate->jit_exit != NULL || tstate->jit_exit->executor == current_executor); + tstate->current_executor = (PyObject *)executor; + if (!current_executor->vm_data.valid) { + assert(tstate->jit_exit->executor == current_executor); + assert(tstate->current_executor == executor); + _PyExecutor_ClearExit(tstate->jit_exit); + // Note: this points to _DYNAMIC_DEOPT!!! + DEOPT_IF(true); + } + } + tier2 op(_MAKE_WARM, (--)) { current_executor->vm_data.warm = true; } @@ -5445,10 +5486,11 @@ dummy_func( // Note: it's safe to use target->op.arg here instead of the oparg given by EXTENDED_ARG. // The invariant in the optimizer is the deopt target always points back to the first EXTENDED_ARG. // So setting it to anything else is wrong. - int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, previous_executor, target->op.arg); + int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, previous_executor, target->op.arg, false); if (succ) { GOTO_TIER_ONE_CONTINUE_TRACING(target); } + exit->temperature = restart_backoff_counter(exit->temperature); GOTO_TIER_ONE(target); } assert(tstate->jit_exit == exit); @@ -5456,6 +5498,41 @@ dummy_func( TIER2_TO_TIER2(exit->executor); } + tier2 op(_COLD_DYNAMIC_EXIT, ( -- )) { + _PyExitData *exit = tstate->jit_exit; + assert(exit != NULL); + _Py_CODEUNIT *target = frame->instr_ptr; + _Py_BackoffCounter temperature = exit->temperature; + _PyExecutorObject *executor; + if (target->op.code == ENTER_EXECUTOR) { + PyCodeObject *code = _PyFrame_GetCode(frame); + executor = code->co_executors->executors[target->op.arg]; + if (executor->trace[2].opcode == _GUARD_EXECUTOR_IP && executor->vm_data.valid) { + Py_INCREF(executor); + assert(tstate->jit_exit == exit); + exit->executor = executor; + TIER2_TO_TIER2(executor); + } + } + if (!backoff_counter_triggers(temperature)) { + exit->temperature = advance_backoff_counter(temperature); + GOTO_TIER_ONE(target); + } + _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); + assert(tstate->current_executor == (PyObject *)previous_executor); + int chain_depth = previous_executor->vm_data.chain_depth + 1; + int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, previous_executor, target->op.arg, true); + if (succ) { + GOTO_TIER_ONE_CONTINUE_TRACING(target); + } + exit->temperature = restart_backoff_counter(exit->temperature); + GOTO_TIER_ONE(target); + } + + tier2 op(_GUARD_EXECUTOR_IP, (ip/4 --)) { + EXIT_IF(frame->instr_ptr != (_Py_CODEUNIT*)ip); + } + tier2 op(_GUARD_IP__PUSH_FRAME, (ip/4 --)) { // Implementation automatically inserted by Tools/cases/tier2_generator.py EXIT_IF(true); @@ -5476,33 +5553,6 @@ dummy_func( EXIT_IF(true); } - // Note: this is different than _COLD_EXIT/_EXIT_TRACE, as it may lead to multiple executors - // from a single exit! - tier2 op(_DYNAMIC_EXIT, (exit_p/4 --)) { - _Py_CODEUNIT *target = frame->instr_ptr; -#if defined(Py_DEBUG) && !defined(_Py_JIT) - _PyExitData *exit = (_PyExitData *)exit_p; - OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - if (frame->lltrace >= 2) { - printf("DYNAMIC EXIT: [UOp "); - _PyUOpPrint(&next_uop[-1]); - printf(", exit %tu, temp %d, target %d -> %s]\n", - exit - current_executor->exits, exit->temperature.value_and_backoff, - (int)(target - _PyFrame_GetBytecode(frame)), - _PyOpcode_OpName[target->op.code]); - } -#endif - if (target->op.code == ENTER_EXECUTOR) { - PyCodeObject *code = _PyFrame_GetCode(frame); - _PyExecutorObject *executor = code->co_executors->executors[target->op.arg]; - tstate->jit_exit = NULL; - TIER2_TO_TIER2(executor); - } - else { - GOTO_TIER_ONE(target); - } - } - label(pop_2_error) { stack_pointer -= 2; assert(WITHIN_STACK_BOUNDS()); diff --git a/Python/ceval.c b/Python/ceval.c index 3197eb61b0958f..0b7781ccfdbde1 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1214,7 +1214,10 @@ _PyTier2Interpreter( tier2_start: next_uop = current_executor->trace; - assert(next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT); + assert(next_uop->opcode == _START_EXECUTOR || + next_uop->opcode == _COLD_EXIT || + next_uop->opcode == _COLD_DYNAMIC_EXIT || + next_uop->opcode == _START_DYNAMIC_EXECUTOR); #undef LOAD_IP #define LOAD_IP(UNUSED) (void)0 @@ -1238,13 +1241,16 @@ _PyTier2Interpreter( uint64_t trace_uop_execution_counter = 0; #endif - assert(next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT); + assert(next_uop->opcode == _START_EXECUTOR || + next_uop->opcode == _COLD_EXIT || + next_uop->opcode == _COLD_DYNAMIC_EXIT || + next_uop->opcode == _START_DYNAMIC_EXECUTOR); tier2_dispatch: for (;;) { uopcode = next_uop->opcode; #ifdef Py_DEBUG if (frame->lltrace >= 3) { - dump_stack(frame, stack_pointer); + // dump_stack(frame, stack_pointer); if (next_uop->opcode == _START_EXECUTOR) { printf("%4d uop: ", 0); } diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 0f7e2175480bd8..b32f11652bc48f 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -401,9 +401,17 @@ do { \ next_instr = frame->instr_ptr + 1; \ JUMP_TO_LABEL(error); \ } \ + /* No progress made */ \ + if (next_instr == this_instr) { \ + opcode = executor->vm_data.opcode; \ + oparg = (oparg & ~255) | executor->vm_data.oparg; \ + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { \ + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); \ + } \ + DISPATCH_GOTO(); \ + } \ if (keep_tracing_bit) { \ - assert(next_instr->op.code != ENTER_EXECUTOR); \ - assert(tstate->interp->jit_state.code_curr_size == 2); \ + assert(tstate->interp->jit_state.code_curr_size == 2 || tstate->interp->jit_state.code_curr_size == 3); \ ENTER_TRACING(); \ DISPATCH_NON_TRACING(); \ } \ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 1b52a27621af5e..173380aff00e75 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7130,6 +7130,35 @@ break; } + case _DYNAMIC_EXIT: { + PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); + _PyExitData *exit = (_PyExitData *)exit_p; + #if defined(Py_DEBUG) && !defined(_Py_JIT) + _Py_CODEUNIT *target = frame->instr_ptr; + OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); + if (frame->lltrace >= 2) { + _PyFrame_SetStackPointer(frame, stack_pointer); + printf("DYNAMIC EXIT: [UOp "); + _PyUOpPrint(&next_uop[-1]); + printf(", exit %tu, temp %d, target %d -> %s]\n", + exit - current_executor->exits, exit->temperature.value_and_backoff, + (int)(target - _PyFrame_GetBytecode(frame)), + _PyOpcode_OpName[target->op.code]); + stack_pointer = _PyFrame_GetStackPointer(frame); + } + #endif + tstate->jit_exit = exit; + assert(exit->is_dynamic); + _PyExecutorObject *exec = exit->executor; + TIER2_TO_TIER2(exec); + break; + } + + case _DYNAMIC_DEOPT: { + GOTO_TIER_ONE(frame->instr_ptr); + break; + } + case _CHECK_VALIDITY: { if (!current_executor->vm_data.valid) { UOP_STAT_INC(uopcode, miss); @@ -7411,6 +7440,27 @@ break; } + case _START_DYNAMIC_EXECUTOR: { + PyObject *executor = (PyObject *)CURRENT_OPERAND0(); + #ifndef _Py_JIT + assert(current_executor == (_PyExecutorObject*)executor); + #endif + assert(tstate->jit_exit != NULL || tstate->jit_exit->executor == current_executor); + tstate->current_executor = (PyObject *)executor; + if (!current_executor->vm_data.valid) { + assert(tstate->jit_exit->executor == current_executor); + assert(tstate->current_executor == executor); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyExecutor_ClearExit(tstate->jit_exit); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + } + break; + } + case _MAKE_WARM: { current_executor->vm_data.warm = true; break; @@ -7484,10 +7534,11 @@ _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); assert(tstate->current_executor == (PyObject *)previous_executor); int chain_depth = previous_executor->vm_data.chain_depth + 1; - int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, previous_executor, target->op.arg); + int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, previous_executor, target->op.arg, false); if (succ) { GOTO_TIER_ONE_CONTINUE_TRACING(target); } + exit->temperature = restart_backoff_counter(exit->temperature); GOTO_TIER_ONE(target); } assert(tstate->jit_exit == exit); @@ -7496,32 +7547,44 @@ break; } - case _DYNAMIC_EXIT: { - PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); + case _COLD_DYNAMIC_EXIT: { + _PyExitData *exit = tstate->jit_exit; + assert(exit != NULL); _Py_CODEUNIT *target = frame->instr_ptr; - #if defined(Py_DEBUG) && !defined(_Py_JIT) - _PyExitData *exit = (_PyExitData *)exit_p; - OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - if (frame->lltrace >= 2) { - _PyFrame_SetStackPointer(frame, stack_pointer); - printf("DYNAMIC EXIT: [UOp "); - _PyUOpPrint(&next_uop[-1]); - printf(", exit %tu, temp %d, target %d -> %s]\n", - exit - current_executor->exits, exit->temperature.value_and_backoff, - (int)(target - _PyFrame_GetBytecode(frame)), - _PyOpcode_OpName[target->op.code]); - stack_pointer = _PyFrame_GetStackPointer(frame); - } - #endif + _Py_BackoffCounter temperature = exit->temperature; + _PyExecutorObject *executor; if (target->op.code == ENTER_EXECUTOR) { PyCodeObject *code = _PyFrame_GetCode(frame); - _PyExecutorObject *executor = code->co_executors->executors[target->op.arg]; - tstate->jit_exit = NULL; - TIER2_TO_TIER2(executor); + executor = code->co_executors->executors[target->op.arg]; + if (executor->trace[2].opcode == _GUARD_EXECUTOR_IP && executor->vm_data.valid) { + Py_INCREF(executor); + assert(tstate->jit_exit == exit); + exit->executor = executor; + TIER2_TO_TIER2(executor); + } } - else { + if (!backoff_counter_triggers(temperature)) { + exit->temperature = advance_backoff_counter(temperature); GOTO_TIER_ONE(target); } + _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); + assert(tstate->current_executor == (PyObject *)previous_executor); + int chain_depth = previous_executor->vm_data.chain_depth + 1; + int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, previous_executor, target->op.arg, true); + if (succ) { + GOTO_TIER_ONE_CONTINUE_TRACING(target); + } + exit->temperature = restart_backoff_counter(exit->temperature); + GOTO_TIER_ONE(target); + break; + } + + case _GUARD_EXECUTOR_IP: { + PyObject *ip = (PyObject *)CURRENT_OPERAND0(); + if (frame->instr_ptr != (_Py_CODEUNIT*)ip) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index ff5b47496e1124..b5d95fd19ae495 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -7716,7 +7716,7 @@ oparg >>= 8; insert_exec_at--; } - int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, NULL, oparg); + int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, NULL, oparg, false); if (succ) { ENTER_TRACING(); } diff --git a/Python/optimizer.c b/Python/optimizer.c index f0ea40b44ef1f5..1035f310c9848b 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -124,6 +124,10 @@ _PyOptimizer_Optimize( assert(!interp->compiling); assert(tstate->interp->jit_state.initial_stack_depth >= 0); #ifndef Py_GIL_DISABLED + // Trace got stomped on by another thread. + if (tstate->interp->jit_state.initial_func == NULL) { + return 0; + } interp->compiling = true; // The first executor in a chain and the MAX_CHAIN_DEPTH'th executor *must* // make progress in order to avoid infinite loops or excessively-long @@ -937,13 +941,20 @@ _PyJit_translate_single_bytecode_to_trace( // Returns 0 for do not enter tracing, 1 on enter tracing. int -_PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *curr_instr, _Py_CODEUNIT *insert_exec_instr, _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit, _PyExecutorObject *prev_exec, int oparg) +_PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *curr_instr, _Py_CODEUNIT *insert_exec_instr, _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit, _PyExecutorObject *prev_exec, int oparg, bool is_dynamic_target) { // A recursive trace. // Don't trace into the inner call because it will stomp on the previous trace, causing endless retraces. if (tstate->interp->jit_state.code_curr_size > 2) { return 0; } + if (oparg > 0xFFFF) { + return 0; + } + // Dynamic exits with progress is wonky. + if (is_dynamic_target && chain_depth >= MAX_CHAIN_DEPTH) { + return 0; + } PyCodeObject *code = _PyFrame_GetCode(frame); #ifdef Py_DEBUG char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); @@ -959,9 +970,18 @@ _PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _ 2 * INSTR_IP(close_loop_instr, code), chain_depth); #endif - add_to_trace(tstate->interp->jit_state.code_buffer, 0, _START_EXECUTOR, 0, (uintptr_t)insert_exec_instr, INSTR_IP(insert_exec_instr, code)); - add_to_trace(tstate->interp->jit_state.code_buffer, 1, _MAKE_WARM, 0, 0, 0); - tstate->interp->jit_state.code_curr_size = 2; + + if (is_dynamic_target) { + add_to_trace(tstate->interp->jit_state.code_buffer, 0, _START_DYNAMIC_EXECUTOR, 0, (uintptr_t)insert_exec_instr, INSTR_IP(insert_exec_instr, code)); + add_to_trace(tstate->interp->jit_state.code_buffer, 1, _MAKE_WARM, 0, 0, 0); + add_to_trace(tstate->interp->jit_state.code_buffer, 2, _GUARD_EXECUTOR_IP, 0, (uintptr_t)curr_instr, 0); + tstate->interp->jit_state.code_curr_size = 3; + } + else { + add_to_trace(tstate->interp->jit_state.code_buffer, 0, _START_EXECUTOR, 0, (uintptr_t)insert_exec_instr, INSTR_IP(insert_exec_instr, code)); + add_to_trace(tstate->interp->jit_state.code_buffer, 1, _MAKE_WARM, 0, 0, 0); + tstate->interp->jit_state.code_curr_size = 2; + } tstate->interp->jit_state.code_max_size = UOP_MAX_TRACE_LENGTH; tstate->interp->jit_state.insert_exec_instr = insert_exec_instr; tstate->interp->jit_state.close_loop_instr = close_loop_instr; @@ -1081,11 +1101,15 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) opcode == _GUARD_IP__PUSH_FRAME || opcode == _GUARD_IP_RETURN_VALUE || opcode == _GUARD_IP_YIELD_VALUE || - opcode == _GUARD_IP_RETURN_GENERATOR + opcode == _GUARD_IP_RETURN_GENERATOR || + opcode == _GUARD_EXECUTOR_IP ) { exit_op = _DYNAMIC_EXIT; unique_target = true; } + if (opcode == _START_DYNAMIC_EXECUTOR) { + exit_op = _DYNAMIC_DEOPT; + } if (unique_target || jump_target != current_jump_target || current_exit_op != exit_op) { make_exit(&buffer[next_spare], exit_op, jump_target); current_exit_op = exit_op; @@ -1114,7 +1138,7 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) } } if (opcode == _JUMP_TO_TOP) { - assert(buffer[0].opcode == _START_EXECUTOR); + assert(buffer[0].opcode == _START_EXECUTOR || buffer[0].opcode == _START_DYNAMIC_EXECUTOR); buffer[i].format = UOP_FORMAT_JUMP; buffer[i].jump_target = 1; } @@ -1161,7 +1185,10 @@ sanity_check(_PyExecutorObject *executor) } bool ended = false; uint32_t i = 0; - CHECK(executor->trace[0].opcode == _START_EXECUTOR || executor->trace[0].opcode == _COLD_EXIT); + CHECK(executor->trace[0].opcode == _START_EXECUTOR || + executor->trace[0].opcode == _COLD_EXIT || + executor->trace[0].opcode == _COLD_DYNAMIC_EXIT || + executor->trace[0].opcode == _START_DYNAMIC_EXECUTOR); for (; i < executor->code_size; i++) { const _PyUOpInstruction *inst = &executor->trace[i]; uint16_t opcode = inst->opcode; @@ -1194,7 +1221,8 @@ sanity_check(_PyExecutorObject *executor) opcode == _HANDLE_PENDING_AND_DEOPT || opcode == _EXIT_TRACE || opcode == _ERROR_POP_N || - opcode == _DYNAMIC_EXIT); + opcode == _DYNAMIC_EXIT || + opcode == _DYNAMIC_DEOPT); } } @@ -1217,15 +1245,15 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil /* Initialize exits */ _PyExecutorObject *cold = _PyExecutor_GetColdExecutor(); + _PyExecutorObject *cold_dynamic = _PyExecutor_GetColdDynamicExecutor(); cold->vm_data.chain_depth = chain_depth; for (int i = 0; i < exit_count; i++) { executor->exits[i].index = i; executor->exits[i].temperature = initial_temperature_backoff_counter(); - executor->exits[i].executor = cold; } int next_exit = exit_count-1; _PyUOpInstruction *dest = (_PyUOpInstruction *)&executor->trace[length]; - assert(buffer[0].opcode == _START_EXECUTOR); + assert(buffer[0].opcode == _START_EXECUTOR || buffer[0].opcode == _START_DYNAMIC_EXECUTOR); buffer[0].operand0 = (uint64_t)executor; for (int i = length-1; i >= 0; i--) { int opcode = buffer[i].opcode; @@ -1235,12 +1263,14 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil _PyExitData *exit = &executor->exits[next_exit]; exit->target = buffer[i].target; dest->operand0 = (uint64_t)exit; + exit->executor = opcode == _EXIT_TRACE ? cold : cold_dynamic; + exit->is_dynamic = (char)(opcode == _DYNAMIC_EXIT); next_exit--; } } assert(next_exit == -1); assert(dest == executor->trace); - assert(dest->opcode == _START_EXECUTOR); + assert(dest->opcode == _START_EXECUTOR || dest->opcode == _START_DYNAMIC_EXECUTOR); _Py_ExecutorInit(executor, dependencies); #ifdef Py_DEBUG char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); @@ -1547,6 +1577,34 @@ _PyExecutor_GetColdExecutor(void) return cold; } +_PyExecutorObject * +_PyExecutor_GetColdDynamicExecutor(void) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (interp->cold_dynamic_executor != NULL) { + return interp->cold_dynamic_executor; + } + _PyExecutorObject *cold = allocate_executor(0, 1); + if (cold == NULL) { + Py_FatalError("Cannot allocate core JIT code"); + } + ((_PyUOpInstruction *)cold->trace)->opcode = _COLD_DYNAMIC_EXIT; +#ifdef _Py_JIT + cold->jit_code = NULL; + cold->jit_size = 0; + // This is initialized to true so we can prevent the executor + // from being immediately detected as cold and invalidated. + cold->vm_data.warm = true; + if (_PyJIT_Compile(cold, cold->trace, 1)) { + Py_DECREF(cold); + Py_FatalError("Cannot allocate core JIT code"); + } +#endif + _Py_SetImmortal((PyObject *)cold); + interp->cold_dynamic_executor = cold; + return cold; +} + void _PyExecutor_ClearExit(_PyExitData *exit) { @@ -1554,7 +1612,12 @@ _PyExecutor_ClearExit(_PyExitData *exit) return; } _PyExecutorObject *old = exit->executor; - exit->executor = _PyExecutor_GetColdExecutor(); + if (exit->is_dynamic) { + exit->executor = _PyExecutor_GetColdDynamicExecutor(); + } + else { + exit->executor = _PyExecutor_GetColdExecutor(); + } Py_DECREF(old); } @@ -1808,6 +1871,7 @@ executor_to_gv(_PyExecutorObject *executor, FILE *out) /* Write all the outgoing edges */ _PyExecutorObject *cold = _PyExecutor_GetColdExecutor(); + _PyExecutorObject *cold_dynamic = _PyExecutor_GetColdDynamicExecutor(); for (uint32_t i = 0; i < executor->code_size; i++) { _PyUOpInstruction const *inst = &executor->trace[i]; uint16_t flags = _PyUop_Flags[inst->opcode]; @@ -1821,7 +1885,7 @@ executor_to_gv(_PyExecutorObject *executor, FILE *out) assert(exit_inst->opcode == _EXIT_TRACE || exit_inst->opcode == _DYNAMIC_EXIT); exit = (_PyExitData *)exit_inst->operand0; } - if (exit != NULL && exit->executor != cold) { + if (exit != NULL && exit->executor != cold && exit->executor != cold_dynamic) { fprintf(out, "executor_%p:i%d -> executor_%p:start\n", executor, i, exit->executor); } if (inst->opcode == _EXIT_TRACE || inst->opcode == _JUMP_TO_TOP) { diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index c576f3b716bc2a..e1a6f29f47d1dc 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -448,6 +448,7 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) buffer[pc].opcode = _NOP; } break; + case _EXIT_TRACE: default: { // Cancel out pushes and pops, repeatedly. So: @@ -481,7 +482,7 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) } /* _PUSH_FRAME doesn't escape or error, but it * does need the IP for the return address */ - bool needs_ip = (opcode == _PUSH_FRAME || opcode == _YIELD_VALUE || opcode == _DYNAMIC_EXIT); + bool needs_ip = (opcode == _PUSH_FRAME || opcode == _YIELD_VALUE || opcode == _DYNAMIC_EXIT || opcode == _EXIT_TRACE); if (_PyUop_Flags[opcode] & HAS_ESCAPES_FLAG) { needs_ip = true; may_have_escaped = true; @@ -491,10 +492,12 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) buffer[last_set_ip].opcode = _SET_IP; last_set_ip = -1; } + if (opcode == _EXIT_TRACE) { + return pc + 1; + } break; } case _JUMP_TO_TOP: - case _EXIT_TRACE: case _DYNAMIC_EXIT: return pc + 1; } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 40bf95d3081bac..50e5287c91aa91 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -3274,6 +3274,14 @@ break; } + case _DYNAMIC_EXIT: { + break; + } + + case _DYNAMIC_DEOPT: { + break; + } + case _CHECK_VALIDITY: { break; } @@ -3399,6 +3407,10 @@ break; } + case _START_DYNAMIC_EXECUTOR: { + break; + } + case _MAKE_WARM: { break; } @@ -3427,6 +3439,14 @@ break; } + case _COLD_DYNAMIC_EXIT: { + break; + } + + case _GUARD_EXECUTOR_IP: { + break; + } + case _GUARD_IP__PUSH_FRAME: { break; } @@ -3443,7 +3463,3 @@ break; } - case _DYNAMIC_EXIT: { - break; - } - diff --git a/Python/pystate.c b/Python/pystate.c index 45b7d1f3ea6ed9..0676a86965cf29 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -827,6 +827,14 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) assert(cold->vm_data.warm); _PyExecutor_Free(cold); } + + struct _PyExecutorObject *cold_dynamic = interp->cold_dynamic_executor; + if (cold_dynamic != NULL) { + interp->cold_dynamic_executor = NULL; + assert(cold_dynamic->vm_data.valid); + assert(cold_dynamic->vm_data.warm); + _PyExecutor_Free(cold_dynamic); + } /* We don't clear sysdict and builtins until the end of this function. Because clearing other attributes can execute arbitrary Python code which requires sysdict and builtins. */ From 253f230f26a3fce983baefbc77ea508b4b981a47 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 5 Nov 2025 18:27:51 +0000 Subject: [PATCH 146/190] fix frame owned by interp --- Python/bytecodes.c | 3 +++ Python/executor_cases.c.h | 3 +++ 2 files changed, 6 insertions(+) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 8c73f35acfe4ee..3b14415f540b91 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5514,6 +5514,9 @@ dummy_func( TIER2_TO_TIER2(executor); } } + if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) { + GOTO_TIER_ONE(target); + } if (!backoff_counter_triggers(temperature)) { exit->temperature = advance_backoff_counter(temperature); GOTO_TIER_ONE(target); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 173380aff00e75..28c139bf9e5760 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7563,6 +7563,9 @@ TIER2_TO_TIER2(executor); } } + if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) { + GOTO_TIER_ONE(target); + } if (!backoff_counter_triggers(temperature)) { exit->temperature = advance_backoff_counter(temperature); GOTO_TIER_ONE(target); From ffa2b72975c13fbea4a3ab068c7f6844ca3266b4 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 5 Nov 2025 18:48:37 +0000 Subject: [PATCH 147/190] Use a different chain depth for dynamic exits --- Include/internal/pycore_optimizer.h | 3 +++ Python/bytecodes.c | 3 +-- Python/executor_cases.c.h | 3 +-- Python/optimizer.c | 12 +++++++----- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 3ea62eab1f4693..daa5d36da474fb 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -132,6 +132,9 @@ static inline uint16_t uop_get_error_target(const _PyUOpInstruction *inst) // handle before rejoining the rest of the program. #define MAX_CHAIN_DEPTH 4 +// The maximum number of side exits arising from unpredictable control-flow. +#define MAX_DYNAMIC_CHAIN_DEPTH 6 + /* Symbols */ /* See explanation in optimizer_symbols.c */ diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 3b14415f540b91..0ecfd0ff993c8c 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5412,7 +5412,6 @@ dummy_func( #ifndef _Py_JIT assert(current_executor == (_PyExecutorObject*)executor); #endif - assert(tstate->jit_exit != NULL || tstate->jit_exit->executor == current_executor); tstate->current_executor = (PyObject *)executor; if (!current_executor->vm_data.valid) { assert(tstate->jit_exit->executor == current_executor); @@ -5507,7 +5506,7 @@ dummy_func( if (target->op.code == ENTER_EXECUTOR) { PyCodeObject *code = _PyFrame_GetCode(frame); executor = code->co_executors->executors[target->op.arg]; - if (executor->trace[2].opcode == _GUARD_EXECUTOR_IP && executor->vm_data.valid) { + if (executor->trace[0].opcode == _START_DYNAMIC_EXECUTOR && executor->vm_data.valid) { Py_INCREF(executor); assert(tstate->jit_exit == exit); exit->executor = executor; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 28c139bf9e5760..d613e4fa67f057 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7445,7 +7445,6 @@ #ifndef _Py_JIT assert(current_executor == (_PyExecutorObject*)executor); #endif - assert(tstate->jit_exit != NULL || tstate->jit_exit->executor == current_executor); tstate->current_executor = (PyObject *)executor; if (!current_executor->vm_data.valid) { assert(tstate->jit_exit->executor == current_executor); @@ -7556,7 +7555,7 @@ if (target->op.code == ENTER_EXECUTOR) { PyCodeObject *code = _PyFrame_GetCode(frame); executor = code->co_executors->executors[target->op.arg]; - if (executor->trace[2].opcode == _GUARD_EXECUTOR_IP && executor->vm_data.valid) { + if (executor->trace[0].opcode == _START_DYNAMIC_EXECUTOR && executor->vm_data.valid) { Py_INCREF(executor); assert(tstate->jit_exit == exit); exit->executor = executor; diff --git a/Python/optimizer.c b/Python/optimizer.c index 1035f310c9848b..24ca1c5c03940b 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -133,7 +133,12 @@ _PyOptimizer_Optimize( // make progress in order to avoid infinite loops or excessively-long // side-exit chains. We can only insert the executor into the bytecode if // this is true, since a deopt won't infinitely re-enter the executor: - chain_depth %= MAX_CHAIN_DEPTH; + if (tstate->interp->jit_state.code_buffer[0].opcode == _START_DYNAMIC_EXECUTOR) { + chain_depth %= MAX_DYNAMIC_CHAIN_DEPTH; + } + else { + chain_depth %= MAX_CHAIN_DEPTH; + } bool progress_needed = chain_depth == 0; PyCodeObject *code = (PyCodeObject *)tstate->interp->jit_state.initial_code; _Py_CODEUNIT *start = tstate->interp->jit_state.insert_exec_instr; @@ -951,10 +956,7 @@ _PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _ if (oparg > 0xFFFF) { return 0; } - // Dynamic exits with progress is wonky. - if (is_dynamic_target && chain_depth >= MAX_CHAIN_DEPTH) { - return 0; - } + PyCodeObject *code = _PyFrame_GetCode(frame); #ifdef Py_DEBUG char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); From 897edf53a384411dbf1fba4ccc20f4d11a050a80 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 5 Nov 2025 18:53:30 +0000 Subject: [PATCH 148/190] remove two chain depths --- Include/internal/pycore_optimizer.h | 3 --- Python/optimizer.c | 7 +------ 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index daa5d36da474fb..3ea62eab1f4693 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -132,9 +132,6 @@ static inline uint16_t uop_get_error_target(const _PyUOpInstruction *inst) // handle before rejoining the rest of the program. #define MAX_CHAIN_DEPTH 4 -// The maximum number of side exits arising from unpredictable control-flow. -#define MAX_DYNAMIC_CHAIN_DEPTH 6 - /* Symbols */ /* See explanation in optimizer_symbols.c */ diff --git a/Python/optimizer.c b/Python/optimizer.c index 24ca1c5c03940b..d4578b5e2bde75 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -133,12 +133,7 @@ _PyOptimizer_Optimize( // make progress in order to avoid infinite loops or excessively-long // side-exit chains. We can only insert the executor into the bytecode if // this is true, since a deopt won't infinitely re-enter the executor: - if (tstate->interp->jit_state.code_buffer[0].opcode == _START_DYNAMIC_EXECUTOR) { - chain_depth %= MAX_DYNAMIC_CHAIN_DEPTH; - } - else { - chain_depth %= MAX_CHAIN_DEPTH; - } + chain_depth %= MAX_CHAIN_DEPTH; bool progress_needed = chain_depth == 0; PyCodeObject *code = (PyCodeObject *)tstate->interp->jit_state.initial_code; _Py_CODEUNIT *start = tstate->interp->jit_state.insert_exec_instr; From 1ef4a37922bf4ef4b6e5ec2349539a626fdf7f22 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 5 Nov 2025 21:11:53 +0000 Subject: [PATCH 149/190] Make sure we don't reenter executors when guard exec ip fails --- Include/internal/pycore_ceval.h | 3 ++- Python/bytecodes.c | 31 +++++++++++++++++++++++++------ Python/ceval_macros.h | 11 +++-------- Python/executor_cases.c.h | 26 +++++++++++++++++++++----- Python/generated_cases.c.h | 6 +++++- Python/optimizer.c | 8 +++++++- Tools/cases_generator/analyzer.py | 2 ++ 7 files changed, 65 insertions(+), 22 deletions(-) diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 33b9fd053f70cb..02da6195225610 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -334,9 +334,10 @@ _PyEval_SpecialMethodCanSuggest(PyObject *self, int oparg); #define _PY_EVAL_PLEASE_STOP_BIT (1U << 5) #define _PY_EVAL_EXPLICIT_MERGE_BIT (1U << 6) #define _PY_EVAL_JIT_INVALIDATE_COLD_BIT (1U << 7) +#define _PY_EVAL_JIT_DO_NOT_REENTER (1U << 8) /* Reserve a few bits for future use */ -#define _PY_EVAL_EVENTS_BITS 8 +#define _PY_EVAL_EVENTS_BITS 9 #define _PY_EVAL_EVENTS_MASK ((1 << _PY_EVAL_EVENTS_BITS)-1) static inline void diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 0ecfd0ff993c8c..ddd55e71fc190c 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -44,7 +44,6 @@ #define USE_COMPUTED_GOTOS 0 #include "ceval_macros.h" -#include "../Include/internal/pycore_stackref.h" /* Flow control macros */ @@ -2988,6 +2987,9 @@ dummy_func( if (succ) { ENTER_TRACING(); } + else { + this_instr[1].counter = restart_backoff_counter(counter); + } } else { ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); @@ -3047,7 +3049,8 @@ dummy_func( /* If the eval breaker is set then stay in tier 1. * This avoids any potentially infinite loops * involving _RESUME_CHECK */ - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & (_PY_EVAL_EVENTS_MASK | _PY_EVAL_JIT_DO_NOT_REENTER)) { + _Py_unset_eval_breaker_bit(tstate, _PY_EVAL_JIT_DO_NOT_REENTER); opcode = executor->vm_data.opcode; oparg = (oparg & ~255) | executor->vm_data.oparg; next_instr = this_instr; @@ -5302,6 +5305,7 @@ dummy_func( tstate->jit_exit = exit; assert(exit->is_dynamic); _PyExecutorObject *exec = exit->executor; + assert(exec->trace[0].opcode == _COLD_DYNAMIC_EXIT || exec->trace[0].opcode == _START_DYNAMIC_EXECUTOR); TIER2_TO_TIER2(exec); } @@ -5412,6 +5416,7 @@ dummy_func( #ifndef _Py_JIT assert(current_executor == (_PyExecutorObject*)executor); #endif + assert(tstate->jit_exit == NULL || tstate->jit_exit->executor == current_executor); tstate->current_executor = (PyObject *)executor; if (!current_executor->vm_data.valid) { assert(tstate->jit_exit->executor == current_executor); @@ -5470,6 +5475,11 @@ dummy_func( PyCodeObject *code = _PyFrame_GetCode(frame); executor = code->co_executors->executors[target->op.arg]; Py_INCREF(executor); + #if Py_DEBUG + if (executor->trace[2].opcode == _GUARD_EXECUTOR_IP) { + assert(executor->trace[2].operand0 == (uint64_t)target); + } + #endif } else { if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) { @@ -5486,10 +5496,10 @@ dummy_func( // The invariant in the optimizer is the deopt target always points back to the first EXTENDED_ARG. // So setting it to anything else is wrong. int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, previous_executor, target->op.arg, false); + exit->temperature = restart_backoff_counter(exit->temperature); if (succ) { GOTO_TIER_ONE_CONTINUE_TRACING(target); } - exit->temperature = restart_backoff_counter(exit->temperature); GOTO_TIER_ONE(target); } assert(tstate->jit_exit == exit); @@ -5500,18 +5510,24 @@ dummy_func( tier2 op(_COLD_DYNAMIC_EXIT, ( -- )) { _PyExitData *exit = tstate->jit_exit; assert(exit != NULL); + assert(exit->is_dynamic); _Py_CODEUNIT *target = frame->instr_ptr; _Py_BackoffCounter temperature = exit->temperature; _PyExecutorObject *executor; if (target->op.code == ENTER_EXECUTOR) { PyCodeObject *code = _PyFrame_GetCode(frame); executor = code->co_executors->executors[target->op.arg]; - if (executor->trace[0].opcode == _START_DYNAMIC_EXECUTOR && executor->vm_data.valid) { + if (executor->vm_data.valid && + executor->trace[0].opcode == _START_DYNAMIC_EXECUTOR) { + assert(executor->trace[2].operand0 == (uint64_t)target); Py_INCREF(executor); assert(tstate->jit_exit == exit); exit->executor = executor; TIER2_TO_TIER2(executor); } + else { + GOTO_TIER_ONE(target); + } } if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) { GOTO_TIER_ONE(target); @@ -5524,15 +5540,18 @@ dummy_func( assert(tstate->current_executor == (PyObject *)previous_executor); int chain_depth = previous_executor->vm_data.chain_depth + 1; int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, previous_executor, target->op.arg, true); + exit->temperature = restart_backoff_counter(exit->temperature); if (succ) { GOTO_TIER_ONE_CONTINUE_TRACING(target); } - exit->temperature = restart_backoff_counter(exit->temperature); GOTO_TIER_ONE(target); } tier2 op(_GUARD_EXECUTOR_IP, (ip/4 --)) { - EXIT_IF(frame->instr_ptr != (_Py_CODEUNIT*)ip); + if (frame->instr_ptr != (_Py_CODEUNIT*)ip) { + _Py_set_eval_breaker_bit(tstate, _PY_EVAL_JIT_DO_NOT_REENTER); + EXIT_IF(true); + } } tier2 op(_GUARD_IP__PUSH_FRAME, (ip/4 --)) { diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index b32f11652bc48f..e12040bcc848ec 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -401,14 +401,9 @@ do { \ next_instr = frame->instr_ptr + 1; \ JUMP_TO_LABEL(error); \ } \ - /* No progress made */ \ - if (next_instr == this_instr) { \ - opcode = executor->vm_data.opcode; \ - oparg = (oparg & ~255) | executor->vm_data.oparg; \ - if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { \ - PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); \ - } \ - DISPATCH_GOTO(); \ + /* Progress made */ \ + if (next_instr != this_instr) { \ + _Py_unset_eval_breaker_bit(tstate, _PY_EVAL_JIT_DO_NOT_REENTER); \ } \ if (keep_tracing_bit) { \ assert(tstate->interp->jit_state.code_curr_size == 2 || tstate->interp->jit_state.code_curr_size == 3); \ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index d613e4fa67f057..c45bf72033b67c 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7150,6 +7150,7 @@ tstate->jit_exit = exit; assert(exit->is_dynamic); _PyExecutorObject *exec = exit->executor; + assert(exec->trace[0].opcode == _COLD_DYNAMIC_EXIT || exec->trace[0].opcode == _START_DYNAMIC_EXECUTOR); TIER2_TO_TIER2(exec); break; } @@ -7445,6 +7446,7 @@ #ifndef _Py_JIT assert(current_executor == (_PyExecutorObject*)executor); #endif + assert(tstate->jit_exit == NULL || tstate->jit_exit->executor == current_executor); tstate->current_executor = (PyObject *)executor; if (!current_executor->vm_data.valid) { assert(tstate->jit_exit->executor == current_executor); @@ -7521,6 +7523,11 @@ PyCodeObject *code = _PyFrame_GetCode(frame); executor = code->co_executors->executors[target->op.arg]; Py_INCREF(executor); + #if Py_DEBUG + if (executor->trace[2].opcode == _GUARD_EXECUTOR_IP) { + assert(executor->trace[2].operand0 == (uint64_t)target); + } + #endif } else { if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) { @@ -7534,10 +7541,10 @@ assert(tstate->current_executor == (PyObject *)previous_executor); int chain_depth = previous_executor->vm_data.chain_depth + 1; int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, previous_executor, target->op.arg, false); + exit->temperature = restart_backoff_counter(exit->temperature); if (succ) { GOTO_TIER_ONE_CONTINUE_TRACING(target); } - exit->temperature = restart_backoff_counter(exit->temperature); GOTO_TIER_ONE(target); } assert(tstate->jit_exit == exit); @@ -7549,18 +7556,24 @@ case _COLD_DYNAMIC_EXIT: { _PyExitData *exit = tstate->jit_exit; assert(exit != NULL); + assert(exit->is_dynamic); _Py_CODEUNIT *target = frame->instr_ptr; _Py_BackoffCounter temperature = exit->temperature; _PyExecutorObject *executor; if (target->op.code == ENTER_EXECUTOR) { PyCodeObject *code = _PyFrame_GetCode(frame); executor = code->co_executors->executors[target->op.arg]; - if (executor->trace[0].opcode == _START_DYNAMIC_EXECUTOR && executor->vm_data.valid) { + if (executor->vm_data.valid && + executor->trace[0].opcode == _START_DYNAMIC_EXECUTOR) { + assert(executor->trace[2].operand0 == (uint64_t)target); Py_INCREF(executor); assert(tstate->jit_exit == exit); exit->executor = executor; TIER2_TO_TIER2(executor); } + else { + GOTO_TIER_ONE(target); + } } if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) { GOTO_TIER_ONE(target); @@ -7573,10 +7586,10 @@ assert(tstate->current_executor == (PyObject *)previous_executor); int chain_depth = previous_executor->vm_data.chain_depth + 1; int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, previous_executor, target->op.arg, true); + exit->temperature = restart_backoff_counter(exit->temperature); if (succ) { GOTO_TIER_ONE_CONTINUE_TRACING(target); } - exit->temperature = restart_backoff_counter(exit->temperature); GOTO_TIER_ONE(target); break; } @@ -7584,8 +7597,11 @@ case _GUARD_EXECUTOR_IP: { PyObject *ip = (PyObject *)CURRENT_OPERAND0(); if (frame->instr_ptr != (_Py_CODEUNIT*)ip) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); + _Py_set_eval_breaker_bit(tstate, _PY_EVAL_JIT_DO_NOT_REENTER); + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } } break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index b5d95fd19ae495..78e8422731ea61 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5509,7 +5509,8 @@ assert(executor->vm_data.code == code); assert(executor->vm_data.valid); assert(tstate->current_executor == NULL); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & (_PY_EVAL_EVENTS_MASK | _PY_EVAL_JIT_DO_NOT_REENTER)) { + _Py_unset_eval_breaker_bit(tstate, _PY_EVAL_JIT_DO_NOT_REENTER); opcode = executor->vm_data.opcode; oparg = (oparg & ~255) | executor->vm_data.oparg; next_instr = this_instr; @@ -7720,6 +7721,9 @@ if (succ) { ENTER_TRACING(); } + else { + this_instr[1].counter = restart_backoff_counter(counter); + } } else { ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); diff --git a/Python/optimizer.c b/Python/optimizer.c index d4578b5e2bde75..e507607f6ff366 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -969,7 +969,9 @@ _PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _ #endif if (is_dynamic_target) { - add_to_trace(tstate->interp->jit_state.code_buffer, 0, _START_DYNAMIC_EXECUTOR, 0, (uintptr_t)insert_exec_instr, INSTR_IP(insert_exec_instr, code)); + assert(curr_instr == frame->instr_ptr); + assert(curr_instr == insert_exec_instr); + add_to_trace(tstate->interp->jit_state.code_buffer, 0, _START_DYNAMIC_EXECUTOR, 0, (uintptr_t)insert_exec_instr, 0); add_to_trace(tstate->interp->jit_state.code_buffer, 1, _MAKE_WARM, 0, 0, 0); add_to_trace(tstate->interp->jit_state.code_buffer, 2, _GUARD_EXECUTOR_IP, 0, (uintptr_t)curr_instr, 0); tstate->interp->jit_state.code_curr_size = 3; @@ -1186,6 +1188,9 @@ sanity_check(_PyExecutorObject *executor) executor->trace[0].opcode == _COLD_EXIT || executor->trace[0].opcode == _COLD_DYNAMIC_EXIT || executor->trace[0].opcode == _START_DYNAMIC_EXECUTOR); + if (executor->trace[0].opcode == _START_DYNAMIC_EXECUTOR) { + CHECK(executor->trace[2].opcode == _GUARD_EXECUTOR_IP); + } for (; i < executor->code_size; i++) { const _PyUOpInstruction *inst = &executor->trace[i]; uint16_t opcode = inst->opcode; @@ -1579,6 +1584,7 @@ _PyExecutor_GetColdDynamicExecutor(void) { PyInterpreterState *interp = _PyInterpreterState_GET(); if (interp->cold_dynamic_executor != NULL) { + assert(interp->cold_dynamic_executor->trace[0].opcode == _COLD_DYNAMIC_EXIT); return interp->cold_dynamic_executor; } _PyExecutorObject *cold = allocate_executor(0, 1); diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index b96e472d53782f..5bba28578bfe92 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -700,6 +700,8 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "_PyLong_CheckExactAndCompact", "_PyExecutor_FromExit", "_PyJit_TryInitializeTracing", + "_Py_unset_eval_breaker_bit", + "_Py_set_eval_breaker_bit", ) From 0e921185063ef8dbde9cb7e483e5b23ca27519d6 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 5 Nov 2025 21:54:05 +0000 Subject: [PATCH 150/190] remove dynamic tracing for now --- Include/internal/pycore_optimizer.h | 2 +- Include/internal/pycore_uop_ids.h | 327 ++++++++++++------------- Include/internal/pycore_uop_metadata.h | 4 - Python/bytecodes.c | 53 +--- Python/ceval.c | 6 +- Python/executor_cases.c.h | 56 +---- Python/generated_cases.c.h | 2 +- Python/optimizer.c | 35 +-- Python/optimizer_cases.c.h | 4 - 9 files changed, 183 insertions(+), 306 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 3ea62eab1f4693..d50e7ccfdd42c6 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -368,7 +368,7 @@ int _PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *curr_instr, _Py_CODEUNIT *insert_exec_instr, _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit, - _PyExecutorObject *prev_exec, int oparg, bool is_dynamic_target); + _PyExecutorObject *prev_exec, int oparg); void _PyJit_FinalizeTracing(PyThreadState *tstate); diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index a390be53e43379..071edfaa7a52be 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -138,52 +138,51 @@ extern "C" { #define _GUARD_CALLABLE_TYPE_1 393 #define _GUARD_DORV_NO_DICT 394 #define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 395 -#define _GUARD_EXECUTOR_IP 396 -#define _GUARD_GLOBALS_VERSION 397 -#define _GUARD_IP_RETURN_GENERATOR 398 -#define _GUARD_IP_RETURN_VALUE 399 -#define _GUARD_IP_YIELD_VALUE 400 -#define _GUARD_IP__PUSH_FRAME 401 -#define _GUARD_IS_FALSE_POP 402 -#define _GUARD_IS_NONE_POP 403 -#define _GUARD_IS_NOT_NONE_POP 404 -#define _GUARD_IS_TRUE_POP 405 -#define _GUARD_KEYS_VERSION 406 -#define _GUARD_NOS_DICT 407 -#define _GUARD_NOS_FLOAT 408 -#define _GUARD_NOS_INT 409 -#define _GUARD_NOS_LIST 410 -#define _GUARD_NOS_NOT_NULL 411 -#define _GUARD_NOS_NULL 412 -#define _GUARD_NOS_OVERFLOWED 413 -#define _GUARD_NOS_TUPLE 414 -#define _GUARD_NOS_UNICODE 415 -#define _GUARD_NOT_EXHAUSTED_LIST 416 -#define _GUARD_NOT_EXHAUSTED_RANGE 417 -#define _GUARD_NOT_EXHAUSTED_TUPLE 418 -#define _GUARD_THIRD_NULL 419 -#define _GUARD_TOS_ANY_SET 420 -#define _GUARD_TOS_DICT 421 -#define _GUARD_TOS_FLOAT 422 -#define _GUARD_TOS_INT 423 -#define _GUARD_TOS_LIST 424 -#define _GUARD_TOS_OVERFLOWED 425 -#define _GUARD_TOS_SLICE 426 -#define _GUARD_TOS_TUPLE 427 -#define _GUARD_TOS_UNICODE 428 -#define _GUARD_TYPE_VERSION 429 -#define _GUARD_TYPE_VERSION_AND_LOCK 430 -#define _HANDLE_PENDING_AND_DEOPT 431 +#define _GUARD_GLOBALS_VERSION 396 +#define _GUARD_IP_RETURN_GENERATOR 397 +#define _GUARD_IP_RETURN_VALUE 398 +#define _GUARD_IP_YIELD_VALUE 399 +#define _GUARD_IP__PUSH_FRAME 400 +#define _GUARD_IS_FALSE_POP 401 +#define _GUARD_IS_NONE_POP 402 +#define _GUARD_IS_NOT_NONE_POP 403 +#define _GUARD_IS_TRUE_POP 404 +#define _GUARD_KEYS_VERSION 405 +#define _GUARD_NOS_DICT 406 +#define _GUARD_NOS_FLOAT 407 +#define _GUARD_NOS_INT 408 +#define _GUARD_NOS_LIST 409 +#define _GUARD_NOS_NOT_NULL 410 +#define _GUARD_NOS_NULL 411 +#define _GUARD_NOS_OVERFLOWED 412 +#define _GUARD_NOS_TUPLE 413 +#define _GUARD_NOS_UNICODE 414 +#define _GUARD_NOT_EXHAUSTED_LIST 415 +#define _GUARD_NOT_EXHAUSTED_RANGE 416 +#define _GUARD_NOT_EXHAUSTED_TUPLE 417 +#define _GUARD_THIRD_NULL 418 +#define _GUARD_TOS_ANY_SET 419 +#define _GUARD_TOS_DICT 420 +#define _GUARD_TOS_FLOAT 421 +#define _GUARD_TOS_INT 422 +#define _GUARD_TOS_LIST 423 +#define _GUARD_TOS_OVERFLOWED 424 +#define _GUARD_TOS_SLICE 425 +#define _GUARD_TOS_TUPLE 426 +#define _GUARD_TOS_UNICODE 427 +#define _GUARD_TYPE_VERSION 428 +#define _GUARD_TYPE_VERSION_AND_LOCK 429 +#define _HANDLE_PENDING_AND_DEOPT 430 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 432 -#define _INIT_CALL_PY_EXACT_ARGS 433 -#define _INIT_CALL_PY_EXACT_ARGS_0 434 -#define _INIT_CALL_PY_EXACT_ARGS_1 435 -#define _INIT_CALL_PY_EXACT_ARGS_2 436 -#define _INIT_CALL_PY_EXACT_ARGS_3 437 -#define _INIT_CALL_PY_EXACT_ARGS_4 438 -#define _INSERT_NULL 439 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 431 +#define _INIT_CALL_PY_EXACT_ARGS 432 +#define _INIT_CALL_PY_EXACT_ARGS_0 433 +#define _INIT_CALL_PY_EXACT_ARGS_1 434 +#define _INIT_CALL_PY_EXACT_ARGS_2 435 +#define _INIT_CALL_PY_EXACT_ARGS_3 436 +#define _INIT_CALL_PY_EXACT_ARGS_4 437 +#define _INSERT_NULL 438 #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER #define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION #define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD @@ -193,179 +192,179 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _IS_NONE 440 +#define _IS_NONE 439 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 441 -#define _ITER_CHECK_RANGE 442 -#define _ITER_CHECK_TUPLE 443 -#define _ITER_JUMP_LIST 444 -#define _ITER_JUMP_RANGE 445 -#define _ITER_JUMP_TUPLE 446 -#define _ITER_NEXT_LIST 447 -#define _ITER_NEXT_LIST_TIER_TWO 448 -#define _ITER_NEXT_RANGE 449 -#define _ITER_NEXT_TUPLE 450 +#define _ITER_CHECK_LIST 440 +#define _ITER_CHECK_RANGE 441 +#define _ITER_CHECK_TUPLE 442 +#define _ITER_JUMP_LIST 443 +#define _ITER_JUMP_RANGE 444 +#define _ITER_JUMP_TUPLE 445 +#define _ITER_NEXT_LIST 446 +#define _ITER_NEXT_LIST_TIER_TWO 447 +#define _ITER_NEXT_RANGE 448 +#define _ITER_NEXT_TUPLE 449 #define _JUMP_BACKWARD_NO_INTERRUPT JUMP_BACKWARD_NO_INTERRUPT -#define _JUMP_TO_TOP 451 +#define _JUMP_TO_TOP 450 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 452 -#define _LOAD_ATTR_CLASS 453 +#define _LOAD_ATTR 451 +#define _LOAD_ATTR_CLASS 452 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 454 -#define _LOAD_ATTR_METHOD_LAZY_DICT 455 -#define _LOAD_ATTR_METHOD_NO_DICT 456 -#define _LOAD_ATTR_METHOD_WITH_VALUES 457 -#define _LOAD_ATTR_MODULE 458 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 459 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 460 -#define _LOAD_ATTR_PROPERTY_FRAME 461 -#define _LOAD_ATTR_SLOT 462 -#define _LOAD_ATTR_WITH_HINT 463 +#define _LOAD_ATTR_INSTANCE_VALUE 453 +#define _LOAD_ATTR_METHOD_LAZY_DICT 454 +#define _LOAD_ATTR_METHOD_NO_DICT 455 +#define _LOAD_ATTR_METHOD_WITH_VALUES 456 +#define _LOAD_ATTR_MODULE 457 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 458 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 459 +#define _LOAD_ATTR_PROPERTY_FRAME 460 +#define _LOAD_ATTR_SLOT 461 +#define _LOAD_ATTR_WITH_HINT 462 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _LOAD_BYTECODE 464 +#define _LOAD_BYTECODE 463 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 465 -#define _LOAD_CONST_INLINE_BORROW 466 -#define _LOAD_CONST_UNDER_INLINE 467 -#define _LOAD_CONST_UNDER_INLINE_BORROW 468 +#define _LOAD_CONST_INLINE 464 +#define _LOAD_CONST_INLINE_BORROW 465 +#define _LOAD_CONST_UNDER_INLINE 466 +#define _LOAD_CONST_UNDER_INLINE_BORROW 467 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 469 -#define _LOAD_FAST_0 470 -#define _LOAD_FAST_1 471 -#define _LOAD_FAST_2 472 -#define _LOAD_FAST_3 473 -#define _LOAD_FAST_4 474 -#define _LOAD_FAST_5 475 -#define _LOAD_FAST_6 476 -#define _LOAD_FAST_7 477 +#define _LOAD_FAST 468 +#define _LOAD_FAST_0 469 +#define _LOAD_FAST_1 470 +#define _LOAD_FAST_2 471 +#define _LOAD_FAST_3 472 +#define _LOAD_FAST_4 473 +#define _LOAD_FAST_5 474 +#define _LOAD_FAST_6 475 +#define _LOAD_FAST_7 476 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR -#define _LOAD_FAST_BORROW 478 -#define _LOAD_FAST_BORROW_0 479 -#define _LOAD_FAST_BORROW_1 480 -#define _LOAD_FAST_BORROW_2 481 -#define _LOAD_FAST_BORROW_3 482 -#define _LOAD_FAST_BORROW_4 483 -#define _LOAD_FAST_BORROW_5 484 -#define _LOAD_FAST_BORROW_6 485 -#define _LOAD_FAST_BORROW_7 486 +#define _LOAD_FAST_BORROW 477 +#define _LOAD_FAST_BORROW_0 478 +#define _LOAD_FAST_BORROW_1 479 +#define _LOAD_FAST_BORROW_2 480 +#define _LOAD_FAST_BORROW_3 481 +#define _LOAD_FAST_BORROW_4 482 +#define _LOAD_FAST_BORROW_5 483 +#define _LOAD_FAST_BORROW_6 484 +#define _LOAD_FAST_BORROW_7 485 #define _LOAD_FAST_BORROW_LOAD_FAST_BORROW LOAD_FAST_BORROW_LOAD_FAST_BORROW #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 487 -#define _LOAD_GLOBAL_BUILTINS 488 -#define _LOAD_GLOBAL_MODULE 489 +#define _LOAD_GLOBAL 486 +#define _LOAD_GLOBAL_BUILTINS 487 +#define _LOAD_GLOBAL_MODULE 488 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 490 -#define _LOAD_SMALL_INT_0 491 -#define _LOAD_SMALL_INT_1 492 -#define _LOAD_SMALL_INT_2 493 -#define _LOAD_SMALL_INT_3 494 -#define _LOAD_SPECIAL 495 +#define _LOAD_SMALL_INT 489 +#define _LOAD_SMALL_INT_0 490 +#define _LOAD_SMALL_INT_1 491 +#define _LOAD_SMALL_INT_2 492 +#define _LOAD_SMALL_INT_3 493 +#define _LOAD_SPECIAL 494 #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _MAKE_CALLARGS_A_TUPLE 496 +#define _MAKE_CALLARGS_A_TUPLE 495 #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_WARM 497 +#define _MAKE_WARM 496 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 498 -#define _MAYBE_EXPAND_METHOD_KW 499 -#define _MONITOR_CALL 500 -#define _MONITOR_CALL_KW 501 -#define _MONITOR_JUMP_BACKWARD 502 -#define _MONITOR_RESUME 503 +#define _MAYBE_EXPAND_METHOD 497 +#define _MAYBE_EXPAND_METHOD_KW 498 +#define _MONITOR_CALL 499 +#define _MONITOR_CALL_KW 500 +#define _MONITOR_JUMP_BACKWARD 501 +#define _MONITOR_RESUME 502 #define _NOP NOP -#define _POP_CALL 504 -#define _POP_CALL_LOAD_CONST_INLINE_BORROW 505 -#define _POP_CALL_ONE 506 -#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 507 -#define _POP_CALL_TWO 508 -#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 509 +#define _POP_CALL 503 +#define _POP_CALL_LOAD_CONST_INLINE_BORROW 504 +#define _POP_CALL_ONE 505 +#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 506 +#define _POP_CALL_TWO 507 +#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 508 #define _POP_EXCEPT POP_EXCEPT #define _POP_ITER POP_ITER -#define _POP_JUMP_IF_FALSE 510 -#define _POP_JUMP_IF_TRUE 511 +#define _POP_JUMP_IF_FALSE 509 +#define _POP_JUMP_IF_TRUE 510 #define _POP_TOP POP_TOP -#define _POP_TOP_FLOAT 512 -#define _POP_TOP_INT 513 -#define _POP_TOP_LOAD_CONST_INLINE 514 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 515 -#define _POP_TOP_NOP 516 -#define _POP_TOP_UNICODE 517 -#define _POP_TWO 518 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW 519 +#define _POP_TOP_FLOAT 511 +#define _POP_TOP_INT 512 +#define _POP_TOP_LOAD_CONST_INLINE 513 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 514 +#define _POP_TOP_NOP 515 +#define _POP_TOP_UNICODE 516 +#define _POP_TWO 517 +#define _POP_TWO_LOAD_CONST_INLINE_BORROW 518 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 520 +#define _PUSH_FRAME 519 #define _PUSH_NULL PUSH_NULL -#define _PUSH_NULL_CONDITIONAL 521 -#define _PY_FRAME_GENERAL 522 -#define _PY_FRAME_KW 523 -#define _QUICKEN_RESUME 524 -#define _REPLACE_WITH_TRUE 525 +#define _PUSH_NULL_CONDITIONAL 520 +#define _PY_FRAME_GENERAL 521 +#define _PY_FRAME_KW 522 +#define _QUICKEN_RESUME 523 +#define _REPLACE_WITH_TRUE 524 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 526 -#define _SEND 527 -#define _SEND_GEN_FRAME 528 +#define _SAVE_RETURN_OFFSET 525 +#define _SEND 526 +#define _SEND_GEN_FRAME 527 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_DYNAMIC_EXECUTOR 529 -#define _START_EXECUTOR 530 -#define _STORE_ATTR 531 -#define _STORE_ATTR_INSTANCE_VALUE 532 -#define _STORE_ATTR_SLOT 533 -#define _STORE_ATTR_WITH_HINT 534 +#define _START_DYNAMIC_EXECUTOR 528 +#define _START_EXECUTOR 529 +#define _STORE_ATTR 530 +#define _STORE_ATTR_INSTANCE_VALUE 531 +#define _STORE_ATTR_SLOT 532 +#define _STORE_ATTR_WITH_HINT 533 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 535 -#define _STORE_FAST_0 536 -#define _STORE_FAST_1 537 -#define _STORE_FAST_2 538 -#define _STORE_FAST_3 539 -#define _STORE_FAST_4 540 -#define _STORE_FAST_5 541 -#define _STORE_FAST_6 542 -#define _STORE_FAST_7 543 +#define _STORE_FAST 534 +#define _STORE_FAST_0 535 +#define _STORE_FAST_1 536 +#define _STORE_FAST_2 537 +#define _STORE_FAST_3 538 +#define _STORE_FAST_4 539 +#define _STORE_FAST_5 540 +#define _STORE_FAST_6 541 +#define _STORE_FAST_7 542 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 544 -#define _STORE_SUBSCR 545 -#define _STORE_SUBSCR_DICT 546 -#define _STORE_SUBSCR_LIST_INT 547 -#define _SWAP 548 -#define _SWAP_2 549 -#define _SWAP_3 550 -#define _TIER2_RESUME_CHECK 551 -#define _TO_BOOL 552 +#define _STORE_SLICE 543 +#define _STORE_SUBSCR 544 +#define _STORE_SUBSCR_DICT 545 +#define _STORE_SUBSCR_LIST_INT 546 +#define _SWAP 547 +#define _SWAP_2 548 +#define _SWAP_3 549 +#define _TIER2_RESUME_CHECK 550 +#define _TO_BOOL 551 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT -#define _TO_BOOL_LIST 553 +#define _TO_BOOL_LIST 552 #define _TO_BOOL_NONE TO_BOOL_NONE -#define _TO_BOOL_STR 554 +#define _TO_BOOL_STR 553 #define _UNARY_INVERT UNARY_INVERT #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 555 -#define _UNPACK_SEQUENCE_LIST 556 -#define _UNPACK_SEQUENCE_TUPLE 557 -#define _UNPACK_SEQUENCE_TWO_TUPLE 558 +#define _UNPACK_SEQUENCE 554 +#define _UNPACK_SEQUENCE_LIST 555 +#define _UNPACK_SEQUENCE_TUPLE 556 +#define _UNPACK_SEQUENCE_TWO_TUPLE 557 #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 558 +#define MAX_UOP_ID 557 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 348ea644eee207..bb519f53e34ba9 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -341,7 +341,6 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_TIER2_RESUME_CHECK] = HAS_PERIODIC_FLAG, [_COLD_EXIT] = 0, [_COLD_DYNAMIC_EXIT] = 0, - [_GUARD_EXECUTOR_IP] = HAS_EXIT_FLAG, [_GUARD_IP__PUSH_FRAME] = HAS_EXIT_FLAG, [_GUARD_IP_YIELD_VALUE] = HAS_EXIT_FLAG, [_GUARD_IP_RETURN_VALUE] = HAS_EXIT_FLAG, @@ -482,7 +481,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_GUARD_CALLABLE_TYPE_1] = "_GUARD_CALLABLE_TYPE_1", [_GUARD_DORV_NO_DICT] = "_GUARD_DORV_NO_DICT", [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT", - [_GUARD_EXECUTOR_IP] = "_GUARD_EXECUTOR_IP", [_GUARD_GLOBALS_VERSION] = "_GUARD_GLOBALS_VERSION", [_GUARD_IP_RETURN_GENERATOR] = "_GUARD_IP_RETURN_GENERATOR", [_GUARD_IP_RETURN_VALUE] = "_GUARD_IP_RETURN_VALUE", @@ -1331,8 +1329,6 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _COLD_DYNAMIC_EXIT: return 0; - case _GUARD_EXECUTOR_IP: - return 0; case _GUARD_IP__PUSH_FRAME: return 0; case _GUARD_IP_YIELD_VALUE: diff --git a/Python/bytecodes.c b/Python/bytecodes.c index ddd55e71fc190c..3fcfab5ef15944 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2983,7 +2983,7 @@ dummy_func( oparg >>= 8; insert_exec_at--; } - int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, NULL, oparg, false); + int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, NULL, oparg); if (succ) { ENTER_TRACING(); } @@ -5303,9 +5303,8 @@ dummy_func( } #endif tstate->jit_exit = exit; - assert(exit->is_dynamic); _PyExecutorObject *exec = exit->executor; - assert(exec->trace[0].opcode == _COLD_DYNAMIC_EXIT || exec->trace[0].opcode == _START_DYNAMIC_EXECUTOR); + assert(exec->trace[0].opcode == _COLD_DYNAMIC_EXIT); TIER2_TO_TIER2(exec); } @@ -5422,6 +5421,7 @@ dummy_func( assert(tstate->jit_exit->executor == current_executor); assert(tstate->current_executor == executor); _PyExecutor_ClearExit(tstate->jit_exit); + _Py_set_eval_breaker_bit(tstate, _PY_EVAL_JIT_DO_NOT_REENTER); // Note: this points to _DYNAMIC_DEOPT!!! DEOPT_IF(true); } @@ -5475,11 +5475,6 @@ dummy_func( PyCodeObject *code = _PyFrame_GetCode(frame); executor = code->co_executors->executors[target->op.arg]; Py_INCREF(executor); - #if Py_DEBUG - if (executor->trace[2].opcode == _GUARD_EXECUTOR_IP) { - assert(executor->trace[2].operand0 == (uint64_t)target); - } - #endif } else { if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) { @@ -5495,7 +5490,7 @@ dummy_func( // Note: it's safe to use target->op.arg here instead of the oparg given by EXTENDED_ARG. // The invariant in the optimizer is the deopt target always points back to the first EXTENDED_ARG. // So setting it to anything else is wrong. - int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, previous_executor, target->op.arg, false); + int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, previous_executor, target->op.arg); exit->temperature = restart_backoff_counter(exit->temperature); if (succ) { GOTO_TIER_ONE_CONTINUE_TRACING(target); @@ -5510,50 +5505,10 @@ dummy_func( tier2 op(_COLD_DYNAMIC_EXIT, ( -- )) { _PyExitData *exit = tstate->jit_exit; assert(exit != NULL); - assert(exit->is_dynamic); _Py_CODEUNIT *target = frame->instr_ptr; - _Py_BackoffCounter temperature = exit->temperature; - _PyExecutorObject *executor; - if (target->op.code == ENTER_EXECUTOR) { - PyCodeObject *code = _PyFrame_GetCode(frame); - executor = code->co_executors->executors[target->op.arg]; - if (executor->vm_data.valid && - executor->trace[0].opcode == _START_DYNAMIC_EXECUTOR) { - assert(executor->trace[2].operand0 == (uint64_t)target); - Py_INCREF(executor); - assert(tstate->jit_exit == exit); - exit->executor = executor; - TIER2_TO_TIER2(executor); - } - else { - GOTO_TIER_ONE(target); - } - } - if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) { - GOTO_TIER_ONE(target); - } - if (!backoff_counter_triggers(temperature)) { - exit->temperature = advance_backoff_counter(temperature); - GOTO_TIER_ONE(target); - } - _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); - assert(tstate->current_executor == (PyObject *)previous_executor); - int chain_depth = previous_executor->vm_data.chain_depth + 1; - int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, previous_executor, target->op.arg, true); - exit->temperature = restart_backoff_counter(exit->temperature); - if (succ) { - GOTO_TIER_ONE_CONTINUE_TRACING(target); - } GOTO_TIER_ONE(target); } - tier2 op(_GUARD_EXECUTOR_IP, (ip/4 --)) { - if (frame->instr_ptr != (_Py_CODEUNIT*)ip) { - _Py_set_eval_breaker_bit(tstate, _PY_EVAL_JIT_DO_NOT_REENTER); - EXIT_IF(true); - } - } - tier2 op(_GUARD_IP__PUSH_FRAME, (ip/4 --)) { // Implementation automatically inserted by Tools/cases/tier2_generator.py EXIT_IF(true); diff --git a/Python/ceval.c b/Python/ceval.c index 0b7781ccfdbde1..074b0f8b86e959 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1216,8 +1216,7 @@ _PyTier2Interpreter( next_uop = current_executor->trace; assert(next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT || - next_uop->opcode == _COLD_DYNAMIC_EXIT || - next_uop->opcode == _START_DYNAMIC_EXECUTOR); + next_uop->opcode == _COLD_DYNAMIC_EXIT); #undef LOAD_IP #define LOAD_IP(UNUSED) (void)0 @@ -1243,8 +1242,7 @@ _PyTier2Interpreter( assert(next_uop->opcode == _START_EXECUTOR || next_uop->opcode == _COLD_EXIT || - next_uop->opcode == _COLD_DYNAMIC_EXIT || - next_uop->opcode == _START_DYNAMIC_EXECUTOR); + next_uop->opcode == _COLD_DYNAMIC_EXIT); tier2_dispatch: for (;;) { uopcode = next_uop->opcode; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index c45bf72033b67c..826c8fc9af69ec 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7148,9 +7148,8 @@ } #endif tstate->jit_exit = exit; - assert(exit->is_dynamic); _PyExecutorObject *exec = exit->executor; - assert(exec->trace[0].opcode == _COLD_DYNAMIC_EXIT || exec->trace[0].opcode == _START_DYNAMIC_EXECUTOR); + assert(exec->trace[0].opcode == _COLD_DYNAMIC_EXIT); TIER2_TO_TIER2(exec); break; } @@ -7454,6 +7453,7 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _PyExecutor_ClearExit(tstate->jit_exit); stack_pointer = _PyFrame_GetStackPointer(frame); + _Py_set_eval_breaker_bit(tstate, _PY_EVAL_JIT_DO_NOT_REENTER); if (true) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -7523,11 +7523,6 @@ PyCodeObject *code = _PyFrame_GetCode(frame); executor = code->co_executors->executors[target->op.arg]; Py_INCREF(executor); - #if Py_DEBUG - if (executor->trace[2].opcode == _GUARD_EXECUTOR_IP) { - assert(executor->trace[2].operand0 == (uint64_t)target); - } - #endif } else { if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) { @@ -7540,7 +7535,7 @@ _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); assert(tstate->current_executor == (PyObject *)previous_executor); int chain_depth = previous_executor->vm_data.chain_depth + 1; - int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, previous_executor, target->op.arg, false); + int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, previous_executor, target->op.arg); exit->temperature = restart_backoff_counter(exit->temperature); if (succ) { GOTO_TIER_ONE_CONTINUE_TRACING(target); @@ -7556,56 +7551,11 @@ case _COLD_DYNAMIC_EXIT: { _PyExitData *exit = tstate->jit_exit; assert(exit != NULL); - assert(exit->is_dynamic); _Py_CODEUNIT *target = frame->instr_ptr; - _Py_BackoffCounter temperature = exit->temperature; - _PyExecutorObject *executor; - if (target->op.code == ENTER_EXECUTOR) { - PyCodeObject *code = _PyFrame_GetCode(frame); - executor = code->co_executors->executors[target->op.arg]; - if (executor->vm_data.valid && - executor->trace[0].opcode == _START_DYNAMIC_EXECUTOR) { - assert(executor->trace[2].operand0 == (uint64_t)target); - Py_INCREF(executor); - assert(tstate->jit_exit == exit); - exit->executor = executor; - TIER2_TO_TIER2(executor); - } - else { - GOTO_TIER_ONE(target); - } - } - if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) { - GOTO_TIER_ONE(target); - } - if (!backoff_counter_triggers(temperature)) { - exit->temperature = advance_backoff_counter(temperature); - GOTO_TIER_ONE(target); - } - _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); - assert(tstate->current_executor == (PyObject *)previous_executor); - int chain_depth = previous_executor->vm_data.chain_depth + 1; - int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, previous_executor, target->op.arg, true); - exit->temperature = restart_backoff_counter(exit->temperature); - if (succ) { - GOTO_TIER_ONE_CONTINUE_TRACING(target); - } GOTO_TIER_ONE(target); break; } - case _GUARD_EXECUTOR_IP: { - PyObject *ip = (PyObject *)CURRENT_OPERAND0(); - if (frame->instr_ptr != (_Py_CODEUNIT*)ip) { - _Py_set_eval_breaker_bit(tstate, _PY_EVAL_JIT_DO_NOT_REENTER); - if (true) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - } - break; - } - case _GUARD_IP_RETURN_VALUE: { PyObject *ip = (PyObject *)CURRENT_OPERAND0(); if (frame->instr_ptr + (frame->return_offset) != (_Py_CODEUNIT *)ip) { diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 78e8422731ea61..c97a6a9dad8bb8 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -7717,7 +7717,7 @@ oparg >>= 8; insert_exec_at--; } - int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, NULL, oparg, false); + int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, NULL, oparg); if (succ) { ENTER_TRACING(); } diff --git a/Python/optimizer.c b/Python/optimizer.c index e507607f6ff366..268a3f3c3b6428 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -941,7 +941,7 @@ _PyJit_translate_single_bytecode_to_trace( // Returns 0 for do not enter tracing, 1 on enter tracing. int -_PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *curr_instr, _Py_CODEUNIT *insert_exec_instr, _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit, _PyExecutorObject *prev_exec, int oparg, bool is_dynamic_target) +_PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *curr_instr, _Py_CODEUNIT *insert_exec_instr, _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit, _PyExecutorObject *prev_exec, int oparg) { // A recursive trace. // Don't trace into the inner call because it will stomp on the previous trace, causing endless retraces. @@ -968,19 +968,10 @@ _PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _ chain_depth); #endif - if (is_dynamic_target) { - assert(curr_instr == frame->instr_ptr); - assert(curr_instr == insert_exec_instr); - add_to_trace(tstate->interp->jit_state.code_buffer, 0, _START_DYNAMIC_EXECUTOR, 0, (uintptr_t)insert_exec_instr, 0); - add_to_trace(tstate->interp->jit_state.code_buffer, 1, _MAKE_WARM, 0, 0, 0); - add_to_trace(tstate->interp->jit_state.code_buffer, 2, _GUARD_EXECUTOR_IP, 0, (uintptr_t)curr_instr, 0); - tstate->interp->jit_state.code_curr_size = 3; - } - else { - add_to_trace(tstate->interp->jit_state.code_buffer, 0, _START_EXECUTOR, 0, (uintptr_t)insert_exec_instr, INSTR_IP(insert_exec_instr, code)); - add_to_trace(tstate->interp->jit_state.code_buffer, 1, _MAKE_WARM, 0, 0, 0); - tstate->interp->jit_state.code_curr_size = 2; - } + add_to_trace(tstate->interp->jit_state.code_buffer, 0, _START_EXECUTOR, 0, (uintptr_t)insert_exec_instr, INSTR_IP(insert_exec_instr, code)); + add_to_trace(tstate->interp->jit_state.code_buffer, 1, _MAKE_WARM, 0, 0, 0); + tstate->interp->jit_state.code_curr_size = 2; + tstate->interp->jit_state.code_max_size = UOP_MAX_TRACE_LENGTH; tstate->interp->jit_state.insert_exec_instr = insert_exec_instr; tstate->interp->jit_state.close_loop_instr = close_loop_instr; @@ -1100,15 +1091,11 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) opcode == _GUARD_IP__PUSH_FRAME || opcode == _GUARD_IP_RETURN_VALUE || opcode == _GUARD_IP_YIELD_VALUE || - opcode == _GUARD_IP_RETURN_GENERATOR || - opcode == _GUARD_EXECUTOR_IP + opcode == _GUARD_IP_RETURN_GENERATOR ) { exit_op = _DYNAMIC_EXIT; unique_target = true; } - if (opcode == _START_DYNAMIC_EXECUTOR) { - exit_op = _DYNAMIC_DEOPT; - } if (unique_target || jump_target != current_jump_target || current_exit_op != exit_op) { make_exit(&buffer[next_spare], exit_op, jump_target); current_exit_op = exit_op; @@ -1186,11 +1173,7 @@ sanity_check(_PyExecutorObject *executor) uint32_t i = 0; CHECK(executor->trace[0].opcode == _START_EXECUTOR || executor->trace[0].opcode == _COLD_EXIT || - executor->trace[0].opcode == _COLD_DYNAMIC_EXIT || - executor->trace[0].opcode == _START_DYNAMIC_EXECUTOR); - if (executor->trace[0].opcode == _START_DYNAMIC_EXECUTOR) { - CHECK(executor->trace[2].opcode == _GUARD_EXECUTOR_IP); - } + executor->trace[0].opcode == _COLD_DYNAMIC_EXIT); for (; i < executor->code_size; i++) { const _PyUOpInstruction *inst = &executor->trace[i]; uint16_t opcode = inst->opcode; @@ -1255,7 +1238,7 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil } int next_exit = exit_count-1; _PyUOpInstruction *dest = (_PyUOpInstruction *)&executor->trace[length]; - assert(buffer[0].opcode == _START_EXECUTOR || buffer[0].opcode == _START_DYNAMIC_EXECUTOR); + assert(buffer[0].opcode == _START_EXECUTOR); buffer[0].operand0 = (uint64_t)executor; for (int i = length-1; i >= 0; i--) { int opcode = buffer[i].opcode; @@ -1272,7 +1255,7 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil } assert(next_exit == -1); assert(dest == executor->trace); - assert(dest->opcode == _START_EXECUTOR || dest->opcode == _START_DYNAMIC_EXECUTOR); + assert(dest->opcode == _START_EXECUTOR); _Py_ExecutorInit(executor, dependencies); #ifdef Py_DEBUG char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 50e5287c91aa91..31affc93e7bb7d 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -3443,10 +3443,6 @@ break; } - case _GUARD_EXECUTOR_IP: { - break; - } - case _GUARD_IP__PUSH_FRAME: { break; } From c75d91c5942a0fc2c3cd84d7801091b59516f5e2 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 5 Nov 2025 22:34:24 +0000 Subject: [PATCH 151/190] Don't limit control-flow exits --- Include/internal/pycore_optimizer.h | 3 ++- Python/bytecodes.c | 8 +++++--- Python/executor_cases.c.h | 6 +++--- Python/optimizer.c | 12 +++++++++--- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index d50e7ccfdd42c6..a1d95c7c2f4aa9 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -37,7 +37,8 @@ typedef struct { typedef struct _PyExitData { uint32_t target; uint16_t index; - char is_dynamic; + char is_dynamic:4; + char is_control_flow:4; _Py_BackoffCounter temperature; struct _PyExecutorObject *executor; } _PyExitData; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 3fcfab5ef15944..47b73e30971a37 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5276,10 +5276,10 @@ dummy_func( if (frame->lltrace >= 2) { printf("SIDE EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); - printf(", exit %tu, temp %d, target %d -> %s]\n", + printf(", exit %tu, temp %d, target %d -> %s, is_control_flow %d]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyFrame_GetBytecode(frame)), - _PyOpcode_OpName[target->op.code]); + _PyOpcode_OpName[target->op.code], exit->is_control_flow); } #endif tstate->jit_exit = exit; @@ -5486,7 +5486,9 @@ dummy_func( } _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); assert(tstate->current_executor == (PyObject *)previous_executor); - int chain_depth = previous_executor->vm_data.chain_depth + 1; + // For control-flow guards, we don't want to increase the chain depth, as those don't actually + // represent deopts but rather just normal programs! + int chain_depth = previous_executor->vm_data.chain_depth + !exit->is_control_flow; // Note: it's safe to use target->op.arg here instead of the oparg given by EXTENDED_ARG. // The invariant in the optimizer is the deopt target always points back to the first EXTENDED_ARG. // So setting it to anything else is wrong. diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 826c8fc9af69ec..57dc7c6902c3cc 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7118,10 +7118,10 @@ _PyFrame_SetStackPointer(frame, stack_pointer); printf("SIDE EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); - printf(", exit %tu, temp %d, target %d -> %s]\n", + printf(", exit %tu, temp %d, target %d -> %s, is_control_flow %d]\n", exit - current_executor->exits, exit->temperature.value_and_backoff, (int)(target - _PyFrame_GetBytecode(frame)), - _PyOpcode_OpName[target->op.code]); + _PyOpcode_OpName[target->op.code], exit->is_control_flow); stack_pointer = _PyFrame_GetStackPointer(frame); } #endif @@ -7534,7 +7534,7 @@ } _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); assert(tstate->current_executor == (PyObject *)previous_executor); - int chain_depth = previous_executor->vm_data.chain_depth + 1; + int chain_depth = previous_executor->vm_data.chain_depth + !exit->is_control_flow; int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, previous_executor, target->op.arg); exit->temperature = restart_backoff_counter(exit->temperature); if (succ) { diff --git a/Python/optimizer.c b/Python/optimizer.c index 268a3f3c3b6428..6531cd64bbd675 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -470,6 +470,7 @@ is_for_iter_test[MAX_UOP_ID + 1] = { [_GUARD_NOT_EXHAUSTED_RANGE] = 1, [_GUARD_NOT_EXHAUSTED_LIST] = 1, [_GUARD_NOT_EXHAUSTED_TUPLE] = 1, + [_FOR_ITER_TIER_TWO] = 1, }; static const uint16_t @@ -757,6 +758,7 @@ _PyJit_translate_single_bytecode_to_trace( // inner loop might start and let the traces rejoin. OPT_STAT_INC(inner_loop); ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); + trace[trace_length-1].operand1 = true; // is_control_flow DPRINTF(2, "JUMP_BACKWARD not to top ends trace %p %p %p\n", next_instr, tstate->interp->jit_state.close_loop_instr, tstate->interp->jit_state.insert_exec_instr); goto done; } @@ -933,6 +935,7 @@ _PyJit_translate_single_bytecode_to_trace( // We previously reversed one. max_length += 1; ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); + trace[trace_length-1].operand1 = true; // is_control_flow } tstate->interp->jit_state.code_curr_size = trace_length; tstate->interp->jit_state.code_max_size = max_length; @@ -1033,13 +1036,14 @@ count_exits(_PyUOpInstruction *buffer, int length) return exit_count; } -static void make_exit(_PyUOpInstruction *inst, int opcode, int target) +static void make_exit(_PyUOpInstruction *inst, int opcode, int target, bool is_control_flow) { inst->opcode = opcode; inst->oparg = 0; inst->operand0 = 0; inst->format = UOP_FORMAT_TARGET; inst->target = target; + inst->operand1 = is_control_flow; #ifdef Py_STATS inst->execution_count = 0; #endif @@ -1096,8 +1100,9 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) exit_op = _DYNAMIC_EXIT; unique_target = true; } + bool is_control_flow = (opcode == _GUARD_IS_FALSE_POP || opcode == _GUARD_IS_TRUE_POP || is_for_iter_test[opcode]); if (unique_target || jump_target != current_jump_target || current_exit_op != exit_op) { - make_exit(&buffer[next_spare], exit_op, jump_target); + make_exit(&buffer[next_spare], exit_op, jump_target, is_control_flow); current_exit_op = exit_op; current_jump_target = jump_target; current_jump = next_spare; @@ -1113,7 +1118,7 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) current_popped = popped; current_error = next_spare; current_error_target = target; - make_exit(&buffer[next_spare], _ERROR_POP_N, 0); + make_exit(&buffer[next_spare], _ERROR_POP_N, 0, false); buffer[next_spare].operand0 = target; next_spare++; } @@ -1250,6 +1255,7 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil dest->operand0 = (uint64_t)exit; exit->executor = opcode == _EXIT_TRACE ? cold : cold_dynamic; exit->is_dynamic = (char)(opcode == _DYNAMIC_EXIT); + exit->is_control_flow = (char)buffer[i].operand1; next_exit--; } } From bf1ddaba4da0cf115d0d0bbb8d751ed705d4922d Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Wed, 5 Nov 2025 22:56:21 +0000 Subject: [PATCH 152/190] fix assertion --- Python/jit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/jit.c b/Python/jit.c index c3f3d686013fe4..d2d5b16b69a9b6 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -560,7 +560,7 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction trace[], siz unsigned char *code = memory; state.trampolines.mem = memory + code_size; unsigned char *data = memory + code_size + state.trampolines.size + code_padding; - assert(trace[0].opcode == _START_EXECUTOR || trace[0].opcode == _COLD_EXIT); + assert(trace[0].opcode == _START_EXECUTOR || trace[0].opcode == _COLD_EXIT || trace[0].opcode == _COLD_DYNAMIC_EXIT); for (size_t i = 0; i < length; i++) { const _PyUOpInstruction *instruction = &trace[i]; group = &stencil_groups[instruction->opcode]; From a64215db13c52438f17120018cb026de45514ba3 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 6 Nov 2025 00:14:14 +0000 Subject: [PATCH 153/190] Remove _DYNAMIC_EXIT jumping for now. --- Python/bytecodes.c | 17 +++++++---------- Python/executor_cases.c.h | 16 ++++++---------- 2 files changed, 13 insertions(+), 20 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 47b73e30971a37..0a45291d4a5d7b 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5289,8 +5289,8 @@ dummy_func( // Note: this is different than _COLD_EXIT/_EXIT_TRACE, as it may lead to multiple executors // from a single exit! tier2 op(_DYNAMIC_EXIT, (exit_p/4 --)) { - _PyExitData *exit = (_PyExitData *)exit_p; #if defined(Py_DEBUG) && !defined(_Py_JIT) + _PyExitData *exit = (_PyExitData *)exit_p; _Py_CODEUNIT *target = frame->instr_ptr; OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); if (frame->lltrace >= 2) { @@ -5302,10 +5302,9 @@ dummy_func( _PyOpcode_OpName[target->op.code]); } #endif - tstate->jit_exit = exit; - _PyExecutorObject *exec = exit->executor; - assert(exec->trace[0].opcode == _COLD_DYNAMIC_EXIT); - TIER2_TO_TIER2(exec); + // Disabled for now (gh-139109) as it slows down dynamic code tremendously. + // Compile and jump to the cold dynamic executors in the future. + GOTO_TIER_ONE(frame->instr_ptr); } tier2 op(_DYNAMIC_DEOPT, (--)) { @@ -5475,6 +5474,9 @@ dummy_func( PyCodeObject *code = _PyFrame_GetCode(frame); executor = code->co_executors->executors[target->op.arg]; Py_INCREF(executor); + assert(tstate->jit_exit == exit); + exit->executor = executor; + TIER2_TO_TIER2(exit->executor); } else { if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) { @@ -5499,14 +5501,9 @@ dummy_func( } GOTO_TIER_ONE(target); } - assert(tstate->jit_exit == exit); - exit->executor = executor; - TIER2_TO_TIER2(exit->executor); } tier2 op(_COLD_DYNAMIC_EXIT, ( -- )) { - _PyExitData *exit = tstate->jit_exit; - assert(exit != NULL); _Py_CODEUNIT *target = frame->instr_ptr; GOTO_TIER_ONE(target); } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 57dc7c6902c3cc..c6120a346a53f0 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7132,8 +7132,8 @@ case _DYNAMIC_EXIT: { PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); - _PyExitData *exit = (_PyExitData *)exit_p; #if defined(Py_DEBUG) && !defined(_Py_JIT) + _PyExitData *exit = (_PyExitData *)exit_p; _Py_CODEUNIT *target = frame->instr_ptr; OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); if (frame->lltrace >= 2) { @@ -7147,10 +7147,8 @@ stack_pointer = _PyFrame_GetStackPointer(frame); } #endif - tstate->jit_exit = exit; - _PyExecutorObject *exec = exit->executor; - assert(exec->trace[0].opcode == _COLD_DYNAMIC_EXIT); - TIER2_TO_TIER2(exec); + + GOTO_TIER_ONE(frame->instr_ptr); break; } @@ -7523,6 +7521,9 @@ PyCodeObject *code = _PyFrame_GetCode(frame); executor = code->co_executors->executors[target->op.arg]; Py_INCREF(executor); + assert(tstate->jit_exit == exit); + exit->executor = executor; + TIER2_TO_TIER2(exit->executor); } else { if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) { @@ -7542,15 +7543,10 @@ } GOTO_TIER_ONE(target); } - assert(tstate->jit_exit == exit); - exit->executor = executor; - TIER2_TO_TIER2(exit->executor); break; } case _COLD_DYNAMIC_EXIT: { - _PyExitData *exit = tstate->jit_exit; - assert(exit != NULL); _Py_CODEUNIT *target = frame->instr_ptr; GOTO_TIER_ONE(target); break; From da5e7b7df877b9cecc8b0b4d11f55fc1e19e8230 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 6 Nov 2025 07:52:05 +0000 Subject: [PATCH 154/190] reduce diff --- Include/internal/pycore_uop_ids.h | 360 ++++++++++++------------- Include/internal/pycore_uop_metadata.h | 8 - Python/bytecodes.c | 20 -- Python/executor_cases.c.h | 27 -- Python/optimizer.c | 5 +- Python/optimizer_cases.c.h | 8 - 6 files changed, 181 insertions(+), 247 deletions(-) diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 071edfaa7a52be..7a33a5b84fd21a 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -109,80 +109,79 @@ extern "C" { #define _DO_CALL 375 #define _DO_CALL_FUNCTION_EX 376 #define _DO_CALL_KW 377 -#define _DYNAMIC_DEOPT 378 -#define _DYNAMIC_EXIT 379 +#define _DYNAMIC_EXIT 378 #define _END_FOR END_FOR #define _END_SEND END_SEND -#define _ERROR_POP_N 380 +#define _ERROR_POP_N 379 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _EXPAND_METHOD 381 -#define _EXPAND_METHOD_KW 382 -#define _FATAL_ERROR 383 +#define _EXPAND_METHOD 380 +#define _EXPAND_METHOD_KW 381 +#define _FATAL_ERROR 382 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 384 -#define _FOR_ITER_GEN_FRAME 385 -#define _FOR_ITER_TIER_TWO 386 +#define _FOR_ITER 383 +#define _FOR_ITER_GEN_FRAME 384 +#define _FOR_ITER_TIER_TWO 385 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BINARY_OP_EXTEND 387 -#define _GUARD_CALLABLE_ISINSTANCE 388 -#define _GUARD_CALLABLE_LEN 389 -#define _GUARD_CALLABLE_LIST_APPEND 390 -#define _GUARD_CALLABLE_STR_1 391 -#define _GUARD_CALLABLE_TUPLE_1 392 -#define _GUARD_CALLABLE_TYPE_1 393 -#define _GUARD_DORV_NO_DICT 394 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 395 -#define _GUARD_GLOBALS_VERSION 396 -#define _GUARD_IP_RETURN_GENERATOR 397 -#define _GUARD_IP_RETURN_VALUE 398 -#define _GUARD_IP_YIELD_VALUE 399 -#define _GUARD_IP__PUSH_FRAME 400 -#define _GUARD_IS_FALSE_POP 401 -#define _GUARD_IS_NONE_POP 402 -#define _GUARD_IS_NOT_NONE_POP 403 -#define _GUARD_IS_TRUE_POP 404 -#define _GUARD_KEYS_VERSION 405 -#define _GUARD_NOS_DICT 406 -#define _GUARD_NOS_FLOAT 407 -#define _GUARD_NOS_INT 408 -#define _GUARD_NOS_LIST 409 -#define _GUARD_NOS_NOT_NULL 410 -#define _GUARD_NOS_NULL 411 -#define _GUARD_NOS_OVERFLOWED 412 -#define _GUARD_NOS_TUPLE 413 -#define _GUARD_NOS_UNICODE 414 -#define _GUARD_NOT_EXHAUSTED_LIST 415 -#define _GUARD_NOT_EXHAUSTED_RANGE 416 -#define _GUARD_NOT_EXHAUSTED_TUPLE 417 -#define _GUARD_THIRD_NULL 418 -#define _GUARD_TOS_ANY_SET 419 -#define _GUARD_TOS_DICT 420 -#define _GUARD_TOS_FLOAT 421 -#define _GUARD_TOS_INT 422 -#define _GUARD_TOS_LIST 423 -#define _GUARD_TOS_OVERFLOWED 424 -#define _GUARD_TOS_SLICE 425 -#define _GUARD_TOS_TUPLE 426 -#define _GUARD_TOS_UNICODE 427 -#define _GUARD_TYPE_VERSION 428 -#define _GUARD_TYPE_VERSION_AND_LOCK 429 -#define _HANDLE_PENDING_AND_DEOPT 430 +#define _GUARD_BINARY_OP_EXTEND 386 +#define _GUARD_CALLABLE_ISINSTANCE 387 +#define _GUARD_CALLABLE_LEN 388 +#define _GUARD_CALLABLE_LIST_APPEND 389 +#define _GUARD_CALLABLE_STR_1 390 +#define _GUARD_CALLABLE_TUPLE_1 391 +#define _GUARD_CALLABLE_TYPE_1 392 +#define _GUARD_DORV_NO_DICT 393 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 394 +#define _GUARD_GLOBALS_VERSION 395 +#define _GUARD_IP_RETURN_GENERATOR 396 +#define _GUARD_IP_RETURN_VALUE 397 +#define _GUARD_IP_YIELD_VALUE 398 +#define _GUARD_IP__PUSH_FRAME 399 +#define _GUARD_IS_FALSE_POP 400 +#define _GUARD_IS_NONE_POP 401 +#define _GUARD_IS_NOT_NONE_POP 402 +#define _GUARD_IS_TRUE_POP 403 +#define _GUARD_KEYS_VERSION 404 +#define _GUARD_NOS_DICT 405 +#define _GUARD_NOS_FLOAT 406 +#define _GUARD_NOS_INT 407 +#define _GUARD_NOS_LIST 408 +#define _GUARD_NOS_NOT_NULL 409 +#define _GUARD_NOS_NULL 410 +#define _GUARD_NOS_OVERFLOWED 411 +#define _GUARD_NOS_TUPLE 412 +#define _GUARD_NOS_UNICODE 413 +#define _GUARD_NOT_EXHAUSTED_LIST 414 +#define _GUARD_NOT_EXHAUSTED_RANGE 415 +#define _GUARD_NOT_EXHAUSTED_TUPLE 416 +#define _GUARD_THIRD_NULL 417 +#define _GUARD_TOS_ANY_SET 418 +#define _GUARD_TOS_DICT 419 +#define _GUARD_TOS_FLOAT 420 +#define _GUARD_TOS_INT 421 +#define _GUARD_TOS_LIST 422 +#define _GUARD_TOS_OVERFLOWED 423 +#define _GUARD_TOS_SLICE 424 +#define _GUARD_TOS_TUPLE 425 +#define _GUARD_TOS_UNICODE 426 +#define _GUARD_TYPE_VERSION 427 +#define _GUARD_TYPE_VERSION_AND_LOCK 428 +#define _HANDLE_PENDING_AND_DEOPT 429 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 431 -#define _INIT_CALL_PY_EXACT_ARGS 432 -#define _INIT_CALL_PY_EXACT_ARGS_0 433 -#define _INIT_CALL_PY_EXACT_ARGS_1 434 -#define _INIT_CALL_PY_EXACT_ARGS_2 435 -#define _INIT_CALL_PY_EXACT_ARGS_3 436 -#define _INIT_CALL_PY_EXACT_ARGS_4 437 -#define _INSERT_NULL 438 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 430 +#define _INIT_CALL_PY_EXACT_ARGS 431 +#define _INIT_CALL_PY_EXACT_ARGS_0 432 +#define _INIT_CALL_PY_EXACT_ARGS_1 433 +#define _INIT_CALL_PY_EXACT_ARGS_2 434 +#define _INIT_CALL_PY_EXACT_ARGS_3 435 +#define _INIT_CALL_PY_EXACT_ARGS_4 436 +#define _INSERT_NULL 437 #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER #define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION #define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD @@ -192,179 +191,178 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _IS_NONE 439 +#define _IS_NONE 438 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 440 -#define _ITER_CHECK_RANGE 441 -#define _ITER_CHECK_TUPLE 442 -#define _ITER_JUMP_LIST 443 -#define _ITER_JUMP_RANGE 444 -#define _ITER_JUMP_TUPLE 445 -#define _ITER_NEXT_LIST 446 -#define _ITER_NEXT_LIST_TIER_TWO 447 -#define _ITER_NEXT_RANGE 448 -#define _ITER_NEXT_TUPLE 449 +#define _ITER_CHECK_LIST 439 +#define _ITER_CHECK_RANGE 440 +#define _ITER_CHECK_TUPLE 441 +#define _ITER_JUMP_LIST 442 +#define _ITER_JUMP_RANGE 443 +#define _ITER_JUMP_TUPLE 444 +#define _ITER_NEXT_LIST 445 +#define _ITER_NEXT_LIST_TIER_TWO 446 +#define _ITER_NEXT_RANGE 447 +#define _ITER_NEXT_TUPLE 448 #define _JUMP_BACKWARD_NO_INTERRUPT JUMP_BACKWARD_NO_INTERRUPT -#define _JUMP_TO_TOP 450 +#define _JUMP_TO_TOP 449 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 451 -#define _LOAD_ATTR_CLASS 452 +#define _LOAD_ATTR 450 +#define _LOAD_ATTR_CLASS 451 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 453 -#define _LOAD_ATTR_METHOD_LAZY_DICT 454 -#define _LOAD_ATTR_METHOD_NO_DICT 455 -#define _LOAD_ATTR_METHOD_WITH_VALUES 456 -#define _LOAD_ATTR_MODULE 457 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 458 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 459 -#define _LOAD_ATTR_PROPERTY_FRAME 460 -#define _LOAD_ATTR_SLOT 461 -#define _LOAD_ATTR_WITH_HINT 462 +#define _LOAD_ATTR_INSTANCE_VALUE 452 +#define _LOAD_ATTR_METHOD_LAZY_DICT 453 +#define _LOAD_ATTR_METHOD_NO_DICT 454 +#define _LOAD_ATTR_METHOD_WITH_VALUES 455 +#define _LOAD_ATTR_MODULE 456 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 457 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 458 +#define _LOAD_ATTR_PROPERTY_FRAME 459 +#define _LOAD_ATTR_SLOT 460 +#define _LOAD_ATTR_WITH_HINT 461 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _LOAD_BYTECODE 463 +#define _LOAD_BYTECODE 462 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 464 -#define _LOAD_CONST_INLINE_BORROW 465 -#define _LOAD_CONST_UNDER_INLINE 466 -#define _LOAD_CONST_UNDER_INLINE_BORROW 467 +#define _LOAD_CONST_INLINE 463 +#define _LOAD_CONST_INLINE_BORROW 464 +#define _LOAD_CONST_UNDER_INLINE 465 +#define _LOAD_CONST_UNDER_INLINE_BORROW 466 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 468 -#define _LOAD_FAST_0 469 -#define _LOAD_FAST_1 470 -#define _LOAD_FAST_2 471 -#define _LOAD_FAST_3 472 -#define _LOAD_FAST_4 473 -#define _LOAD_FAST_5 474 -#define _LOAD_FAST_6 475 -#define _LOAD_FAST_7 476 +#define _LOAD_FAST 467 +#define _LOAD_FAST_0 468 +#define _LOAD_FAST_1 469 +#define _LOAD_FAST_2 470 +#define _LOAD_FAST_3 471 +#define _LOAD_FAST_4 472 +#define _LOAD_FAST_5 473 +#define _LOAD_FAST_6 474 +#define _LOAD_FAST_7 475 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR -#define _LOAD_FAST_BORROW 477 -#define _LOAD_FAST_BORROW_0 478 -#define _LOAD_FAST_BORROW_1 479 -#define _LOAD_FAST_BORROW_2 480 -#define _LOAD_FAST_BORROW_3 481 -#define _LOAD_FAST_BORROW_4 482 -#define _LOAD_FAST_BORROW_5 483 -#define _LOAD_FAST_BORROW_6 484 -#define _LOAD_FAST_BORROW_7 485 +#define _LOAD_FAST_BORROW 476 +#define _LOAD_FAST_BORROW_0 477 +#define _LOAD_FAST_BORROW_1 478 +#define _LOAD_FAST_BORROW_2 479 +#define _LOAD_FAST_BORROW_3 480 +#define _LOAD_FAST_BORROW_4 481 +#define _LOAD_FAST_BORROW_5 482 +#define _LOAD_FAST_BORROW_6 483 +#define _LOAD_FAST_BORROW_7 484 #define _LOAD_FAST_BORROW_LOAD_FAST_BORROW LOAD_FAST_BORROW_LOAD_FAST_BORROW #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 486 -#define _LOAD_GLOBAL_BUILTINS 487 -#define _LOAD_GLOBAL_MODULE 488 +#define _LOAD_GLOBAL 485 +#define _LOAD_GLOBAL_BUILTINS 486 +#define _LOAD_GLOBAL_MODULE 487 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 489 -#define _LOAD_SMALL_INT_0 490 -#define _LOAD_SMALL_INT_1 491 -#define _LOAD_SMALL_INT_2 492 -#define _LOAD_SMALL_INT_3 493 -#define _LOAD_SPECIAL 494 +#define _LOAD_SMALL_INT 488 +#define _LOAD_SMALL_INT_0 489 +#define _LOAD_SMALL_INT_1 490 +#define _LOAD_SMALL_INT_2 491 +#define _LOAD_SMALL_INT_3 492 +#define _LOAD_SPECIAL 493 #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _MAKE_CALLARGS_A_TUPLE 495 +#define _MAKE_CALLARGS_A_TUPLE 494 #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_WARM 496 +#define _MAKE_WARM 495 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 497 -#define _MAYBE_EXPAND_METHOD_KW 498 -#define _MONITOR_CALL 499 -#define _MONITOR_CALL_KW 500 -#define _MONITOR_JUMP_BACKWARD 501 -#define _MONITOR_RESUME 502 +#define _MAYBE_EXPAND_METHOD 496 +#define _MAYBE_EXPAND_METHOD_KW 497 +#define _MONITOR_CALL 498 +#define _MONITOR_CALL_KW 499 +#define _MONITOR_JUMP_BACKWARD 500 +#define _MONITOR_RESUME 501 #define _NOP NOP -#define _POP_CALL 503 -#define _POP_CALL_LOAD_CONST_INLINE_BORROW 504 -#define _POP_CALL_ONE 505 -#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 506 -#define _POP_CALL_TWO 507 -#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 508 +#define _POP_CALL 502 +#define _POP_CALL_LOAD_CONST_INLINE_BORROW 503 +#define _POP_CALL_ONE 504 +#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 505 +#define _POP_CALL_TWO 506 +#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 507 #define _POP_EXCEPT POP_EXCEPT #define _POP_ITER POP_ITER -#define _POP_JUMP_IF_FALSE 509 -#define _POP_JUMP_IF_TRUE 510 +#define _POP_JUMP_IF_FALSE 508 +#define _POP_JUMP_IF_TRUE 509 #define _POP_TOP POP_TOP -#define _POP_TOP_FLOAT 511 -#define _POP_TOP_INT 512 -#define _POP_TOP_LOAD_CONST_INLINE 513 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 514 -#define _POP_TOP_NOP 515 -#define _POP_TOP_UNICODE 516 -#define _POP_TWO 517 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW 518 +#define _POP_TOP_FLOAT 510 +#define _POP_TOP_INT 511 +#define _POP_TOP_LOAD_CONST_INLINE 512 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 513 +#define _POP_TOP_NOP 514 +#define _POP_TOP_UNICODE 515 +#define _POP_TWO 516 +#define _POP_TWO_LOAD_CONST_INLINE_BORROW 517 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 519 +#define _PUSH_FRAME 518 #define _PUSH_NULL PUSH_NULL -#define _PUSH_NULL_CONDITIONAL 520 -#define _PY_FRAME_GENERAL 521 -#define _PY_FRAME_KW 522 -#define _QUICKEN_RESUME 523 -#define _REPLACE_WITH_TRUE 524 +#define _PUSH_NULL_CONDITIONAL 519 +#define _PY_FRAME_GENERAL 520 +#define _PY_FRAME_KW 521 +#define _QUICKEN_RESUME 522 +#define _REPLACE_WITH_TRUE 523 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 525 -#define _SEND 526 -#define _SEND_GEN_FRAME 527 +#define _SAVE_RETURN_OFFSET 524 +#define _SEND 525 +#define _SEND_GEN_FRAME 526 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_DYNAMIC_EXECUTOR 528 -#define _START_EXECUTOR 529 -#define _STORE_ATTR 530 -#define _STORE_ATTR_INSTANCE_VALUE 531 -#define _STORE_ATTR_SLOT 532 -#define _STORE_ATTR_WITH_HINT 533 +#define _START_EXECUTOR 527 +#define _STORE_ATTR 528 +#define _STORE_ATTR_INSTANCE_VALUE 529 +#define _STORE_ATTR_SLOT 530 +#define _STORE_ATTR_WITH_HINT 531 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 534 -#define _STORE_FAST_0 535 -#define _STORE_FAST_1 536 -#define _STORE_FAST_2 537 -#define _STORE_FAST_3 538 -#define _STORE_FAST_4 539 -#define _STORE_FAST_5 540 -#define _STORE_FAST_6 541 -#define _STORE_FAST_7 542 +#define _STORE_FAST 532 +#define _STORE_FAST_0 533 +#define _STORE_FAST_1 534 +#define _STORE_FAST_2 535 +#define _STORE_FAST_3 536 +#define _STORE_FAST_4 537 +#define _STORE_FAST_5 538 +#define _STORE_FAST_6 539 +#define _STORE_FAST_7 540 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 543 -#define _STORE_SUBSCR 544 -#define _STORE_SUBSCR_DICT 545 -#define _STORE_SUBSCR_LIST_INT 546 -#define _SWAP 547 -#define _SWAP_2 548 -#define _SWAP_3 549 -#define _TIER2_RESUME_CHECK 550 -#define _TO_BOOL 551 +#define _STORE_SLICE 541 +#define _STORE_SUBSCR 542 +#define _STORE_SUBSCR_DICT 543 +#define _STORE_SUBSCR_LIST_INT 544 +#define _SWAP 545 +#define _SWAP_2 546 +#define _SWAP_3 547 +#define _TIER2_RESUME_CHECK 548 +#define _TO_BOOL 549 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT -#define _TO_BOOL_LIST 552 +#define _TO_BOOL_LIST 550 #define _TO_BOOL_NONE TO_BOOL_NONE -#define _TO_BOOL_STR 553 +#define _TO_BOOL_STR 551 #define _UNARY_INVERT UNARY_INVERT #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 554 -#define _UNPACK_SEQUENCE_LIST 555 -#define _UNPACK_SEQUENCE_TUPLE 556 -#define _UNPACK_SEQUENCE_TWO_TUPLE 557 +#define _UNPACK_SEQUENCE 552 +#define _UNPACK_SEQUENCE_LIST 553 +#define _UNPACK_SEQUENCE_TUPLE 554 +#define _UNPACK_SEQUENCE_TWO_TUPLE 555 #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 557 +#define MAX_UOP_ID 555 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index bb519f53e34ba9..4b033a8b6c2d41 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -316,7 +316,6 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_SAVE_RETURN_OFFSET] = HAS_ARG_FLAG, [_EXIT_TRACE] = HAS_ESCAPES_FLAG, [_DYNAMIC_EXIT] = HAS_ESCAPES_FLAG, - [_DYNAMIC_DEOPT] = 0, [_CHECK_VALIDITY] = HAS_DEOPT_FLAG, [_LOAD_CONST_INLINE] = HAS_PURE_FLAG, [_POP_TOP_LOAD_CONST_INLINE] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, @@ -332,7 +331,6 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_CONST_UNDER_INLINE] = 0, [_LOAD_CONST_UNDER_INLINE_BORROW] = 0, [_START_EXECUTOR] = HAS_DEOPT_FLAG, - [_START_DYNAMIC_EXECUTOR] = HAS_DEOPT_FLAG, [_MAKE_WARM] = 0, [_FATAL_ERROR] = 0, [_DEOPT] = 0, @@ -452,7 +450,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_DEOPT] = "_DEOPT", [_DICT_MERGE] = "_DICT_MERGE", [_DICT_UPDATE] = "_DICT_UPDATE", - [_DYNAMIC_DEOPT] = "_DYNAMIC_DEOPT", [_DYNAMIC_EXIT] = "_DYNAMIC_EXIT", [_END_FOR] = "_END_FOR", [_END_SEND] = "_END_SEND", @@ -639,7 +636,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_SET_FUNCTION_ATTRIBUTE] = "_SET_FUNCTION_ATTRIBUTE", [_SET_IP] = "_SET_IP", [_SET_UPDATE] = "_SET_UPDATE", - [_START_DYNAMIC_EXECUTOR] = "_START_DYNAMIC_EXECUTOR", [_START_EXECUTOR] = "_START_EXECUTOR", [_STORE_ATTR] = "_STORE_ATTR", [_STORE_ATTR_INSTANCE_VALUE] = "_STORE_ATTR_INSTANCE_VALUE", @@ -1279,8 +1275,6 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _DYNAMIC_EXIT: return 0; - case _DYNAMIC_DEOPT: - return 0; case _CHECK_VALIDITY: return 0; case _LOAD_CONST_INLINE: @@ -1311,8 +1305,6 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _START_EXECUTOR: return 0; - case _START_DYNAMIC_EXECUTOR: - return 0; case _MAKE_WARM: return 0; case _FATAL_ERROR: diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 0a45291d4a5d7b..b57824af37559a 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5307,10 +5307,6 @@ dummy_func( GOTO_TIER_ONE(frame->instr_ptr); } - tier2 op(_DYNAMIC_DEOPT, (--)) { - GOTO_TIER_ONE(frame->instr_ptr); - } - tier2 op(_CHECK_VALIDITY, (--)) { DEOPT_IF(!current_executor->vm_data.valid); } @@ -5410,22 +5406,6 @@ dummy_func( } } - tier2 op(_START_DYNAMIC_EXECUTOR, (executor/4 --)) { -#ifndef _Py_JIT - assert(current_executor == (_PyExecutorObject*)executor); -#endif - assert(tstate->jit_exit == NULL || tstate->jit_exit->executor == current_executor); - tstate->current_executor = (PyObject *)executor; - if (!current_executor->vm_data.valid) { - assert(tstate->jit_exit->executor == current_executor); - assert(tstate->current_executor == executor); - _PyExecutor_ClearExit(tstate->jit_exit); - _Py_set_eval_breaker_bit(tstate, _PY_EVAL_JIT_DO_NOT_REENTER); - // Note: this points to _DYNAMIC_DEOPT!!! - DEOPT_IF(true); - } - } - tier2 op(_MAKE_WARM, (--)) { current_executor->vm_data.warm = true; } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index c6120a346a53f0..ada9a486d024d4 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7152,11 +7152,6 @@ break; } - case _DYNAMIC_DEOPT: { - GOTO_TIER_ONE(frame->instr_ptr); - break; - } - case _CHECK_VALIDITY: { if (!current_executor->vm_data.valid) { UOP_STAT_INC(uopcode, miss); @@ -7438,28 +7433,6 @@ break; } - case _START_DYNAMIC_EXECUTOR: { - PyObject *executor = (PyObject *)CURRENT_OPERAND0(); - #ifndef _Py_JIT - assert(current_executor == (_PyExecutorObject*)executor); - #endif - assert(tstate->jit_exit == NULL || tstate->jit_exit->executor == current_executor); - tstate->current_executor = (PyObject *)executor; - if (!current_executor->vm_data.valid) { - assert(tstate->jit_exit->executor == current_executor); - assert(tstate->current_executor == executor); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyExecutor_ClearExit(tstate->jit_exit); - stack_pointer = _PyFrame_GetStackPointer(frame); - _Py_set_eval_breaker_bit(tstate, _PY_EVAL_JIT_DO_NOT_REENTER); - if (true) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } - } - break; - } - case _MAKE_WARM: { current_executor->vm_data.warm = true; break; diff --git a/Python/optimizer.c b/Python/optimizer.c index 6531cd64bbd675..981482d980a201 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1129,7 +1129,7 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) } } if (opcode == _JUMP_TO_TOP) { - assert(buffer[0].opcode == _START_EXECUTOR || buffer[0].opcode == _START_DYNAMIC_EXECUTOR); + assert(buffer[0].opcode == _START_EXECUTOR); buffer[i].format = UOP_FORMAT_JUMP; buffer[i].jump_target = 1; } @@ -1211,8 +1211,7 @@ sanity_check(_PyExecutorObject *executor) opcode == _HANDLE_PENDING_AND_DEOPT || opcode == _EXIT_TRACE || opcode == _ERROR_POP_N || - opcode == _DYNAMIC_EXIT || - opcode == _DYNAMIC_DEOPT); + opcode == _DYNAMIC_EXIT); } } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 31affc93e7bb7d..c12f9af02f873e 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -3278,10 +3278,6 @@ break; } - case _DYNAMIC_DEOPT: { - break; - } - case _CHECK_VALIDITY: { break; } @@ -3407,10 +3403,6 @@ break; } - case _START_DYNAMIC_EXECUTOR: { - break; - } - case _MAKE_WARM: { break; } From 83c85c35434f90311e1e4ad28179559d62527278 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 6 Nov 2025 08:29:11 +0000 Subject: [PATCH 155/190] rework ad-hoc generation of guards --- Python/bytecodes.c | 28 +++++++--- Python/executor_cases.c.h | 58 +++++++++++++------ Tools/cases_generator/tier2_generator.py | 71 ++++++++++++++---------- 3 files changed, 101 insertions(+), 56 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index b57824af37559a..318443daebc89c 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5489,23 +5489,35 @@ dummy_func( } tier2 op(_GUARD_IP__PUSH_FRAME, (ip/4 --)) { - // Implementation automatically inserted by Tools/cases/tier2_generator.py - EXIT_IF(true); + _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_CORRESPONDING_UOP; + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += OFFSET_OF_CORRESPONDING_UOP; + EXIT_IF(true); + } } tier2 op(_GUARD_IP_YIELD_VALUE, (ip/4 --)) { - // Implementation automatically inserted by Tools/cases/tier2_generator.py - EXIT_IF(true); + _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_CORRESPONDING_UOP; + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += OFFSET_OF_CORRESPONDING_UOP; + EXIT_IF(true); + } } tier2 op(_GUARD_IP_RETURN_VALUE, (ip/4 --)) { - // Implementation automatically inserted by Tools/cases/tier2_generator.py - EXIT_IF(true); + _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_CORRESPONDING_UOP; + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += OFFSET_OF_CORRESPONDING_UOP; + EXIT_IF(true); + } } tier2 op(_GUARD_IP_RETURN_GENERATOR, (ip/4 --)) { - // Implementation automatically inserted by Tools/cases/tier2_generator.py - EXIT_IF(true); + _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_CORRESPONDING_UOP; + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += OFFSET_OF_CORRESPONDING_UOP; + EXIT_IF(true); + } } label(pop_2_error) { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index ada9a486d024d4..e9f101c8c3a4a3 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -8,6 +8,11 @@ #endif #define TIER_TWO 2 + #define OFFSET_OF_RETURN_VALUE ((frame->return_offset)) + #define OFFSET_OF_YIELD_VALUE ((1+INLINE_CACHE_ENTRIES_SEND)) + #define OFFSET_OF__PUSH_FRAME ((0)) + #define OFFSET_OF_RETURN_GENERATOR ((frame->return_offset)) + case _NOP: { break; } @@ -7525,44 +7530,61 @@ break; } - case _GUARD_IP_RETURN_VALUE: { + case _GUARD_IP__PUSH_FRAME: { PyObject *ip = (PyObject *)CURRENT_OPERAND0(); - if (frame->instr_ptr + (frame->return_offset) != (_Py_CODEUNIT *)ip) { - frame->instr_ptr += (frame->return_offset); - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); + _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF__PUSH_FRAME; + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += OFFSET_OF__PUSH_FRAME; + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } } break; } case _GUARD_IP_YIELD_VALUE: { PyObject *ip = (PyObject *)CURRENT_OPERAND0(); - if (frame->instr_ptr + (1+INLINE_CACHE_ENTRIES_SEND) != (_Py_CODEUNIT *)ip) { - frame->instr_ptr += (1+INLINE_CACHE_ENTRIES_SEND); - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); + _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_YIELD_VALUE; + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += OFFSET_OF_YIELD_VALUE; + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } } break; } - case _GUARD_IP__PUSH_FRAME: { + case _GUARD_IP_RETURN_VALUE: { PyObject *ip = (PyObject *)CURRENT_OPERAND0(); - if (frame->instr_ptr + (0) != (_Py_CODEUNIT *)ip) { - frame->instr_ptr += (0); - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); + _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_VALUE; + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += OFFSET_OF_RETURN_VALUE; + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } } break; } case _GUARD_IP_RETURN_GENERATOR: { PyObject *ip = (PyObject *)CURRENT_OPERAND0(); - if (frame->instr_ptr + (frame->return_offset) != (_Py_CODEUNIT *)ip) { - frame->instr_ptr += (frame->return_offset); - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); + _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_GENERATOR; + if (target != (_Py_CODEUNIT *)ip) { + frame->instr_ptr += OFFSET_OF_RETURN_GENERATOR; + if (true) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } } break; } + #undef OFFSET_OFRETURN_VALUE + #undef OFFSET_OFYIELD_VALUE + #undef OFFSET_OF_PUSH_FRAME + #undef OFFSET_OFRETURN_GENERATOR + #undef TIER_TWO diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index 8a9db068382a4f..ab37c489824927 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -63,6 +63,7 @@ class Tier2Emitter(Emitter): def __init__(self, out: CWriter, labels: dict[str, Label]): super().__init__(out, labels) self._replacers["oparg"] = self.oparg + self._replacers["OFFSET_OF_CORRESPONDING_UOP"] = self.offset_of_corresponding_uop def goto_error(self, offset: int, storage: Storage) -> str: # To do: Add jump targets for popping values. @@ -134,6 +135,19 @@ def oparg( self.out.emit_at(uop.name[-1], tkn) return True + def offset_of_corresponding_uop( + self, + tkn: Token, + tkn_iter: TokenIterator, + uop: CodeSection, + storage: Storage, + inst: Instruction | None, + ) -> bool: + assert uop.name.startswith("_GUARD_IP") + rest = uop.name[len("_GUARD_IP"):] + self.emit(f" OFFSET_OF{rest};\n") + next(tkn_iter) + return True def write_uop(uop: Uop, emitter: Emitter, stack: Stack) -> Stack: locals: dict[str, Local] = {} @@ -165,33 +179,6 @@ def write_uop(uop: Uop, emitter: Emitter, stack: Stack) -> Stack: SKIPS = ("_EXTENDED_ARG",) -def generate_guard_ips( - analysis: Analysis, - emitter: Tier2Emitter, -) -> None: - for name, uop in analysis.uops.items(): - for stmt in uop.body.body: - tkn_iter = iter(stmt.tokens()) - for token in tkn_iter: - if token.kind == "IDENTIFIER" and token.text == "LOAD_IP": - offset = [] - while token.kind != "SEMI": - offset.append(token.text) - token = next(tkn_iter) - # 1: to remove the LOAD_IP text - offset_str = "".join(offset[1:]) - emitter.emit(f"case _GUARD_IP_{name}: {{\n") - emitter.emit("PyObject *ip = (PyObject *)CURRENT_OPERAND0();\n") - emitter.emit(f"if (frame->instr_ptr + {offset_str} != (_Py_CODEUNIT *)ip) {{\n") - emitter.emit(f"frame->instr_ptr += {offset_str};\n") - emitter.emit(f"UOP_STAT_INC(uopcode, miss);\n") - emitter.emit("JUMP_TO_JUMP_TARGET();\n") - emitter.emit("}\n") - emitter.emit("break;\n") - emitter.emit("}\n") - emitter.emit("\n") - - def generate_tier2( filenames: list[str], analysis: Analysis, outfile: TextIO, lines: bool ) -> None: @@ -207,13 +194,35 @@ def generate_tier2( out = CWriter(outfile, 2, lines) emitter = Tier2Emitter(out, analysis.labels) out.emit("\n") + offset_strs: list[tuple[str, str]] = [] + for name, uop in analysis.uops.items(): + if not f"_GUARD_IP_{name}" in analysis.uops: + continue + tkn_iter = uop.body.tokens() + found = False + offset_str = "" + for token in tkn_iter: + if token.kind == "IDENTIFIER" and token.text == "LOAD_IP": + if found: + raise analysis_error("Cannot have two LOAD_IP in a guarded single uop.", uop.body.open) + offset = [] + while token.kind != "SEMI": + offset.append(token.text) + token = next(tkn_iter) + # 1: to remove the LOAD_IP text + offset_str = "".join(offset[1:]) + found = True + assert offset_str + out.emit(f"#define OFFSET_OF_{name} ({offset_str})\n") + offset_strs.append((name, offset_str)) + + out.emit("\n") + for name, uop in analysis.uops.items(): if uop.properties.tier == 1: continue if uop.is_super(): continue - if name.startswith("_GUARD_IP"): - continue why_not_viable = uop.why_not_viable() if why_not_viable is not None: out.emit( @@ -231,7 +240,9 @@ def generate_tier2( out.emit("}") out.emit("\n\n") - generate_guard_ips(analysis, emitter) + for name, offset_str in offset_strs: + out.emit(f"#undef OFFSET_OF{name}\n") + out.emit("\n") outfile.write("#undef TIER_TWO\n") From 6c77ee38980261bb4e5908d668d191673cac9386 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 6 Nov 2025 08:44:03 +0000 Subject: [PATCH 156/190] fix jit builds --- Python/executor_cases.c.h | 8 ++++---- Tools/cases_generator/tier2_generator.py | 9 +++++---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index e9f101c8c3a4a3..7e9c672f6c2e98 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -8,10 +8,6 @@ #endif #define TIER_TWO 2 - #define OFFSET_OF_RETURN_VALUE ((frame->return_offset)) - #define OFFSET_OF_YIELD_VALUE ((1+INLINE_CACHE_ENTRIES_SEND)) - #define OFFSET_OF__PUSH_FRAME ((0)) - #define OFFSET_OF_RETURN_GENERATOR ((frame->return_offset)) case _NOP: { break; @@ -7531,6 +7527,7 @@ } case _GUARD_IP__PUSH_FRAME: { + #define OFFSET_OF__PUSH_FRAME ((0)) PyObject *ip = (PyObject *)CURRENT_OPERAND0(); _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF__PUSH_FRAME; if (target != (_Py_CODEUNIT *)ip) { @@ -7544,6 +7541,7 @@ } case _GUARD_IP_YIELD_VALUE: { + #define OFFSET_OF_YIELD_VALUE ((1+INLINE_CACHE_ENTRIES_SEND)) PyObject *ip = (PyObject *)CURRENT_OPERAND0(); _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_YIELD_VALUE; if (target != (_Py_CODEUNIT *)ip) { @@ -7557,6 +7555,7 @@ } case _GUARD_IP_RETURN_VALUE: { + #define OFFSET_OF_RETURN_VALUE ((frame->return_offset)) PyObject *ip = (PyObject *)CURRENT_OPERAND0(); _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_VALUE; if (target != (_Py_CODEUNIT *)ip) { @@ -7570,6 +7569,7 @@ } case _GUARD_IP_RETURN_GENERATOR: { + #define OFFSET_OF_RETURN_GENERATOR ((frame->return_offset)) PyObject *ip = (PyObject *)CURRENT_OPERAND0(); _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_RETURN_GENERATOR; if (target != (_Py_CODEUNIT *)ip) { diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index ab37c489824927..c23eb4a50f4a7c 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -194,7 +194,7 @@ def generate_tier2( out = CWriter(outfile, 2, lines) emitter = Tier2Emitter(out, analysis.labels) out.emit("\n") - offset_strs: list[tuple[str, str]] = [] + offset_strs: dict[str, tuple[str, str]] = {} for name, uop in analysis.uops.items(): if not f"_GUARD_IP_{name}" in analysis.uops: continue @@ -213,8 +213,7 @@ def generate_tier2( offset_str = "".join(offset[1:]) found = True assert offset_str - out.emit(f"#define OFFSET_OF_{name} ({offset_str})\n") - offset_strs.append((name, offset_str)) + offset_strs[f"_GUARD_IP_{name}"] = (name, offset_str) out.emit("\n") @@ -232,6 +231,8 @@ def generate_tier2( out.emit(f"case {uop.name}: {{\n") declare_variables(uop, out) stack = Stack() + if name_offset_pair := offset_strs.get(name): + out.emit(f"#define OFFSET_OF_{name_offset_pair[0]} ({name_offset_pair[1]})\n") stack = write_uop(uop, emitter, stack) out.start_line() if not uop.properties.always_exits: @@ -240,7 +241,7 @@ def generate_tier2( out.emit("}") out.emit("\n\n") - for name, offset_str in offset_strs: + for name, offset_str in offset_strs.values(): out.emit(f"#undef OFFSET_OF{name}\n") out.emit("\n") outfile.write("#undef TIER_TWO\n") From 3fd3ab51f073194f6badfe887f1efd0525bf7f9a Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 6 Nov 2025 08:48:24 +0000 Subject: [PATCH 157/190] more future-proofing --- Python/executor_cases.c.h | 8 ++++---- Tools/cases_generator/tier2_generator.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 7e9c672f6c2e98..1e5230bad9764d 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7537,6 +7537,7 @@ JUMP_TO_JUMP_TARGET(); } } + #undef OFFSET_OF__PUSH_FRAME break; } @@ -7551,6 +7552,7 @@ JUMP_TO_JUMP_TARGET(); } } + #undef OFFSET_OF_YIELD_VALUE break; } @@ -7565,6 +7567,7 @@ JUMP_TO_JUMP_TARGET(); } } + #undef OFFSET_OF_RETURN_VALUE break; } @@ -7579,12 +7582,9 @@ JUMP_TO_JUMP_TARGET(); } } + #undef OFFSET_OF_RETURN_GENERATOR break; } - #undef OFFSET_OFRETURN_VALUE - #undef OFFSET_OFYIELD_VALUE - #undef OFFSET_OF_PUSH_FRAME - #undef OFFSET_OFRETURN_GENERATOR #undef TIER_TWO diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index c23eb4a50f4a7c..9cba92d7ee7575 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -234,6 +234,8 @@ def generate_tier2( if name_offset_pair := offset_strs.get(name): out.emit(f"#define OFFSET_OF_{name_offset_pair[0]} ({name_offset_pair[1]})\n") stack = write_uop(uop, emitter, stack) + if name_offset_pair: + out.emit(f"#undef OFFSET_OF_{name_offset_pair[0]}\n") out.start_line() if not uop.properties.always_exits: out.emit("break;\n") @@ -241,8 +243,6 @@ def generate_tier2( out.emit("}") out.emit("\n\n") - for name, offset_str in offset_strs.values(): - out.emit(f"#undef OFFSET_OF{name}\n") out.emit("\n") outfile.write("#undef TIER_TWO\n") From 6429b2f88b1b3f83e8951393d0dbf7de7a45081c Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 6 Nov 2025 19:22:50 +0000 Subject: [PATCH 158/190] lint --- Tools/cases_generator/tier2_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index 9cba92d7ee7575..bff1a4370b48be 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -235,7 +235,7 @@ def generate_tier2( out.emit(f"#define OFFSET_OF_{name_offset_pair[0]} ({name_offset_pair[1]})\n") stack = write_uop(uop, emitter, stack) if name_offset_pair: - out.emit(f"#undef OFFSET_OF_{name_offset_pair[0]}\n") + out.emit(f"#undef OFFSET_OF_{name_offset_pair[0]}\n") out.start_line() if not uop.properties.always_exits: out.emit("break;\n") From 5cfc7a9c7647808c930b070e8e734ee4b9d20883 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 6 Nov 2025 20:31:24 +0000 Subject: [PATCH 159/190] special case first instr properly --- Python/optimizer.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/Python/optimizer.c b/Python/optimizer.c index 981482d980a201..cef8ae4fadce95 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -628,6 +628,16 @@ _PyJit_translate_single_bytecode_to_trace( goto done; } + /* Special case the first instruction, + * so that we can guarantee forward progress */ + if (progress_needed && tstate->interp->jit_state.code_curr_size <= 2) { + if (OPCODE_HAS_EXIT(opcode) || OPCODE_HAS_DEOPT(opcode)) { + opcode = _PyOpcode_Deopt[opcode]; + } + assert(!OPCODE_HAS_EXIT(opcode)); + assert(!OPCODE_HAS_DEOPT(opcode)); + } + // Strange control-flow bool has_dynamic_jump_taken = OPCODE_HAS_UNPREDICTABLE_JUMP(opcode) && (next_instr != this_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]); @@ -696,15 +706,6 @@ _PyJit_translate_single_bytecode_to_trace( assert(opcode != ENTER_EXECUTOR && opcode != EXTENDED_ARG); assert(!_PyErr_Occurred(tstate)); - /* Special case the first instruction, - * so that we can guarantee forward progress */ - if (progress_needed && tstate->interp->jit_state.code_curr_size <= 2) { - if (OPCODE_HAS_EXIT(opcode) || OPCODE_HAS_DEOPT(opcode)) { - opcode = _PyOpcode_Deopt[opcode]; - } - assert(!OPCODE_HAS_EXIT(opcode)); - assert(!OPCODE_HAS_DEOPT(opcode)); - } if (OPCODE_HAS_EXIT(opcode)) { // Make space for side exit and final _EXIT_TRACE: From c5de2756564af7e9e9da654f54ad2ce0afec8f9d Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 6 Nov 2025 20:33:05 +0000 Subject: [PATCH 160/190] move strange control flow detection up --- Python/optimizer.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Python/optimizer.c b/Python/optimizer.c index cef8ae4fadce95..b31b4ddfefe2f5 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -628,6 +628,14 @@ _PyJit_translate_single_bytecode_to_trace( goto done; } + // Strange control-flow + bool has_dynamic_jump_taken = OPCODE_HAS_UNPREDICTABLE_JUMP(opcode) && + (next_instr != this_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]); + if (has_dynamic_jump_taken) { + DPRINTF(2, "Unsupported: dynamic jump taken\n"); + goto unsupported; + } + /* Special case the first instruction, * so that we can guarantee forward progress */ if (progress_needed && tstate->interp->jit_state.code_curr_size <= 2) { @@ -638,14 +646,6 @@ _PyJit_translate_single_bytecode_to_trace( assert(!OPCODE_HAS_DEOPT(opcode)); } - // Strange control-flow - bool has_dynamic_jump_taken = OPCODE_HAS_UNPREDICTABLE_JUMP(opcode) && - (next_instr != this_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]); - if (has_dynamic_jump_taken) { - DPRINTF(2, "Unsupported: dynamic jump taken\n"); - goto unsupported; - } - // This happens when a recursive call happens that we can't trace. Such as Python -> C -> Python calls // If we haven't guarded the IP, then it's untraceable. if (frame != tstate->interp->jit_state.prev_instr_frame && !needs_guard_ip) { From b7b3c2309b74b7f86bae225ae993b0393182baa2 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 6 Nov 2025 20:45:17 +0000 Subject: [PATCH 161/190] move code to correcdt places --- Python/optimizer.c | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/Python/optimizer.c b/Python/optimizer.c index b31b4ddfefe2f5..601f249e9c4cdb 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -605,6 +605,24 @@ _PyJit_translate_single_bytecode_to_trace( int old_stack_level = tstate->interp->jit_state.prev_instr_stacklevel; + // Strange control-flow + bool has_dynamic_jump_taken = OPCODE_HAS_UNPREDICTABLE_JUMP(opcode) && + (next_instr != this_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]); + if (has_dynamic_jump_taken) { + DPRINTF(2, "Unsupported: dynamic jump taken\n"); + goto unsupported; + } + + /* Special case the first instruction, + * so that we can guarantee forward progress */ + if (progress_needed && tstate->interp->jit_state.code_curr_size <= 3) { + if (OPCODE_HAS_EXIT(opcode) || OPCODE_HAS_DEOPT(opcode)) { + opcode = _PyOpcode_Deopt[opcode]; + } + assert(!OPCODE_HAS_EXIT(opcode)); + assert(!OPCODE_HAS_DEOPT(opcode)); + } + bool needs_guard_ip = _PyOpcode_NeedsGuardIp[opcode]; DPRINTF(2, "%p %d: %s(%d) %d %d\n", old_code, target, _PyOpcode_OpName[opcode], oparg, needs_guard_ip, old_stack_level); @@ -628,24 +646,6 @@ _PyJit_translate_single_bytecode_to_trace( goto done; } - // Strange control-flow - bool has_dynamic_jump_taken = OPCODE_HAS_UNPREDICTABLE_JUMP(opcode) && - (next_instr != this_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]); - if (has_dynamic_jump_taken) { - DPRINTF(2, "Unsupported: dynamic jump taken\n"); - goto unsupported; - } - - /* Special case the first instruction, - * so that we can guarantee forward progress */ - if (progress_needed && tstate->interp->jit_state.code_curr_size <= 2) { - if (OPCODE_HAS_EXIT(opcode) || OPCODE_HAS_DEOPT(opcode)) { - opcode = _PyOpcode_Deopt[opcode]; - } - assert(!OPCODE_HAS_EXIT(opcode)); - assert(!OPCODE_HAS_DEOPT(opcode)); - } - // This happens when a recursive call happens that we can't trace. Such as Python -> C -> Python calls // If we haven't guarded the IP, then it's untraceable. if (frame != tstate->interp->jit_state.prev_instr_frame && !needs_guard_ip) { From cda3dcea645fb9b6a74619cd99b1b4c12ecdb599 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 6 Nov 2025 21:41:12 +0000 Subject: [PATCH 162/190] fix a bug where we point FOR_ITER_TIER_TWO --- Python/optimizer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/optimizer.c b/Python/optimizer.c index 601f249e9c4cdb..845ed3d86c09dc 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -825,7 +825,7 @@ _PyJit_translate_single_bytecode_to_trace( if (uop == _TIER2_RESUME_CHECK) { target = next_inst; } - else if (uop != _FOR_ITER_TIER_TWO) { + else { int extended_arg = orig_oparg > 255; uint32_t jump_target = next_inst + orig_oparg + extended_arg; assert(_Py_GetBaseCodeUnit(old_code, jump_target).op.code == END_FOR); From 3f212a45982180aa9601b5957f18ee3af9cbfe06 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Thu, 6 Nov 2025 22:02:45 +0000 Subject: [PATCH 163/190] cleanup --- Include/internal/pycore_ceval.h | 3 +-- Python/bytecodes.c | 3 +-- Python/ceval_macros.h | 4 ---- Python/generated_cases.c.h | 3 +-- 4 files changed, 3 insertions(+), 10 deletions(-) diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 02da6195225610..33b9fd053f70cb 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -334,10 +334,9 @@ _PyEval_SpecialMethodCanSuggest(PyObject *self, int oparg); #define _PY_EVAL_PLEASE_STOP_BIT (1U << 5) #define _PY_EVAL_EXPLICIT_MERGE_BIT (1U << 6) #define _PY_EVAL_JIT_INVALIDATE_COLD_BIT (1U << 7) -#define _PY_EVAL_JIT_DO_NOT_REENTER (1U << 8) /* Reserve a few bits for future use */ -#define _PY_EVAL_EVENTS_BITS 9 +#define _PY_EVAL_EVENTS_BITS 8 #define _PY_EVAL_EVENTS_MASK ((1 << _PY_EVAL_EVENTS_BITS)-1) static inline void diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 318443daebc89c..2e720d3fb300da 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3049,8 +3049,7 @@ dummy_func( /* If the eval breaker is set then stay in tier 1. * This avoids any potentially infinite loops * involving _RESUME_CHECK */ - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & (_PY_EVAL_EVENTS_MASK | _PY_EVAL_JIT_DO_NOT_REENTER)) { - _Py_unset_eval_breaker_bit(tstate, _PY_EVAL_JIT_DO_NOT_REENTER); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { opcode = executor->vm_data.opcode; oparg = (oparg & ~255) | executor->vm_data.oparg; next_instr = this_instr; diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index e12040bcc848ec..a8290242a5c1ef 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -401,10 +401,6 @@ do { \ next_instr = frame->instr_ptr + 1; \ JUMP_TO_LABEL(error); \ } \ - /* Progress made */ \ - if (next_instr != this_instr) { \ - _Py_unset_eval_breaker_bit(tstate, _PY_EVAL_JIT_DO_NOT_REENTER); \ - } \ if (keep_tracing_bit) { \ assert(tstate->interp->jit_state.code_curr_size == 2 || tstate->interp->jit_state.code_curr_size == 3); \ ENTER_TRACING(); \ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index c97a6a9dad8bb8..e500ec93fff4ce 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5509,8 +5509,7 @@ assert(executor->vm_data.code == code); assert(executor->vm_data.valid); assert(tstate->current_executor == NULL); - if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & (_PY_EVAL_EVENTS_MASK | _PY_EVAL_JIT_DO_NOT_REENTER)) { - _Py_unset_eval_breaker_bit(tstate, _PY_EVAL_JIT_DO_NOT_REENTER); + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { opcode = executor->vm_data.opcode; oparg = (oparg & ~255) | executor->vm_data.oparg; next_instr = this_instr; From 4f29dd31e0a9d3353919b54aac2bb6002b9511a6 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 7 Nov 2025 20:17:32 +0000 Subject: [PATCH 164/190] Partially address review --- Include/internal/pycore_interp_structs.h | 5 +- Include/internal/pycore_optimizer.h | 11 ++-- Python/bytecodes.c | 21 ++++---- Python/ceval.c | 6 ++- Python/executor_cases.c.h | 14 +++--- Python/generated_cases.c.h | 4 +- Python/optimizer.c | 64 +++++++++++++----------- Python/optimizer_analysis.c | 1 + Python/optimizer_bytecodes.c | 4 ++ Python/optimizer_cases.c.h | 1 + 10 files changed, 69 insertions(+), 62 deletions(-) diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index e37cecdd1d54ff..b0f5d0abc3fcae 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -768,15 +768,14 @@ typedef struct _PyJitTracerState { int prev_instr_stacklevel; int specialize_counter; _PyUOpInstruction *code_buffer; - _Py_CODEUNIT *insert_exec_instr; + _Py_CODEUNIT *start_instr; _Py_CODEUNIT *close_loop_instr; + _Py_CODEUNIT *jump_backward_instr; PyCodeObject *initial_code; // Strong PyFunctionObject *initial_func; // Strong _Py_CODEUNIT *prev_instr; PyCodeObject *prev_instr_code; // Strong struct _PyExitData *prev_exit; - struct _PyExecutorObject *prev_executor; // Strong - _Py_CODEUNIT *jump_backward_instr; _PyInterpreterFrame *prev_instr_frame; _PyBloomFilter dependencies; } _PyJitTracerState; diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index a1d95c7c2f4aa9..28c6c3ecaea122 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -36,9 +36,9 @@ typedef struct { typedef struct _PyExitData { uint32_t target; - uint16_t index; - char is_dynamic:4; - char is_control_flow:4; + uint16_t index:14; + uint16_t is_dynamic:1; + uint16_t is_control_flow:1; _Py_BackoffCounter temperature; struct _PyExecutorObject *executor; } _PyExitData; @@ -351,6 +351,7 @@ static inline int is_terminator(const _PyUOpInstruction *uop) int opcode = uop->opcode; return ( opcode == _EXIT_TRACE || + opcode == _DEOPT || opcode == _JUMP_TO_TOP || opcode == _DYNAMIC_EXIT ); @@ -367,9 +368,9 @@ int _PyJit_translate_single_bytecode_to_trace(PyThreadState *tstate, _PyInterpre int _PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, - _Py_CODEUNIT *curr_instr, _Py_CODEUNIT *insert_exec_instr, + _Py_CODEUNIT *curr_instr, _Py_CODEUNIT *start_instr, _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit, - _PyExecutorObject *prev_exec, int oparg); + int oparg); void _PyJit_FinalizeTracing(PyThreadState *tstate); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 2e720d3fb300da..dfc7e2952420cd 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1220,7 +1220,7 @@ dummy_func( PyObject *result = PyStackRef_AsPyObjectSteal(retval); if (IS_JIT_TRACING()) { #if _Py_TIER2 - _PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr); + _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL); LEAVE_TRACING(); int err = bail_tracing_and_jit(tstate, frame); if (err < 0) { @@ -2983,7 +2983,7 @@ dummy_func( oparg >>= 8; insert_exec_at--; } - int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, NULL, oparg); + int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, oparg); if (succ) { ENTER_TRACING(); } @@ -5268,7 +5268,7 @@ dummy_func( tier2 op(_EXIT_TRACE, (exit_p/4 --)) { _PyExitData *exit = (_PyExitData *)exit_p; #if defined(Py_DEBUG) && !defined(_Py_JIT) - const _Py_CODEUNIT *target = ((frame->owner >= FRAME_OWNED_BY_INTERPRETER) + const _Py_CODEUNIT *target = ((frame->owner == FRAME_OWNED_BY_INTERPRETER) ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame)) + exit->target; OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); @@ -5285,8 +5285,6 @@ dummy_func( TIER2_TO_TIER2(exit->executor); } - // Note: this is different than _COLD_EXIT/_EXIT_TRACE, as it may lead to multiple executors - // from a single exit! tier2 op(_DYNAMIC_EXIT, (exit_p/4 --)) { #if defined(Py_DEBUG) && !defined(_Py_JIT) _PyExitData *exit = (_PyExitData *)exit_p; @@ -5415,7 +5413,8 @@ dummy_func( } tier2 op(_DEOPT, (--)) { - GOTO_TIER_ONE(_PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + GOTO_TIER_ONE((frame->owner == FRAME_OWNED_BY_INTERPRETER) + ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); } tier2 op(_HANDLE_PENDING_AND_DEOPT, (--)) { @@ -5445,8 +5444,8 @@ dummy_func( tier2 op(_COLD_EXIT, ( -- )) { _PyExitData *exit = tstate->jit_exit; assert(exit != NULL); - _Py_CODEUNIT *target = ((frame->owner >= FRAME_OWNED_BY_INTERPRETER) - ? (_Py_CODEUNIT *)_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame)) + exit->target; + assert(frame->owner < FRAME_OWNED_BY_INTERPRETER); + _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; _Py_BackoffCounter temperature = exit->temperature; _PyExecutorObject *executor; if (target->op.code == ENTER_EXECUTOR) { @@ -5458,9 +5457,6 @@ dummy_func( TIER2_TO_TIER2(exit->executor); } else { - if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) { - GOTO_TIER_ONE(target); - } if (!backoff_counter_triggers(temperature)) { exit->temperature = advance_backoff_counter(temperature); GOTO_TIER_ONE(target); @@ -5473,7 +5469,7 @@ dummy_func( // Note: it's safe to use target->op.arg here instead of the oparg given by EXTENDED_ARG. // The invariant in the optimizer is the deopt target always points back to the first EXTENDED_ARG. // So setting it to anything else is wrong. - int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, previous_executor, target->op.arg); + int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, target->op.arg); exit->temperature = restart_backoff_counter(exit->temperature); if (succ) { GOTO_TIER_ONE_CONTINUE_TRACING(target); @@ -5483,6 +5479,7 @@ dummy_func( } tier2 op(_COLD_DYNAMIC_EXIT, ( -- )) { + // TODO (gh-139109): This should be similar to _COLD_EXIT in the future. _Py_CODEUNIT *target = frame->instr_ptr; GOTO_TIER_ONE(target); } diff --git a/Python/ceval.c b/Python/ceval.c index 074b0f8b86e959..00c30cc3fd4bcf 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1105,9 +1105,11 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int _Py_CODEUNIT *next_instr; _PyStackRef *stack_pointer; entry.stack[0] = PyStackRef_NULL; - entry.frame.f_funcobj = PyStackRef_NULL; -#if defined(Py_DEBUG) +#ifdef Py_STACKREF_DEBUG + entry.frame.f_funcobj = PyStackRef_None; +#elif defined(Py_DEBUG) /* Set these to invalid but identifiable values for debugging. */ + entry.frame.f_funcobj = (_PyStackRef){.bits = 0xaaa0}; entry.frame.f_locals = (PyObject*)0xaaa1; entry.frame.frame_obj = (PyFrameObject*)0xaaa2; entry.frame.f_globals = (PyObject*)0xaaa3; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 1e5230bad9764d..16db3e2efcfc90 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7111,7 +7111,7 @@ PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); _PyExitData *exit = (_PyExitData *)exit_p; #if defined(Py_DEBUG) && !defined(_Py_JIT) - const _Py_CODEUNIT *target = ((frame->owner >= FRAME_OWNED_BY_INTERPRETER) + const _Py_CODEUNIT *target = ((frame->owner == FRAME_OWNED_BY_INTERPRETER) ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame)) + exit->target; OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); @@ -7446,7 +7446,8 @@ } case _DEOPT: { - GOTO_TIER_ONE(_PyFrame_GetBytecode(frame) + CURRENT_TARGET()); + GOTO_TIER_ONE((frame->owner == FRAME_OWNED_BY_INTERPRETER) + ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame) + CURRENT_TARGET()); break; } @@ -7487,8 +7488,8 @@ case _COLD_EXIT: { _PyExitData *exit = tstate->jit_exit; assert(exit != NULL); - _Py_CODEUNIT *target = ((frame->owner >= FRAME_OWNED_BY_INTERPRETER) - ? (_Py_CODEUNIT *)_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame)) + exit->target; + assert(frame->owner < FRAME_OWNED_BY_INTERPRETER); + _Py_CODEUNIT *target = _PyFrame_GetBytecode(frame) + exit->target; _Py_BackoffCounter temperature = exit->temperature; _PyExecutorObject *executor; if (target->op.code == ENTER_EXECUTOR) { @@ -7500,9 +7501,6 @@ TIER2_TO_TIER2(exit->executor); } else { - if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) { - GOTO_TIER_ONE(target); - } if (!backoff_counter_triggers(temperature)) { exit->temperature = advance_backoff_counter(temperature); GOTO_TIER_ONE(target); @@ -7510,7 +7508,7 @@ _PyExecutorObject *previous_executor = _PyExecutor_FromExit(exit); assert(tstate->current_executor == (PyObject *)previous_executor); int chain_depth = previous_executor->vm_data.chain_depth + !exit->is_control_flow; - int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, previous_executor, target->op.arg); + int succ = _PyJit_TryInitializeTracing(tstate, frame, target, target, target, STACK_LEVEL(), chain_depth, exit, target->op.arg); exit->temperature = restart_backoff_counter(exit->temperature); if (succ) { GOTO_TIER_ONE_CONTINUE_TRACING(target); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index e500ec93fff4ce..33ac4152bd5e22 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -7557,7 +7557,7 @@ stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - _PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr); + _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL); stack_pointer = _PyFrame_GetStackPointer(frame); LEAVE_TRACING(); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -7716,7 +7716,7 @@ oparg >>= 8; insert_exec_at--; } - int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, NULL, oparg); + int succ = _PyJit_TryInitializeTracing(tstate, frame, this_instr, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL, oparg); if (succ) { ENTER_TRACING(); } diff --git a/Python/optimizer.c b/Python/optimizer.c index 845ed3d86c09dc..19d86115154e7c 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -136,7 +136,7 @@ _PyOptimizer_Optimize( chain_depth %= MAX_CHAIN_DEPTH; bool progress_needed = chain_depth == 0; PyCodeObject *code = (PyCodeObject *)tstate->interp->jit_state.initial_code; - _Py_CODEUNIT *start = tstate->interp->jit_state.insert_exec_instr; + _Py_CODEUNIT *start = tstate->interp->jit_state.start_instr; if (progress_needed && !has_space_for_executor(code, start)) { interp->compiling = false; return 0; @@ -608,10 +608,6 @@ _PyJit_translate_single_bytecode_to_trace( // Strange control-flow bool has_dynamic_jump_taken = OPCODE_HAS_UNPREDICTABLE_JUMP(opcode) && (next_instr != this_instr + 1 + _PyOpcode_Caches[_PyOpcode_Deopt[opcode]]); - if (has_dynamic_jump_taken) { - DPRINTF(2, "Unsupported: dynamic jump taken\n"); - goto unsupported; - } /* Special case the first instruction, * so that we can guarantee forward progress */ @@ -624,6 +620,10 @@ _PyJit_translate_single_bytecode_to_trace( } bool needs_guard_ip = _PyOpcode_NeedsGuardIp[opcode]; + if (has_dynamic_jump_taken && !needs_guard_ip) { + DPRINTF(2, "Unsupported: dynamic jump taken\n"); + goto unsupported; + } DPRINTF(2, "%p %d: %s(%d) %d %d\n", old_code, target, _PyOpcode_OpName[opcode], oparg, needs_guard_ip, old_stack_level); #ifdef Py_DEBUG @@ -749,7 +749,7 @@ _PyJit_translate_single_bytecode_to_trace( case JUMP_BACKWARD_NO_INTERRUPT: { if ((next_instr != tstate->interp->jit_state.close_loop_instr) && - (next_instr != tstate->interp->jit_state.insert_exec_instr) && + (next_instr != tstate->interp->jit_state.start_instr) && tstate->interp->jit_state.code_curr_size > 5 && // These are coroutines, and we want to unroll those usually. opcode != JUMP_BACKWARD_NO_INTERRUPT) { @@ -760,7 +760,7 @@ _PyJit_translate_single_bytecode_to_trace( OPT_STAT_INC(inner_loop); ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); trace[trace_length-1].operand1 = true; // is_control_flow - DPRINTF(2, "JUMP_BACKWARD not to top ends trace %p %p %p\n", next_instr, tstate->interp->jit_state.close_loop_instr, tstate->interp->jit_state.insert_exec_instr); + DPRINTF(2, "JUMP_BACKWARD not to top ends trace %p %p %p\n", next_instr, tstate->interp->jit_state.close_loop_instr, tstate->interp->jit_state.start_instr); goto done; } break; @@ -772,7 +772,9 @@ _PyJit_translate_single_bytecode_to_trace( * start with RESUME_CHECK */ ADD_TO_TRACE(_TIER2_RESUME_CHECK, 0, 0, target); break; - + case INTERPRETER_EXIT: + ADD_TO_TRACE(_DEOPT, 0, 0, target); + goto done;; default: { const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; @@ -862,18 +864,18 @@ _PyJit_translate_single_bytecode_to_trace( PyCodeObject *new_code = (PyCodeObject *)PyStackRef_AsPyObjectBorrow(frame->f_executable); PyFunctionObject *new_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - if (new_func != NULL) { - operand = (uintptr_t)new_func; - DPRINTF(2, "Adding %p func to op\n", (void *)operand); - _Py_BloomFilter_Add(dependencies, new_func); - } - else if (new_code != NULL && !Py_IsNone((PyObject*)new_code)) { - operand = (uintptr_t)new_code | 1; - DPRINTF(2, "Adding %p code to op\n", (void *)operand); - _Py_BloomFilter_Add(dependencies, new_code); - } - else { - operand = 0; + operand = 0; + if (frame->owner < FRAME_OWNED_BY_INTERPRETER) { + if (new_func != NULL) { + operand = (uintptr_t)new_func; + DPRINTF(2, "Adding %p func to op\n", (void *)operand); + _Py_BloomFilter_Add(dependencies, new_func); + } + else if (new_code != NULL && !Py_IsNone((PyObject*)new_code)) { + operand = (uintptr_t)new_code | 1; + DPRINTF(2, "Adding %p code to op\n", (void *)operand); + _Py_BloomFilter_Add(dependencies, new_code); + } } ADD_TO_TRACE(uop, oparg, operand, target); trace[trace_length - 1].operand1 = PyStackRef_IsNone(frame->f_executable) ? 2 : ((int)(frame->stackpointer - _PyFrame_Stackbase(frame))); @@ -913,8 +915,11 @@ _PyJit_translate_single_bytecode_to_trace( } } // Loop back to the start - int is_first_instr = tstate->interp->jit_state.close_loop_instr == next_instr || tstate->interp->jit_state.insert_exec_instr == next_instr; + int is_first_instr = tstate->interp->jit_state.close_loop_instr == next_instr || tstate->interp->jit_state.start_instr == next_instr; if (is_first_instr && tstate->interp->jit_state.code_curr_size > 5) { + if (needs_guard_ip) { + ADD_TO_TRACE(_SET_IP, 0, (uintptr_t)next_instr, 0); + } ADD_TO_TRACE(_JUMP_TO_TOP, 0, 0, 0); goto done; } @@ -945,7 +950,10 @@ _PyJit_translate_single_bytecode_to_trace( // Returns 0 for do not enter tracing, 1 on enter tracing. int -_PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *curr_instr, _Py_CODEUNIT *insert_exec_instr, _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit, _PyExecutorObject *prev_exec, int oparg) +_PyJit_TryInitializeTracing( + PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *curr_instr, + _Py_CODEUNIT *start_instr, _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, + _PyExitData *exit, int oparg) { // A recursive trace. // Don't trace into the inner call because it will stomp on the previous trace, causing endless retraces. @@ -972,12 +980,12 @@ _PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _ chain_depth); #endif - add_to_trace(tstate->interp->jit_state.code_buffer, 0, _START_EXECUTOR, 0, (uintptr_t)insert_exec_instr, INSTR_IP(insert_exec_instr, code)); + add_to_trace(tstate->interp->jit_state.code_buffer, 0, _START_EXECUTOR, 0, (uintptr_t)start_instr, INSTR_IP(start_instr, code)); add_to_trace(tstate->interp->jit_state.code_buffer, 1, _MAKE_WARM, 0, 0, 0); tstate->interp->jit_state.code_curr_size = 2; tstate->interp->jit_state.code_max_size = UOP_MAX_TRACE_LENGTH; - tstate->interp->jit_state.insert_exec_instr = insert_exec_instr; + tstate->interp->jit_state.start_instr = start_instr; tstate->interp->jit_state.close_loop_instr = close_loop_instr; tstate->interp->jit_state.initial_code = (PyCodeObject *)Py_NewRef(code); tstate->interp->jit_state.initial_func = (PyFunctionObject *)Py_XNewRef(PyStackRef_AsPyObjectBorrow(frame->f_funcobj)); @@ -993,9 +1001,9 @@ _PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, _ tstate->interp->jit_state.prev_instr_oparg = oparg; tstate->interp->jit_state.prev_instr_stacklevel = curr_stackdepth; tstate->interp->jit_state.prev_instr_is_super = false; - assert(curr_instr->op.code == JUMP_BACKWARD_JIT || (prev_exec != NULL && exit != NULL)); + assert(curr_instr->op.code == JUMP_BACKWARD_JIT || (exit != NULL)); tstate->interp->jit_state.jump_backward_instr = curr_instr; - tstate->interp->jit_state.prev_executor = (_PyExecutorObject *)Py_XNewRef(prev_exec); + assert(curr_instr->op.code == JUMP_BACKWARD_JIT || (exit != NULL)); _Py_BloomFilter_Init(&tstate->interp->jit_state.dependencies); return 1; } @@ -1006,7 +1014,6 @@ _PyJit_FinalizeTracing(PyThreadState *tstate) Py_CLEAR(tstate->interp->jit_state.initial_code); Py_CLEAR(tstate->interp->jit_state.initial_func); Py_CLEAR(tstate->interp->jit_state.prev_instr_code); - Py_CLEAR(tstate->interp->jit_state.prev_executor); tstate->interp->jit_state.code_curr_size = 2; tstate->interp->jit_state.code_max_size = UOP_MAX_TRACE_LENGTH - 1; } @@ -1077,9 +1084,6 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) for (int i = 0; i < length; i++) { _PyUOpInstruction *inst = &buffer[i]; int opcode = inst->opcode; - if (inst->format != UOP_FORMAT_TARGET) { - fprintf(stdout, "I: %d\n", i); - } int32_t target = (int32_t)uop_get_target(inst); uint16_t exit_flags = _PyUop_Flags[opcode] & (HAS_EXIT_FLAG | HAS_DEOPT_FLAG | HAS_PERIODIC_FLAG); if (exit_flags) { diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index e1a6f29f47d1dc..ad5f1b0388d5a7 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -499,6 +499,7 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) } case _JUMP_TO_TOP: case _DYNAMIC_EXIT: + case _DEOPT: return pc + 1; } } diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 98dabe989aed14..fdc4e49aaf4445 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -1022,6 +1022,10 @@ dummy_func(void) { ctx->done = true; } + op(_DEOPT, (--)) { + ctx->done = true; + } + op(_REPLACE_WITH_TRUE, (value -- res)) { REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_True); res = sym_new_const(ctx, Py_True); diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index c12f9af02f873e..397502cec784f0 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -3412,6 +3412,7 @@ } case _DEOPT: { + ctx->done = true; break; } From 5af4b0aadcdb7da89b6a2ca8d011ea11c442285e Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 7 Nov 2025 20:21:54 +0000 Subject: [PATCH 165/190] remove nedsguardip table --- Include/internal/pycore_opcode_metadata.h | 94 ++++++------------- Include/internal/pycore_uop_metadata.h | 12 +-- Python/optimizer.c | 2 +- Tools/cases_generator/generators_common.py | 2 + .../opcode_metadata_generator.py | 14 +-- .../cases_generator/uop_metadata_generator.py | 4 +- 6 files changed, 44 insertions(+), 84 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 1c8c49a41060d3..18f8baf980e656 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1032,6 +1032,7 @@ enum InstructionFormat { #define HAS_NO_SAVE_IP_FLAG (8192) #define HAS_PERIODIC_FLAG (16384) #define HAS_UNPREDICTABLE_JUMP_FLAG (32768) +#define HAS_NEEDS_GUARD_IP_FLAG (65536) #define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG)) #define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_CONST_FLAG)) #define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NAME_FLAG)) @@ -1048,6 +1049,7 @@ enum InstructionFormat { #define OPCODE_HAS_NO_SAVE_IP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NO_SAVE_IP_FLAG)) #define OPCODE_HAS_PERIODIC(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_PERIODIC_FLAG)) #define OPCODE_HAS_UNPREDICTABLE_JUMP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_UNPREDICTABLE_JUMP_FLAG)) +#define OPCODE_HAS_NEEDS_GUARD_IP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NEEDS_GUARD_IP_FLAG)) #define OPARG_SIMPLE 0 #define OPARG_CACHE_1 1 @@ -1064,7 +1066,7 @@ enum InstructionFormat { struct opcode_metadata { uint8_t valid_entry; uint8_t instr_format; - uint16_t flags; + uint32_t flags; }; extern const struct opcode_metadata _PyOpcode_opcode_metadata[267]; @@ -1079,7 +1081,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG }, [BINARY_OP_SUBSCR_DICT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [BINARY_OP_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG }, + [BINARY_OP_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [BINARY_OP_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_SUBSCR_LIST_SLICE] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_SUBSCR_STR_INT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, @@ -1096,22 +1098,22 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [BUILD_TEMPLATE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [CACHE] = { true, INSTR_FMT_IX, 0 }, - [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [CALL_BOUND_METHOD_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [CALL_BOUND_METHOD_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [CALL_INTRINSIC_1] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_INTRINSIC_2] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_ISINSTANCE] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_KW_BOUND_METHOD] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [CALL_KW_BOUND_METHOD] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [CALL_KW_NON_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_KW_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_KW_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [CALL_LEN] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1119,8 +1121,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [CALL_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_NON_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [CALL_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [CALL_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [CALL_STR_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_TUPLE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_TYPE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, @@ -1145,7 +1147,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [DELETE_SUBSCR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, + [END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [END_FOR] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG }, [END_SEND] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_PURE_FLAG }, [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1154,7 +1156,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [FORMAT_SIMPLE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [FORMAT_WITH_SPEC] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, - [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, @@ -1166,13 +1168,13 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_FROM] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, + [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INSTRUMENTED_CALL_FUNCTION_EX] = { true, INSTR_FMT_IX, HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INSTRUMENTED_CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INSTRUMENTED_END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [INSTRUMENTED_END_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG }, [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, + [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [INSTRUMENTED_INSTRUCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_JUMP_FORWARD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, @@ -1185,9 +1187,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INTERPRETER_EXIT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INTERPRETER_EXIT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [IS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_BACKWARD_JIT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1199,7 +1201,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [LOAD_ATTR] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG }, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG }, @@ -1207,7 +1209,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [LOAD_ATTR_MODULE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [LOAD_ATTR_PROPERTY] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [LOAD_ATTR_SLOT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1255,10 +1257,10 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [RESERVED] = { true, INSTR_FMT_IX, 0 }, [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [RESUME_CHECK] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, - [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, - [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, - [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [SETUP_ANNOTATIONS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SET_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SET_FUNCTION_ATTRIBUTE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, @@ -1294,7 +1296,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [UNPACK_SEQUENCE_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [WITH_EXCEPT_START] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [ANNOTATIONS_PLACEHOLDER] = { true, -1, HAS_PURE_FLAG }, [JUMP] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_IF_FALSE] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1774,40 +1776,6 @@ const uint8_t _PyOpcode_Caches[256] = { }; #endif -extern const uint8_t _PyOpcode_NeedsGuardIp[256]; -#ifdef NEED_OPCODE_METADATA -const uint8_t _PyOpcode_NeedsGuardIp[256] = { - [INTERPRETER_EXIT] = 1, - [RETURN_VALUE] = 1, - [YIELD_VALUE] = 1, - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = 1, - [INSTRUMENTED_FOR_ITER] = 1, - [RETURN_GENERATOR] = 1, - [BINARY_OP_SUBSCR_GETITEM] = 1, - [INSTRUMENTED_RETURN_VALUE] = 1, - [SEND] = 1, - [SEND_GEN] = 1, - [INSTRUMENTED_YIELD_VALUE] = 1, - [INSTRUMENTED_END_ASYNC_FOR] = 1, - [END_ASYNC_FOR] = 1, - [LOAD_ATTR_PROPERTY] = 1, - [FOR_ITER_GEN] = 1, - [CALL] = 1, - [INSTRUMENTED_CALL] = 1, - [CALL_PY_GENERAL] = 1, - [CALL_BOUND_METHOD_GENERAL] = 1, - [CALL_BOUND_METHOD_EXACT_ARGS] = 1, - [CALL_PY_EXACT_ARGS] = 1, - [CALL_ALLOC_AND_ENTER_INIT] = 1, - [CALL_KW_PY] = 1, - [CALL_KW_BOUND_METHOD] = 1, - [CALL_KW] = 1, - [INSTRUMENTED_CALL_KW] = 1, - [CALL_FUNCTION_EX] = 1, - [INSTRUMENTED_CALL_FUNCTION_EX] = 1, -}; -#endif - extern const uint8_t _PyOpcode_Deopt[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Deopt[256] = { diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 4b033a8b6c2d41..d5a3c362d875e6 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -11,7 +11,7 @@ extern "C" { #include #include "pycore_uop_ids.h" -extern const uint16_t _PyUop_Flags[MAX_UOP_ID+1]; +extern const uint32_t _PyUop_Flags[MAX_UOP_ID+1]; typedef struct _rep_range { uint8_t start; uint8_t stop; } ReplicationRange; extern const ReplicationRange _PyUop_Replication[MAX_UOP_ID+1]; extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1]; @@ -19,7 +19,7 @@ extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1]; extern int _PyUop_num_popped(int opcode, int oparg); #ifdef NEED_OPCODE_METADATA -const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { +const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_NOP] = HAS_PURE_FLAG, [_CHECK_PERIODIC] = HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CHECK_PERIODIC_IF_NOT_YIELD_FROM] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -128,12 +128,12 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_DELETE_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_INTRINSIC_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_INTRINSIC_2] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_RETURN_VALUE] = HAS_ESCAPES_FLAG, + [_RETURN_VALUE] = HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG, [_GET_AITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GET_ANEXT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GET_AWAITABLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SEND_GEN_FRAME] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, - [_YIELD_VALUE] = HAS_ARG_FLAG, + [_YIELD_VALUE] = HAS_ARG_FLAG | HAS_NEEDS_GUARD_IP_FLAG, [_POP_EXCEPT] = HAS_ESCAPES_FLAG, [_LOAD_COMMON_CONSTANT] = HAS_ARG_FLAG, [_LOAD_BUILD_CLASS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -256,7 +256,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_INIT_CALL_PY_EXACT_ARGS_3] = HAS_PURE_FLAG, [_INIT_CALL_PY_EXACT_ARGS_4] = HAS_PURE_FLAG, [_INIT_CALL_PY_EXACT_ARGS] = HAS_ARG_FLAG | HAS_PURE_FLAG, - [_PUSH_FRAME] = 0, + [_PUSH_FRAME] = HAS_NEEDS_GUARD_IP_FLAG, [_GUARD_NOS_NULL] = HAS_DEOPT_FLAG, [_GUARD_NOS_NOT_NULL] = HAS_EXIT_FLAG, [_GUARD_THIRD_NULL] = HAS_DEOPT_FLAG, @@ -293,7 +293,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_MAKE_CALLARGS_A_TUPLE] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_MAKE_FUNCTION] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SET_FUNCTION_ATTRIBUTE] = HAS_ARG_FLAG, - [_RETURN_GENERATOR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_RETURN_GENERATOR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG, [_BUILD_SLICE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CONVERT_VALUE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_FORMAT_SIMPLE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, diff --git a/Python/optimizer.c b/Python/optimizer.c index 19d86115154e7c..dbceb5aa5b6987 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -619,7 +619,7 @@ _PyJit_translate_single_bytecode_to_trace( assert(!OPCODE_HAS_DEOPT(opcode)); } - bool needs_guard_ip = _PyOpcode_NeedsGuardIp[opcode]; + bool needs_guard_ip = OPCODE_HAS_NEEDS_GUARD_IP(opcode); if (has_dynamic_jump_taken && !needs_guard_ip) { DPRINTF(2, "Unsupported: dynamic jump taken\n"); goto unsupported; diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 560996ca1b4a58..67eba7dec09cce 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -757,6 +757,8 @@ def cflags(p: Properties) -> str: flags.append("HAS_NO_SAVE_IP_FLAG") if p.unpredictable_jump: flags.append("HAS_UNPREDICTABLE_JUMP_FLAG") + if p.needs_guard_ip: + flags.append("HAS_NEEDS_GUARD_IP_FLAG") if flags: return " | ".join(flags) else: diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py index 78f19bd7aece24..21ae785a0ec445 100644 --- a/Tools/cases_generator/opcode_metadata_generator.py +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -57,6 +57,7 @@ "NO_SAVE_IP", "PERIODIC", "UNPREDICTABLE_JUMP", + "NEEDS_GUARD_IP", ] @@ -185,16 +186,6 @@ def generate_cache_table(analysis: Analysis, out: CWriter) -> None: out.emit("#endif\n\n") -def generate_needs_guard_ip_table(analysis: Analysis, out: CWriter) -> None: - out.emit("extern const uint8_t _PyOpcode_NeedsGuardIp[256];\n") - out.emit("#ifdef NEED_OPCODE_METADATA\n") - out.emit("const uint8_t _PyOpcode_NeedsGuardIp[256] = {\n") - for inst in analysis.instructions.values(): - if inst.properties.needs_guard_ip: - out.emit(f"[{inst.name}] = 1,\n") - out.emit("};\n") - out.emit("#endif\n\n") - def generate_name_table(analysis: Analysis, out: CWriter) -> None: table_size = 256 + len(analysis.pseudos) out.emit(f"extern const char *_PyOpcode_OpName[{table_size}];\n") @@ -212,7 +203,7 @@ def generate_metadata_table(analysis: Analysis, out: CWriter) -> None: out.emit("struct opcode_metadata {\n") out.emit("uint8_t valid_entry;\n") out.emit("uint8_t instr_format;\n") - out.emit("uint16_t flags;\n") + out.emit("uint32_t flags;\n") out.emit("};\n\n") out.emit( f"extern const struct opcode_metadata _PyOpcode_opcode_metadata[{table_size}];\n" @@ -393,7 +384,6 @@ def generate_opcode_metadata( generate_expansion_table(analysis, out) generate_name_table(analysis, out) generate_cache_table(analysis, out) - generate_needs_guard_ip_table(analysis, out) generate_deopt_table(analysis, out) generate_extra_cases(analysis, out) generate_pseudo_targets(analysis, out) diff --git a/Tools/cases_generator/uop_metadata_generator.py b/Tools/cases_generator/uop_metadata_generator.py index 1cc23837a72dea..0e0396e5143348 100644 --- a/Tools/cases_generator/uop_metadata_generator.py +++ b/Tools/cases_generator/uop_metadata_generator.py @@ -23,13 +23,13 @@ def generate_names_and_flags(analysis: Analysis, out: CWriter) -> None: - out.emit("extern const uint16_t _PyUop_Flags[MAX_UOP_ID+1];\n") + out.emit("extern const uint32_t _PyUop_Flags[MAX_UOP_ID+1];\n") out.emit("typedef struct _rep_range { uint8_t start; uint8_t stop; } ReplicationRange;\n") out.emit("extern const ReplicationRange _PyUop_Replication[MAX_UOP_ID+1];\n") out.emit("extern const char * const _PyOpcode_uop_name[MAX_UOP_ID+1];\n\n") out.emit("extern int _PyUop_num_popped(int opcode, int oparg);\n\n") out.emit("#ifdef NEED_OPCODE_METADATA\n") - out.emit("const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = {\n") + out.emit("const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = {\n") for uop in analysis.uops.values(): if uop.is_viable() and uop.properties.tier != 1: out.emit(f"[{uop.name}] = {cflags(uop.properties)},\n") From 46c079a710e2cd7c23b6a3631535a671fe3acf3a Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 7 Nov 2025 20:34:49 +0000 Subject: [PATCH 166/190] Cleanup cases generator --- Include/internal/pycore_opcode_metadata.h | 2 +- Python/bytecodes.c | 16 +++---- Python/executor_cases.c.h | 1 - Tools/cases_generator/analyzer.py | 5 +- Tools/cases_generator/tier2_generator.py | 57 +++++++++++++---------- 5 files changed, 45 insertions(+), 36 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 18f8baf980e656..825321739c40c5 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1189,7 +1189,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, - [INTERPRETER_EXIT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INTERPRETER_EXIT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [IS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_BACKWARD_JIT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index dfc7e2952420cd..ac308c918d0ffa 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5485,33 +5485,33 @@ dummy_func( } tier2 op(_GUARD_IP__PUSH_FRAME, (ip/4 --)) { - _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_CORRESPONDING_UOP; + _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF(_PUSH_FRAME); if (target != (_Py_CODEUNIT *)ip) { - frame->instr_ptr += OFFSET_OF_CORRESPONDING_UOP; + frame->instr_ptr += OFFSET_OF(_PUSH_FRAME); EXIT_IF(true); } } tier2 op(_GUARD_IP_YIELD_VALUE, (ip/4 --)) { - _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_CORRESPONDING_UOP; + _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF(YIELD_VALUE); if (target != (_Py_CODEUNIT *)ip) { - frame->instr_ptr += OFFSET_OF_CORRESPONDING_UOP; + frame->instr_ptr += OFFSET_OF(YIELD_VALUE); EXIT_IF(true); } } tier2 op(_GUARD_IP_RETURN_VALUE, (ip/4 --)) { - _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_CORRESPONDING_UOP; + _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF(RETURN_VALUE); if (target != (_Py_CODEUNIT *)ip) { - frame->instr_ptr += OFFSET_OF_CORRESPONDING_UOP; + frame->instr_ptr += OFFSET_OF(RETURN_VALUE); EXIT_IF(true); } } tier2 op(_GUARD_IP_RETURN_GENERATOR, (ip/4 --)) { - _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF_CORRESPONDING_UOP; + _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF(RETURN_GENERATOR); if (target != (_Py_CODEUNIT *)ip) { - frame->instr_ptr += OFFSET_OF_CORRESPONDING_UOP; + frame->instr_ptr += OFFSET_OF(RETURN_GENERATOR); EXIT_IF(true); } } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 16db3e2efcfc90..84ce8f862a9d2c 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -8,7 +8,6 @@ #endif #define TIER_TWO 2 - case _NOP: { break; } diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 5bba28578bfe92..532d5df14f4136 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -984,7 +984,10 @@ def compute_properties(op: parser.CodeDef) -> Properties: no_save_ip=no_save_ip, tier=tier_variable(op), needs_prev=variable_used(op, "prev_instr"), - needs_guard_ip=(isinstance(op, parser.InstDef) and (unpredictable_jump and "replaced" not in op.annotations)) or variable_used(op, "LLTRACE_RESUME_FRAME") or variable_used(op, "DISPATCH_INLINED"), + needs_guard_ip=(isinstance(op, parser.InstDef) + and (unpredictable_jump and "replaced" not in op.annotations)) + or variable_used(op, "LOAD_IP") + or variable_used(op, "DISPATCH_INLINED"), unpredictable_jump=unpredictable_jump, ) diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index bff1a4370b48be..b6eab511232e5c 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -63,7 +63,7 @@ class Tier2Emitter(Emitter): def __init__(self, out: CWriter, labels: dict[str, Label]): super().__init__(out, labels) self._replacers["oparg"] = self.oparg - self._replacers["OFFSET_OF_CORRESPONDING_UOP"] = self.offset_of_corresponding_uop + self._replacers["OFFSET_OF"] = self.offset_of def goto_error(self, offset: int, storage: Storage) -> str: # To do: Add jump targets for popping values. @@ -135,7 +135,7 @@ def oparg( self.out.emit_at(uop.name[-1], tkn) return True - def offset_of_corresponding_uop( + def offset_of( self, tkn: Token, tkn_iter: TokenIterator, @@ -144,14 +144,21 @@ def offset_of_corresponding_uop( inst: Instruction | None, ) -> bool: assert uop.name.startswith("_GUARD_IP") - rest = uop.name[len("_GUARD_IP"):] - self.emit(f" OFFSET_OF{rest};\n") + # LPAREN + next(tkn_iter) + inst = next(tkn_iter) + self.emit(f" OFFSET_OF_{inst.text};\n") + # RPAREN + next(tkn_iter) + # SEMI next(tkn_iter) return True -def write_uop(uop: Uop, emitter: Emitter, stack: Stack) -> Stack: +def write_uop(uop: Uop, emitter: Emitter, stack: Stack, offset_strs: dict[str, tuple[str, str]]) -> Stack: locals: dict[str, Local] = {} try: + if name_offset_pair := offset_strs.get(uop.name): + emitter.emit(f"#define OFFSET_OF_{name_offset_pair[0]} ({name_offset_pair[1]})\n") emitter.out.start_line() if uop.properties.oparg: emitter.emit("oparg = CURRENT_OPARG();\n") @@ -172,6 +179,8 @@ def write_uop(uop: Uop, emitter: Emitter, stack: Stack) -> Stack: idx += 1 _, storage = emitter.emit_tokens(uop, storage, None, False) storage.flush(emitter.out) + if name_offset_pair: + emitter.emit(f"#undef OFFSET_OF_{name_offset_pair[0]}\n") except StackError as ex: raise analysis_error(ex.args[0], uop.body.open) from None return storage.stack @@ -179,21 +188,7 @@ def write_uop(uop: Uop, emitter: Emitter, stack: Stack) -> Stack: SKIPS = ("_EXTENDED_ARG",) -def generate_tier2( - filenames: list[str], analysis: Analysis, outfile: TextIO, lines: bool -) -> None: - write_header(__file__, filenames, outfile) - outfile.write( - """ -#ifdef TIER_ONE - #error "This file is for Tier 2 only" -#endif -#define TIER_TWO 2 -""" - ) - out = CWriter(outfile, 2, lines) - emitter = Tier2Emitter(out, analysis.labels) - out.emit("\n") +def populate_offset_strs(analysis: Analysis) -> dict[str, tuple[str, str]]: offset_strs: dict[str, tuple[str, str]] = {} for name, uop in analysis.uops.items(): if not f"_GUARD_IP_{name}" in analysis.uops: @@ -214,7 +209,23 @@ def generate_tier2( found = True assert offset_str offset_strs[f"_GUARD_IP_{name}"] = (name, offset_str) + return offset_strs +def generate_tier2( + filenames: list[str], analysis: Analysis, outfile: TextIO, lines: bool +) -> None: + write_header(__file__, filenames, outfile) + outfile.write( + """ +#ifdef TIER_ONE + #error "This file is for Tier 2 only" +#endif +#define TIER_TWO 2 +""" + ) + out = CWriter(outfile, 2, lines) + emitter = Tier2Emitter(out, analysis.labels) + offset_strs = populate_offset_strs(analysis) out.emit("\n") for name, uop in analysis.uops.items(): @@ -231,11 +242,7 @@ def generate_tier2( out.emit(f"case {uop.name}: {{\n") declare_variables(uop, out) stack = Stack() - if name_offset_pair := offset_strs.get(name): - out.emit(f"#define OFFSET_OF_{name_offset_pair[0]} ({name_offset_pair[1]})\n") - stack = write_uop(uop, emitter, stack) - if name_offset_pair: - out.emit(f"#undef OFFSET_OF_{name_offset_pair[0]}\n") + stack = write_uop(uop, emitter, stack, offset_strs) out.start_line() if not uop.properties.always_exits: out.emit("break;\n") From 4aeb4abfcaa200c20a173e22d6875f8752db3d6a Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 7 Nov 2025 20:47:52 +0000 Subject: [PATCH 167/190] massive refactoring of the struct --- Include/internal/pycore_interp_structs.h | 18 ++++---- Include/internal/pycore_optimizer.h | 2 +- Lib/test/test_ast/test_ast.py | 4 +- Python/ceval.c | 8 ++-- Python/optimizer.c | 57 +++++++++++++----------- Python/optimizer_analysis.c | 4 +- Python/pystate.c | 4 -- 7 files changed, 49 insertions(+), 48 deletions(-) diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index b0f5d0abc3fcae..cac5e358a673f0 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -758,24 +758,26 @@ struct _Py_unique_id_pool { typedef _Py_CODEUNIT *(*_PyJitEntryFuncPtr)(struct _PyExecutorObject *exec, _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate); typedef struct _PyJitTracerState { + struct { + int stack_depth; + int chain_depth; + struct _PyExitData *exit; + PyCodeObject *code; // Strong + PyFunctionObject *func; // Strong + _Py_CODEUNIT *start_instr; + _Py_CODEUNIT *close_loop_instr; + _Py_CODEUNIT *jump_backward_instr; + } initial_state; bool dependencies_still_valid; bool prev_instr_is_super; int code_max_size; int code_curr_size; - int initial_stack_depth; - int initial_chain_depth; int prev_instr_oparg; int prev_instr_stacklevel; int specialize_counter; _PyUOpInstruction *code_buffer; - _Py_CODEUNIT *start_instr; - _Py_CODEUNIT *close_loop_instr; - _Py_CODEUNIT *jump_backward_instr; - PyCodeObject *initial_code; // Strong - PyFunctionObject *initial_func; // Strong _Py_CODEUNIT *prev_instr; PyCodeObject *prev_instr_code; // Strong - struct _PyExitData *prev_exit; _PyInterpreterFrame *prev_instr_frame; _PyBloomFilter dependencies; } _PyJitTracerState; diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 28c6c3ecaea122..eea5608621e9e7 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -91,7 +91,7 @@ PyAPI_FUNC(void) _Py_Executors_InvalidateCold(PyInterpreterState *interp); #define TRACE_STACK_SIZE 5 int _Py_uop_analyze_and_optimize( - PyFunctionObject *initial_func, + PyFunctionObject *func, _PyUOpInstruction *trace, int trace_len, int curr_stackentries, _PyBloomFilter *dependencies); diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index 551de5851daace..78ee7971849914 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -3047,8 +3047,8 @@ def test_source_segment_missing_info(self): class NodeTransformerTests(ASTTestMixin, unittest.TestCase): def assertASTTransformation(self, transformer_class, - initial_code, expected_code): - initial_ast = ast.parse(dedent(initial_code)) + code, expected_code): + initial_ast = ast.parse(dedent(code)) expected_ast = ast.parse(dedent(expected_code)) transformer = transformer_class() diff --git a/Python/ceval.c b/Python/ceval.c index 00c30cc3fd4bcf..1b4ce9233c1925 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1001,15 +1001,15 @@ bail_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame) err = _PyOptimizer_Optimize(frame, tstate); } // Deal with backoffs - _PyExitData *exit = tstate->interp->jit_state.prev_exit; + _PyExitData *exit = tstate->interp->jit_state.initial_state.exit; if (exit == NULL) { // We hold a strong reference to the code object, so the instruction won't be freed. if (err <= 0) { - _Py_BackoffCounter counter = tstate->interp->jit_state.jump_backward_instr[1].counter; - tstate->interp->jit_state.jump_backward_instr[1].counter = restart_backoff_counter(counter); + _Py_BackoffCounter counter = tstate->interp->jit_state.initial_state.jump_backward_instr[1].counter; + tstate->interp->jit_state.initial_state.jump_backward_instr[1].counter = restart_backoff_counter(counter); } else { - tstate->interp->jit_state.jump_backward_instr[1].counter = initial_jump_backoff_counter(); + tstate->interp->jit_state.initial_state.jump_backward_instr[1].counter = initial_jump_backoff_counter(); } } else { diff --git a/Python/optimizer.c b/Python/optimizer.c index dbceb5aa5b6987..90bf7069d8fe53 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -119,13 +119,13 @@ _PyOptimizer_Optimize( _PyInterpreterFrame *frame, PyThreadState *tstate) { PyInterpreterState *interp = _PyInterpreterState_GET(); - int chain_depth = tstate->interp->jit_state.initial_chain_depth; + int chain_depth = tstate->interp->jit_state.initial_state.chain_depth; assert(interp->jit); assert(!interp->compiling); - assert(tstate->interp->jit_state.initial_stack_depth >= 0); + assert(tstate->interp->jit_state.initial_state.stack_depth >= 0); #ifndef Py_GIL_DISABLED // Trace got stomped on by another thread. - if (tstate->interp->jit_state.initial_func == NULL) { + if (tstate->interp->jit_state.initial_state.func == NULL) { return 0; } interp->compiling = true; @@ -135,8 +135,8 @@ _PyOptimizer_Optimize( // this is true, since a deopt won't infinitely re-enter the executor: chain_depth %= MAX_CHAIN_DEPTH; bool progress_needed = chain_depth == 0; - PyCodeObject *code = (PyCodeObject *)tstate->interp->jit_state.initial_code; - _Py_CODEUNIT *start = tstate->interp->jit_state.start_instr; + PyCodeObject *code = (PyCodeObject *)tstate->interp->jit_state.initial_state.code; + _Py_CODEUNIT *start = tstate->interp->jit_state.initial_state.start_instr; if (progress_needed && !has_space_for_executor(code, start)) { interp->compiling = false; return 0; @@ -171,9 +171,9 @@ _PyOptimizer_Optimize( else { executor->vm_data.code = NULL; } - _PyExitData *prev_exit = tstate->interp->jit_state.prev_exit; - if (prev_exit != NULL) { - prev_exit->executor = executor; + _PyExitData *exit = tstate->interp->jit_state.initial_state.exit; + if (exit != NULL) { + exit->executor = executor; } executor->vm_data.chain_depth = chain_depth; assert(executor->vm_data.valid); @@ -569,7 +569,7 @@ _PyJit_translate_single_bytecode_to_trace( if (old_code == NULL) { return 0; } - bool progress_needed = (tstate->interp->jit_state.initial_chain_depth % MAX_CHAIN_DEPTH) == 0;; + bool progress_needed = (tstate->interp->jit_state.initial_state.chain_depth % MAX_CHAIN_DEPTH) == 0; _PyBloomFilter *dependencies = &tstate->interp->jit_state.dependencies; _Py_BloomFilter_Add(dependencies, old_code); int trace_length = tstate->interp->jit_state.code_curr_size; @@ -748,8 +748,8 @@ _PyJit_translate_single_bytecode_to_trace( _Py_FALLTHROUGH; case JUMP_BACKWARD_NO_INTERRUPT: { - if ((next_instr != tstate->interp->jit_state.close_loop_instr) && - (next_instr != tstate->interp->jit_state.start_instr) && + if ((next_instr != tstate->interp->jit_state.initial_state.close_loop_instr) && + (next_instr != tstate->interp->jit_state.initial_state.start_instr) && tstate->interp->jit_state.code_curr_size > 5 && // These are coroutines, and we want to unroll those usually. opcode != JUMP_BACKWARD_NO_INTERRUPT) { @@ -760,7 +760,8 @@ _PyJit_translate_single_bytecode_to_trace( OPT_STAT_INC(inner_loop); ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); trace[trace_length-1].operand1 = true; // is_control_flow - DPRINTF(2, "JUMP_BACKWARD not to top ends trace %p %p %p\n", next_instr, tstate->interp->jit_state.close_loop_instr, tstate->interp->jit_state.start_instr); + DPRINTF(2, "JUMP_BACKWARD not to top ends trace %p %p %p\n", next_instr, + tstate->interp->jit_state.initial_state.close_loop_instr, tstate->interp->jit_state.initial_state.start_instr); goto done; } break; @@ -915,7 +916,8 @@ _PyJit_translate_single_bytecode_to_trace( } } // Loop back to the start - int is_first_instr = tstate->interp->jit_state.close_loop_instr == next_instr || tstate->interp->jit_state.start_instr == next_instr; + int is_first_instr = tstate->interp->jit_state.initial_state.close_loop_instr == next_instr || + tstate->interp->jit_state.initial_state.start_instr == next_instr; if (is_first_instr && tstate->interp->jit_state.code_curr_size > 5) { if (needs_guard_ip) { ADD_TO_TRACE(_SET_IP, 0, (uintptr_t)next_instr, 0); @@ -985,13 +987,13 @@ _PyJit_TryInitializeTracing( tstate->interp->jit_state.code_curr_size = 2; tstate->interp->jit_state.code_max_size = UOP_MAX_TRACE_LENGTH; - tstate->interp->jit_state.start_instr = start_instr; - tstate->interp->jit_state.close_loop_instr = close_loop_instr; - tstate->interp->jit_state.initial_code = (PyCodeObject *)Py_NewRef(code); - tstate->interp->jit_state.initial_func = (PyFunctionObject *)Py_XNewRef(PyStackRef_AsPyObjectBorrow(frame->f_funcobj)); - tstate->interp->jit_state.prev_exit = exit; - tstate->interp->jit_state.initial_stack_depth = curr_stackdepth; - tstate->interp->jit_state.initial_chain_depth = chain_depth; + tstate->interp->jit_state.initial_state.start_instr = start_instr; + tstate->interp->jit_state.initial_state.close_loop_instr = close_loop_instr; + tstate->interp->jit_state.initial_state.code = (PyCodeObject *)Py_NewRef(code); + tstate->interp->jit_state.initial_state.func = (PyFunctionObject *)Py_XNewRef(PyStackRef_AsPyObjectBorrow(frame->f_funcobj)); + tstate->interp->jit_state.initial_state.exit = exit; + tstate->interp->jit_state.initial_state.stack_depth = curr_stackdepth; + tstate->interp->jit_state.initial_state.chain_depth = chain_depth; tstate->interp->jit_state.prev_instr_frame = frame; tstate->interp->jit_state.dependencies_still_valid = true; tstate->interp->jit_state.specialize_counter = 0; @@ -1002,7 +1004,7 @@ _PyJit_TryInitializeTracing( tstate->interp->jit_state.prev_instr_stacklevel = curr_stackdepth; tstate->interp->jit_state.prev_instr_is_super = false; assert(curr_instr->op.code == JUMP_BACKWARD_JIT || (exit != NULL)); - tstate->interp->jit_state.jump_backward_instr = curr_instr; + tstate->interp->jit_state.initial_state.jump_backward_instr = curr_instr; assert(curr_instr->op.code == JUMP_BACKWARD_JIT || (exit != NULL)); _Py_BloomFilter_Init(&tstate->interp->jit_state.dependencies); return 1; @@ -1011,8 +1013,8 @@ _PyJit_TryInitializeTracing( void _PyJit_FinalizeTracing(PyThreadState *tstate) { - Py_CLEAR(tstate->interp->jit_state.initial_code); - Py_CLEAR(tstate->interp->jit_state.initial_func); + Py_CLEAR(tstate->interp->jit_state.initial_state.code); + Py_CLEAR(tstate->interp->jit_state.initial_state.func); Py_CLEAR(tstate->interp->jit_state.prev_instr_code); tstate->interp->jit_state.code_curr_size = 2; tstate->interp->jit_state.code_max_size = UOP_MAX_TRACE_LENGTH - 1; @@ -1335,7 +1337,7 @@ uop_optimize( // It is the optimizer's responsibility to add the dependencies it requires on its own. _PyBloomFilter new_dependencies; _Py_BloomFilter_Init(&new_dependencies); - _Py_BloomFilter_Add(&new_dependencies, tstate->interp->jit_state.initial_code); + _Py_BloomFilter_Add(&new_dependencies, tstate->interp->jit_state.initial_state.code); PyInterpreterState *interp = _PyInterpreterState_GET(); _PyUOpInstruction *buffer = interp->jit_state.code_buffer; OPT_STAT_INC(attempts); @@ -1344,7 +1346,7 @@ uop_optimize( if (env_var == NULL || *env_var == '\0' || *env_var > '0') { is_noopt = false; } - int curr_stackentries = tstate->interp->jit_state.initial_stack_depth; + int curr_stackentries = tstate->interp->jit_state.initial_state.stack_depth; int length = interp->jit_state.code_curr_size; // Trace too short, don't bother. if (length <= 5) { @@ -1354,7 +1356,7 @@ uop_optimize( assert(length < UOP_MAX_TRACE_LENGTH); OPT_STAT_INC(traces_created); if (!is_noopt) { - length = _Py_uop_analyze_and_optimize(tstate->interp->jit_state.initial_func, buffer, + length = _Py_uop_analyze_and_optimize(tstate->interp->jit_state.initial_state.func, buffer, length, curr_stackentries, &new_dependencies); if (length <= 0) { @@ -1379,7 +1381,8 @@ uop_optimize( OPT_HIST(effective_trace_length(buffer, length), optimized_trace_length_hist); length = prepare_for_execution(buffer, length); assert(length <= UOP_MAX_TRACE_LENGTH); - _PyExecutorObject *executor = make_executor_from_uops(buffer, length, &new_dependencies, tstate->interp->jit_state.initial_chain_depth); + _PyExecutorObject *executor = make_executor_from_uops( + buffer, length, &new_dependencies, tstate->interp->jit_state.initial_state.chain_depth); if (executor == NULL) { return -1; } diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index ad5f1b0388d5a7..a24906cfdce570 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -511,7 +511,7 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) // > 0 - length of optimized trace int _Py_uop_analyze_and_optimize( - PyFunctionObject *initial_func, + PyFunctionObject *func, _PyUOpInstruction *buffer, int length, int curr_stacklen, @@ -521,7 +521,7 @@ _Py_uop_analyze_and_optimize( OPT_STAT_INC(optimizer_attempts); length = optimize_uops( - initial_func, buffer, + func, buffer, length, curr_stacklen, dependencies); if (length == 0) { diff --git a/Python/pystate.c b/Python/pystate.c index 0676a86965cf29..36d62ecae2ee0b 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -547,10 +547,6 @@ init_interpreter(PyInterpreterState *interp, #ifdef _Py_TIER2 interp->jit_state.code_buffer = NULL; - interp->jit_state.initial_stack_depth = -1; - interp->jit_state.initial_chain_depth = -1; - interp->jit_state.initial_code = NULL; - interp->jit_state.initial_func = NULL; #endif llist_init(&interp->mem_free_queue.head); llist_init(&interp->asyncio_tasks_head); From fe3a6a1743711c17bc2370370f5bfe14724f0f61 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 7 Nov 2025 20:58:37 +0000 Subject: [PATCH 168/190] massive refactoring 2 --- Include/internal/pycore_interp_structs.h | 24 +++--- Python/bytecodes.c | 20 ++--- Python/ceval.c | 2 +- Python/ceval_macros.h | 4 +- Python/generated_cases.c.h | 50 ++++++------- Python/optimizer.c | 86 +++++++++++----------- Tools/cases_generator/generators_common.py | 2 +- 7 files changed, 95 insertions(+), 93 deletions(-) diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index cac5e358a673f0..704a0842499365 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -768,18 +768,20 @@ typedef struct _PyJitTracerState { _Py_CODEUNIT *close_loop_instr; _Py_CODEUNIT *jump_backward_instr; } initial_state; - bool dependencies_still_valid; - bool prev_instr_is_super; - int code_max_size; - int code_curr_size; - int prev_instr_oparg; - int prev_instr_stacklevel; - int specialize_counter; _PyUOpInstruction *code_buffer; - _Py_CODEUNIT *prev_instr; - PyCodeObject *prev_instr_code; // Strong - _PyInterpreterFrame *prev_instr_frame; - _PyBloomFilter dependencies; + struct { + bool dependencies_still_valid; + bool instr_is_super; + int code_max_size; + int code_curr_size; + int instr_oparg; + int instr_stacklevel; + int specialize_counter; + _Py_CODEUNIT *instr; + PyCodeObject *instr_code; // Strong + _PyInterpreterFrame *instr_frame; + _PyBloomFilter dependencies; + } prev_state;; } _PyJitTracerState; /* PyInterpreterState holds the global state for one of the runtime's diff --git a/Python/bytecodes.c b/Python/bytecodes.c index ac308c918d0ffa..42fb4170eae725 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5673,24 +5673,24 @@ dummy_func( } // Super instructions. Instruction deopted. There's a mismatch in what the stack expects // in the optimizer. So we have to reflect in the trace correctly. - if ((tstate->interp->jit_state.prev_instr->op.code == CALL_LIST_APPEND && + if ((tstate->interp->jit_state.prev_state.instr->op.code == CALL_LIST_APPEND && opcode == POP_TOP) || - (tstate->interp->jit_state.prev_instr->op.code == BINARY_OP_INPLACE_ADD_UNICODE && + (tstate->interp->jit_state.prev_state.instr->op.code == BINARY_OP_INPLACE_ADD_UNICODE && opcode == STORE_FAST)) { - tstate->interp->jit_state.prev_instr_is_super = true; + tstate->interp->jit_state.prev_state.instr_is_super = true; } else { - tstate->interp->jit_state.prev_instr = next_instr; + tstate->interp->jit_state.prev_state.instr = next_instr; } - tstate->interp->jit_state.specialize_counter = 0; + tstate->interp->jit_state.prev_state.specialize_counter = 0; PyCodeObject *prev_code = (PyCodeObject *)Py_NewRef(PyStackRef_AsPyObjectBorrow(frame->f_executable)); - if (tstate->interp->jit_state.prev_instr_code != prev_code) { - Py_SETREF(tstate->interp->jit_state.prev_instr_code, prev_code); + if (tstate->interp->jit_state.prev_state.instr_code != prev_code) { + Py_SETREF(tstate->interp->jit_state.prev_state.instr_code, prev_code); } - tstate->interp->jit_state.prev_instr_frame = frame; - tstate->interp->jit_state.prev_instr_oparg = oparg; - tstate->interp->jit_state.prev_instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); + tstate->interp->jit_state.prev_state.instr_frame = frame; + tstate->interp->jit_state.prev_state.instr_oparg = oparg; + tstate->interp->jit_state.prev_state.instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); DISPATCH_GOTO_NON_TRACING(); #else Py_FatalError("JIT label executed in non-jit build."); diff --git a/Python/ceval.c b/Python/ceval.c index 1b4ce9233c1925..ce6493fdd85284 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1017,7 +1017,7 @@ bail_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame) // to be valid to access. if (err <= 0) { // Some opcodes will forever be unchanged. Don't ever bother specializing for them ever again. - if (tstate->interp->jit_state.prev_instr->op.code == INTERPRETER_EXIT) { + if (tstate->interp->jit_state.prev_state.instr->op.code == INTERPRETER_EXIT) { exit->temperature = initial_unreachable_backoff_counter(); } else { diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index a8290242a5c1ef..219874ac240bd6 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -134,7 +134,7 @@ #if (_Py_TAIL_CALL_INTERP || USE_COMPUTED_GOTOS) && _Py_TIER2 # define IS_JIT_TRACING() (DISPATCH_TABLE_VAR == TRACING_DISPATCH_TABLE) -# define IS_JIT_TRACING_MAKING_PROGRESS() (IS_JIT_TRACING() && tstate->interp->jit_state.specialize_counter < MAX_SPECIALIZATION_TRIES) +# define IS_JIT_TRACING_MAKING_PROGRESS() (IS_JIT_TRACING() && tstate->interp->jit_state.prev_state.specialize_counter < MAX_SPECIALIZATION_TRIES) # define ENTER_TRACING() \ DISPATCH_TABLE_VAR = TRACING_DISPATCH_TABLE; # define LEAVE_TRACING() \ @@ -402,7 +402,7 @@ do { \ JUMP_TO_LABEL(error); \ } \ if (keep_tracing_bit) { \ - assert(tstate->interp->jit_state.code_curr_size == 2 || tstate->interp->jit_state.code_curr_size == 3); \ + assert(tstate->interp->jit_state.prev_state.code_curr_size == 2); \ ENTER_TRACING(); \ DISPATCH_NON_TRACING(); \ } \ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 33ac4152bd5e22..24c4271c88fab2 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -45,7 +45,7 @@ _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); stack_pointer = _PyFrame_GetStackPointer(frame); #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; + tstate->interp->jit_state.prev_state.specialize_counter++; #endif DISPATCH_SAME_OPARG(); } @@ -1539,7 +1539,7 @@ _Py_Specialize_Call(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); stack_pointer = _PyFrame_GetStackPointer(frame); #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; + tstate->interp->jit_state.prev_state.specialize_counter++; #endif DISPATCH_SAME_OPARG(); } @@ -2832,7 +2832,7 @@ _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); stack_pointer = _PyFrame_GetStackPointer(frame); #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; + tstate->interp->jit_state.prev_state.specialize_counter++; #endif DISPATCH_SAME_OPARG(); } @@ -4678,7 +4678,7 @@ _Py_Specialize_CompareOp(left, right, next_instr, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; + tstate->interp->jit_state.prev_state.specialize_counter++; #endif DISPATCH_SAME_OPARG(); } @@ -4926,7 +4926,7 @@ _Py_Specialize_ContainsOp(right, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; + tstate->interp->jit_state.prev_state.specialize_counter++; #endif DISPATCH_SAME_OPARG(); } @@ -5665,7 +5665,7 @@ _Py_Specialize_ForIter(iter, null_or_index, next_instr, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; + tstate->interp->jit_state.prev_state.specialize_counter++; #endif DISPATCH_SAME_OPARG(); } @@ -7647,7 +7647,7 @@ FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, desired); next_instr = this_instr; #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; + tstate->interp->jit_state.prev_state.specialize_counter++; #endif DISPATCH_SAME_OPARG(); } @@ -7880,7 +7880,7 @@ _Py_Specialize_LoadAttr(owner, next_instr, name); stack_pointer = _PyFrame_GetStackPointer(frame); #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; + tstate->interp->jit_state.prev_state.specialize_counter++; #endif DISPATCH_SAME_OPARG(); } @@ -9188,7 +9188,7 @@ _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); stack_pointer = _PyFrame_GetStackPointer(frame); #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; + tstate->interp->jit_state.prev_state.specialize_counter++; #endif DISPATCH_SAME_OPARG(); } @@ -9511,7 +9511,7 @@ _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); stack_pointer = _PyFrame_GetStackPointer(frame); #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; + tstate->interp->jit_state.prev_state.specialize_counter++; #endif DISPATCH_SAME_OPARG(); } @@ -10499,7 +10499,7 @@ _Py_Specialize_Send(receiver, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; + tstate->interp->jit_state.prev_state.specialize_counter++; #endif DISPATCH_SAME_OPARG(); } @@ -10801,7 +10801,7 @@ _Py_Specialize_StoreAttr(owner, next_instr, name); stack_pointer = _PyFrame_GetStackPointer(frame); #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; + tstate->interp->jit_state.prev_state.specialize_counter++; #endif DISPATCH_SAME_OPARG(); } @@ -11301,7 +11301,7 @@ _Py_Specialize_StoreSubscr(container, sub, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; + tstate->interp->jit_state.prev_state.specialize_counter++; #endif DISPATCH_SAME_OPARG(); } @@ -11513,7 +11513,7 @@ _Py_Specialize_ToBool(value, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; + tstate->interp->jit_state.prev_state.specialize_counter++; #endif DISPATCH_SAME_OPARG(); } @@ -11896,7 +11896,7 @@ _Py_Specialize_UnpackSequence(seq, next_instr, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); #if _Py_TIER2 - tstate->interp->jit_state.specialize_counter++; + tstate->interp->jit_state.prev_state.specialize_counter++; #endif DISPATCH_SAME_OPARG(); } @@ -12360,25 +12360,25 @@ JUMP_TO_LABEL(error); } DISPATCH_GOTO_NON_TRACING(); } - if ((tstate->interp->jit_state.prev_instr->op.code == CALL_LIST_APPEND && + if ((tstate->interp->jit_state.prev_state.instr->op.code == CALL_LIST_APPEND && opcode == POP_TOP) || - (tstate->interp->jit_state.prev_instr->op.code == BINARY_OP_INPLACE_ADD_UNICODE && + (tstate->interp->jit_state.prev_state.instr->op.code == BINARY_OP_INPLACE_ADD_UNICODE && opcode == STORE_FAST)) { - tstate->interp->jit_state.prev_instr_is_super = true; + tstate->interp->jit_state.prev_state.instr_is_super = true; } else { - tstate->interp->jit_state.prev_instr = next_instr; + tstate->interp->jit_state.prev_state.instr = next_instr; } - tstate->interp->jit_state.specialize_counter = 0; + tstate->interp->jit_state.prev_state.specialize_counter = 0; PyCodeObject *prev_code = (PyCodeObject *)Py_NewRef(PyStackRef_AsPyObjectBorrow(frame->f_executable)); - if (tstate->interp->jit_state.prev_instr_code != prev_code) { + if (tstate->interp->jit_state.prev_state.instr_code != prev_code) { _PyFrame_SetStackPointer(frame, stack_pointer); - Py_SETREF(tstate->interp->jit_state.prev_instr_code, prev_code); + Py_SETREF(tstate->interp->jit_state.prev_state.instr_code, prev_code); stack_pointer = _PyFrame_GetStackPointer(frame); } - tstate->interp->jit_state.prev_instr_frame = frame; - tstate->interp->jit_state.prev_instr_oparg = oparg; - tstate->interp->jit_state.prev_instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); + tstate->interp->jit_state.prev_state.instr_frame = frame; + tstate->interp->jit_state.prev_state.instr_oparg = oparg; + tstate->interp->jit_state.prev_state.instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); DISPATCH_GOTO_NON_TRACING(); #else Py_FatalError("JIT label executed in non-jit build."); diff --git a/Python/optimizer.c b/Python/optimizer.c index 90bf7069d8fe53..1e858ca3689ad1 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -142,7 +142,7 @@ _PyOptimizer_Optimize( return 0; } // One of our dependencies while tracing was invalidated. Not worth compiling. - if (!tstate->interp->jit_state.dependencies_still_valid) { + if (!tstate->interp->jit_state.prev_state.dependencies_still_valid) { interp->compiling = false; return 0; } @@ -562,7 +562,7 @@ _PyJit_translate_single_bytecode_to_trace( } #endif - PyCodeObject *old_code = tstate->interp->jit_state.prev_instr_code; + PyCodeObject *old_code = tstate->interp->jit_state.prev_state.instr_code; // Something else finalized the trace. This can happen in multi-threaded scenarios as our trace // addition from bytecode execution to here is not atomic. // Though in GIL builds, the GIL protects the rest. @@ -570,18 +570,18 @@ _PyJit_translate_single_bytecode_to_trace( return 0; } bool progress_needed = (tstate->interp->jit_state.initial_state.chain_depth % MAX_CHAIN_DEPTH) == 0; - _PyBloomFilter *dependencies = &tstate->interp->jit_state.dependencies; + _PyBloomFilter *dependencies = &tstate->interp->jit_state.prev_state.dependencies; _Py_BloomFilter_Add(dependencies, old_code); - int trace_length = tstate->interp->jit_state.code_curr_size; + int trace_length = tstate->interp->jit_state.prev_state.code_curr_size; _PyUOpInstruction *trace = tstate->interp->jit_state.code_buffer; - int max_length = tstate->interp->jit_state.code_max_size; + int max_length = tstate->interp->jit_state.prev_state.code_max_size; int is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL); if (is_sys_tracing) { goto full; } - _Py_CODEUNIT *this_instr = tstate->interp->jit_state.prev_instr; + _Py_CODEUNIT *this_instr = tstate->interp->jit_state.prev_state.instr; _Py_CODEUNIT *target_instr = this_instr; uint32_t target = 0; @@ -591,10 +591,10 @@ _PyJit_translate_single_bytecode_to_trace( // Rewind EXTENDED_ARG so that we see the whole thing. // We must point to the first EXTENDED_ARG when deopting. - int oparg = tstate->interp->jit_state.prev_instr_oparg; + int oparg = tstate->interp->jit_state.prev_state.instr_oparg; int opcode = this_instr->op.code; // Failed specialization many times. Deopt! - if (tstate->interp->jit_state.specialize_counter >= MAX_SPECIALIZATION_TRIES) { + if (tstate->interp->jit_state.prev_state.specialize_counter >= MAX_SPECIALIZATION_TRIES) { opcode = _PyOpcode_Deopt[opcode]; } int rewind_oparg = oparg; @@ -603,7 +603,7 @@ _PyJit_translate_single_bytecode_to_trace( target--; } - int old_stack_level = tstate->interp->jit_state.prev_instr_stacklevel; + int old_stack_level = tstate->interp->jit_state.prev_state.instr_stacklevel; // Strange control-flow bool has_dynamic_jump_taken = OPCODE_HAS_UNPREDICTABLE_JUMP(opcode) && @@ -611,7 +611,7 @@ _PyJit_translate_single_bytecode_to_trace( /* Special case the first instruction, * so that we can guarantee forward progress */ - if (progress_needed && tstate->interp->jit_state.code_curr_size <= 3) { + if (progress_needed && tstate->interp->jit_state.prev_state.code_curr_size <= 3) { if (OPCODE_HAS_EXIT(opcode) || OPCODE_HAS_DEOPT(opcode)) { opcode = _PyOpcode_Deopt[opcode]; } @@ -633,8 +633,8 @@ _PyJit_translate_single_bytecode_to_trace( #endif // Skip over super instructions. - if (tstate->interp->jit_state.prev_instr_is_super) { - tstate->interp->jit_state.prev_instr_is_super = false; + if (tstate->interp->jit_state.prev_state.instr_is_super) { + tstate->interp->jit_state.prev_state.instr_is_super = false; return 1; } @@ -642,13 +642,13 @@ _PyJit_translate_single_bytecode_to_trace( goto full; } - if (!tstate->interp->jit_state.dependencies_still_valid) { + if (!tstate->interp->jit_state.prev_state.dependencies_still_valid) { goto done; } // This happens when a recursive call happens that we can't trace. Such as Python -> C -> Python calls // If we haven't guarded the IP, then it's untraceable. - if (frame != tstate->interp->jit_state.prev_instr_frame && !needs_guard_ip) { + if (frame != tstate->interp->jit_state.prev_state.instr_frame && !needs_guard_ip) { DPRINTF(2, "Unsupported: unguardable jump taken\n"); goto unsupported; } @@ -750,7 +750,7 @@ _PyJit_translate_single_bytecode_to_trace( { if ((next_instr != tstate->interp->jit_state.initial_state.close_loop_instr) && (next_instr != tstate->interp->jit_state.initial_state.start_instr) && - tstate->interp->jit_state.code_curr_size > 5 && + tstate->interp->jit_state.prev_state.code_curr_size > 5 && // These are coroutines, and we want to unroll those usually. opcode != JUMP_BACKWARD_NO_INTERRUPT) { // We encountered a JUMP_BACKWARD but not to the top of our own loop. @@ -918,7 +918,7 @@ _PyJit_translate_single_bytecode_to_trace( // Loop back to the start int is_first_instr = tstate->interp->jit_state.initial_state.close_loop_instr == next_instr || tstate->interp->jit_state.initial_state.start_instr == next_instr; - if (is_first_instr && tstate->interp->jit_state.code_curr_size > 5) { + if (is_first_instr && tstate->interp->jit_state.prev_state.code_curr_size > 5) { if (needs_guard_ip) { ADD_TO_TRACE(_SET_IP, 0, (uintptr_t)next_instr, 0); } @@ -926,27 +926,27 @@ _PyJit_translate_single_bytecode_to_trace( goto done; } DPRINTF(2, "Trace continuing\n"); - tstate->interp->jit_state.code_curr_size = trace_length; - tstate->interp->jit_state.code_max_size = max_length; + tstate->interp->jit_state.prev_state.code_curr_size = trace_length; + tstate->interp->jit_state.prev_state.code_max_size = max_length; return 1; done: DPRINTF(2, "Trace done\n"); - tstate->interp->jit_state.code_curr_size = trace_length; - tstate->interp->jit_state.code_max_size = max_length; + tstate->interp->jit_state.prev_state.code_curr_size = trace_length; + tstate->interp->jit_state.prev_state.code_max_size = max_length; return 0; full: DPRINTF(2, "Trace full\n"); if (!is_terminator(&tstate->interp->jit_state.code_buffer[trace_length-1])) { // Undo the last few instructions. - trace_length = tstate->interp->jit_state.code_curr_size; - max_length = tstate->interp->jit_state.code_max_size; + trace_length = tstate->interp->jit_state.prev_state.code_curr_size; + max_length = tstate->interp->jit_state.prev_state.code_max_size; // We previously reversed one. max_length += 1; ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); trace[trace_length-1].operand1 = true; // is_control_flow } - tstate->interp->jit_state.code_curr_size = trace_length; - tstate->interp->jit_state.code_max_size = max_length; + tstate->interp->jit_state.prev_state.code_curr_size = trace_length; + tstate->interp->jit_state.prev_state.code_max_size = max_length; return 0; } @@ -959,7 +959,7 @@ _PyJit_TryInitializeTracing( { // A recursive trace. // Don't trace into the inner call because it will stomp on the previous trace, causing endless retraces. - if (tstate->interp->jit_state.code_curr_size > 2) { + if (tstate->interp->jit_state.prev_state.code_curr_size > 2) { return 0; } if (oparg > 0xFFFF) { @@ -984,9 +984,9 @@ _PyJit_TryInitializeTracing( add_to_trace(tstate->interp->jit_state.code_buffer, 0, _START_EXECUTOR, 0, (uintptr_t)start_instr, INSTR_IP(start_instr, code)); add_to_trace(tstate->interp->jit_state.code_buffer, 1, _MAKE_WARM, 0, 0, 0); - tstate->interp->jit_state.code_curr_size = 2; + tstate->interp->jit_state.prev_state.code_curr_size = 2; - tstate->interp->jit_state.code_max_size = UOP_MAX_TRACE_LENGTH; + tstate->interp->jit_state.prev_state.code_max_size = UOP_MAX_TRACE_LENGTH; tstate->interp->jit_state.initial_state.start_instr = start_instr; tstate->interp->jit_state.initial_state.close_loop_instr = close_loop_instr; tstate->interp->jit_state.initial_state.code = (PyCodeObject *)Py_NewRef(code); @@ -994,19 +994,19 @@ _PyJit_TryInitializeTracing( tstate->interp->jit_state.initial_state.exit = exit; tstate->interp->jit_state.initial_state.stack_depth = curr_stackdepth; tstate->interp->jit_state.initial_state.chain_depth = chain_depth; - tstate->interp->jit_state.prev_instr_frame = frame; - tstate->interp->jit_state.dependencies_still_valid = true; - tstate->interp->jit_state.specialize_counter = 0; - tstate->interp->jit_state.prev_instr_code = (PyCodeObject *)Py_NewRef(_PyFrame_GetCode(frame)); - tstate->interp->jit_state.prev_instr = curr_instr; - tstate->interp->jit_state.prev_instr_frame = frame; - tstate->interp->jit_state.prev_instr_oparg = oparg; - tstate->interp->jit_state.prev_instr_stacklevel = curr_stackdepth; - tstate->interp->jit_state.prev_instr_is_super = false; + tstate->interp->jit_state.prev_state.instr_frame = frame; + tstate->interp->jit_state.prev_state.dependencies_still_valid = true; + tstate->interp->jit_state.prev_state.specialize_counter = 0; + tstate->interp->jit_state.prev_state.instr_code = (PyCodeObject *)Py_NewRef(_PyFrame_GetCode(frame)); + tstate->interp->jit_state.prev_state.instr = curr_instr; + tstate->interp->jit_state.prev_state.instr_frame = frame; + tstate->interp->jit_state.prev_state.instr_oparg = oparg; + tstate->interp->jit_state.prev_state.instr_stacklevel = curr_stackdepth; + tstate->interp->jit_state.prev_state.instr_is_super = false; assert(curr_instr->op.code == JUMP_BACKWARD_JIT || (exit != NULL)); tstate->interp->jit_state.initial_state.jump_backward_instr = curr_instr; assert(curr_instr->op.code == JUMP_BACKWARD_JIT || (exit != NULL)); - _Py_BloomFilter_Init(&tstate->interp->jit_state.dependencies); + _Py_BloomFilter_Init(&tstate->interp->jit_state.prev_state.dependencies); return 1; } @@ -1015,9 +1015,9 @@ _PyJit_FinalizeTracing(PyThreadState *tstate) { Py_CLEAR(tstate->interp->jit_state.initial_state.code); Py_CLEAR(tstate->interp->jit_state.initial_state.func); - Py_CLEAR(tstate->interp->jit_state.prev_instr_code); - tstate->interp->jit_state.code_curr_size = 2; - tstate->interp->jit_state.code_max_size = UOP_MAX_TRACE_LENGTH - 1; + Py_CLEAR(tstate->interp->jit_state.prev_state.instr_code); + tstate->interp->jit_state.prev_state.code_curr_size = 2; + tstate->interp->jit_state.prev_state.code_max_size = UOP_MAX_TRACE_LENGTH - 1; } @@ -1347,7 +1347,7 @@ uop_optimize( is_noopt = false; } int curr_stackentries = tstate->interp->jit_state.initial_state.stack_depth; - int length = interp->jit_state.code_curr_size; + int length = interp->jit_state.prev_state.code_curr_size; // Trace too short, don't bother. if (length <= 5) { return 0; @@ -1725,9 +1725,9 @@ _PyJit_Tracer_InvalidateDependency(PyThreadState *tstate, void *obj) _Py_BloomFilter_Init(&obj_filter); _Py_BloomFilter_Add(&obj_filter, obj); - if (bloom_filter_may_contain(&tstate->interp->jit_state.dependencies, &obj_filter)) + if (bloom_filter_may_contain(&tstate->interp->jit_state.prev_state.dependencies, &obj_filter)) { - tstate->interp->jit_state.dependencies_still_valid = false; + tstate->interp->jit_state.prev_state.dependencies_still_valid = false; } } /* Invalidate all executors */ diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 67eba7dec09cce..4ede31992a7dfb 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -162,7 +162,7 @@ def dispatch_same_oparg( assert "specializing" in uop.annotations, uop.name self.out.start_line() self.emit("#if _Py_TIER2\n") - self.emit("tstate->interp->jit_state.specialize_counter++;\n") + self.emit("tstate->interp->jit_state.prev_state.specialize_counter++;\n") self.emit("#endif\n") self.emit(tkn) emit_to(self.out, tkn_iter, "SEMI") From 10fa14aabba2d89572dd440322c013af60906fcb Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 7 Nov 2025 21:05:21 +0000 Subject: [PATCH 169/190] reduce diff --- Include/internal/pycore_code.h | 2 -- Python/specialize.c | 11 ++++++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 7970d674440c93..2d7d81d491c157 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -672,8 +672,6 @@ PyAPI_FUNC(int) _PyCode_ReturnsOnlyNone(PyCodeObject *); * compare bytes and str which can raise a BytesWarning exception. */ extern PyObject* _PyCode_ConstantKey(PyObject *obj); -#define NO_LOC_4 (128 | (PY_CODE_LOCATION_INFO_NONE << 3) | 3) - #ifdef __cplusplus } diff --git a/Python/specialize.c b/Python/specialize.c index 8a7ba00e2e101d..a1c5dedd61563b 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -3196,6 +3196,12 @@ _Py_GatherStats_GetIter(_PyStackRef iterable) * be lifted. */ +#define NO_LOC_4 (128 | (PY_CODE_LOCATION_INFO_NONE << 3) | 3) + +static const PyBytesObject no_location = { + PyVarObject_HEAD_INIT(&PyBytes_Type, 1) + .ob_sval = { NO_LOC_4 } +}; #ifdef Py_GIL_DISABLED static _PyCodeArray init_cleanup_tlbc = { @@ -3204,11 +3210,6 @@ static _PyCodeArray init_cleanup_tlbc = { }; #endif -static const PyBytesObject no_location = { - PyVarObject_HEAD_INIT(&PyBytes_Type, 1) - .ob_sval = { NO_LOC_4 } -}; - const struct _PyCode8 _Py_InitCleanup = { _PyVarObject_HEAD_INIT(&PyCode_Type, 3), .co_consts = (PyObject *)&_Py_SINGLETON(tuple_empty), From 0f978c44d64481492bf99ae441f4264aa03da1a5 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 7 Nov 2025 21:08:12 +0000 Subject: [PATCH 170/190] fix windows --- Include/internal/pycore_interp_structs.h | 50 +++++++++++++----------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 704a0842499365..a2b5afa8736c73 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -757,31 +757,35 @@ struct _Py_unique_id_pool { typedef _Py_CODEUNIT *(*_PyJitEntryFuncPtr)(struct _PyExecutorObject *exec, _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate); +typedef struct _PyJitTracerInitialState { + int stack_depth; + int chain_depth; + struct _PyExitData *exit; + PyCodeObject *code; // Strong + PyFunctionObject *func; // Strong + _Py_CODEUNIT *start_instr; + _Py_CODEUNIT *close_loop_instr; + _Py_CODEUNIT *jump_backward_instr; +} _PyJitTracerInitialState; + +typedef struct _PyJitTracerPreviousState { + bool dependencies_still_valid; + bool instr_is_super; + int code_max_size; + int code_curr_size; + int instr_oparg; + int instr_stacklevel; + int specialize_counter; + _Py_CODEUNIT *instr; + PyCodeObject *instr_code; // Strong + _PyInterpreterFrame *instr_frame; + _PyBloomFilter dependencies; +} _PyJitTracerPreviousState; + typedef struct _PyJitTracerState { - struct { - int stack_depth; - int chain_depth; - struct _PyExitData *exit; - PyCodeObject *code; // Strong - PyFunctionObject *func; // Strong - _Py_CODEUNIT *start_instr; - _Py_CODEUNIT *close_loop_instr; - _Py_CODEUNIT *jump_backward_instr; - } initial_state; _PyUOpInstruction *code_buffer; - struct { - bool dependencies_still_valid; - bool instr_is_super; - int code_max_size; - int code_curr_size; - int instr_oparg; - int instr_stacklevel; - int specialize_counter; - _Py_CODEUNIT *instr; - PyCodeObject *instr_code; // Strong - _PyInterpreterFrame *instr_frame; - _PyBloomFilter dependencies; - } prev_state;; + _PyJitTracerInitialState initial_state; + _PyJitTracerPreviousState prev_state; } _PyJitTracerState; /* PyInterpreterState holds the global state for one of the runtime's From 3c8044698565689ee005df134bffbfeec80c3356 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 7 Nov 2025 21:11:24 +0000 Subject: [PATCH 171/190] make mypy happy --- Tools/cases_generator/tier2_generator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index b6eab511232e5c..8a93b6277155ef 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -146,8 +146,8 @@ def offset_of( assert uop.name.startswith("_GUARD_IP") # LPAREN next(tkn_iter) - inst = next(tkn_iter) - self.emit(f" OFFSET_OF_{inst.text};\n") + tok = next(tkn_iter) + self.emit(f" OFFSET_OF_{tok.text};\n") # RPAREN next(tkn_iter) # SEMI From aaf68732f8a3f67ff94dad51a52b393870d6dfac Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 7 Nov 2025 22:02:00 +0000 Subject: [PATCH 172/190] Move to thread state --- Include/internal/pycore_interp_structs.h | 33 ---- Include/internal/pycore_tstate.h | 40 ++++- Python/bytecodes.c | 28 ++-- Python/ceval.c | 11 +- Python/ceval_macros.h | 4 +- Python/generated_cases.c.h | 58 +++---- Python/optimizer.c | 172 +++++++++++---------- Python/pystate.c | 18 ++- Tools/cases_generator/generators_common.py | 2 +- 9 files changed, 187 insertions(+), 179 deletions(-) diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index a2b5afa8736c73..39023923c7e2b7 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -14,8 +14,6 @@ extern "C" { #include "pycore_structs.h" // PyHamtObject #include "pycore_tstate.h" // _PyThreadStateImpl #include "pycore_typedefs.h" // _PyRuntimeState -#include "pycore_uop.h" // struct _PyUOpInstruction - #define CODE_MAX_WATCHERS 8 #define CONTEXT_MAX_WATCHERS 8 @@ -757,36 +755,6 @@ struct _Py_unique_id_pool { typedef _Py_CODEUNIT *(*_PyJitEntryFuncPtr)(struct _PyExecutorObject *exec, _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate); -typedef struct _PyJitTracerInitialState { - int stack_depth; - int chain_depth; - struct _PyExitData *exit; - PyCodeObject *code; // Strong - PyFunctionObject *func; // Strong - _Py_CODEUNIT *start_instr; - _Py_CODEUNIT *close_loop_instr; - _Py_CODEUNIT *jump_backward_instr; -} _PyJitTracerInitialState; - -typedef struct _PyJitTracerPreviousState { - bool dependencies_still_valid; - bool instr_is_super; - int code_max_size; - int code_curr_size; - int instr_oparg; - int instr_stacklevel; - int specialize_counter; - _Py_CODEUNIT *instr; - PyCodeObject *instr_code; // Strong - _PyInterpreterFrame *instr_frame; - _PyBloomFilter dependencies; -} _PyJitTracerPreviousState; - -typedef struct _PyJitTracerState { - _PyUOpInstruction *code_buffer; - _PyJitTracerInitialState initial_state; - _PyJitTracerPreviousState prev_state; -} _PyJitTracerState; /* PyInterpreterState holds the global state for one of the runtime's interpreters. Typically the initial (main) interpreter is the only one. @@ -963,7 +931,6 @@ struct _is { struct types_state types; struct callable_cache callable_cache; PyObject *common_consts[NUM_COMMON_CONSTANTS]; - _PyJitTracerState jit_state; bool jit; bool compiling; struct _PyExecutorObject *executor_list_head; diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index bad968428c73a1..04041b273d7560 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -12,7 +12,8 @@ extern "C" { #include "pycore_freelist_state.h" // struct _Py_freelists #include "pycore_mimalloc.h" // struct _mimalloc_thread_state #include "pycore_qsbr.h" // struct qsbr - +#include "pycore_uop.h" // struct _PyUOpInstruction +#include "pycore_structs.h" #ifdef Py_GIL_DISABLED struct _gc_thread_state { @@ -21,6 +22,39 @@ struct _gc_thread_state { }; #endif +#if _Py_TIER2 +typedef struct _PyJitTracerInitialState { + int stack_depth; + int chain_depth; + struct _PyExitData *exit; + PyCodeObject *code; // Strong + PyFunctionObject *func; // Strong + _Py_CODEUNIT *start_instr; + _Py_CODEUNIT *close_loop_instr; + _Py_CODEUNIT *jump_backward_instr; +} _PyJitTracerInitialState; + +typedef struct _PyJitTracerPreviousState { + bool dependencies_still_valid; + bool instr_is_super; + int code_max_size; + int code_curr_size; + int instr_oparg; + int instr_stacklevel; + int specialize_counter; + _Py_CODEUNIT *instr; + PyCodeObject *instr_code; // Strong + struct _PyInterpreterFrame *instr_frame; + _PyBloomFilter dependencies; +} _PyJitTracerPreviousState; + +typedef struct _PyJitTracerState { + _PyUOpInstruction *code_buffer; + _PyJitTracerInitialState initial_state; + _PyJitTracerPreviousState prev_state; +} _PyJitTracerState; +#endif + // Every PyThreadState is actually allocated as a _PyThreadStateImpl. The // PyThreadState fields are exposed as part of the C API, although most fields // are intended to be private. The _PyThreadStateImpl fields not exposed. @@ -75,7 +109,9 @@ typedef struct _PyThreadStateImpl { #if defined(Py_REF_DEBUG) && defined(Py_GIL_DISABLED) Py_ssize_t reftotal; // this thread's total refcount operations #endif - +#if _Py_TIER2 + _PyJitTracerState jit_state; +#endif } _PyThreadStateImpl; #ifdef __cplusplus diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 42fb4170eae725..283a18424715b6 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2970,13 +2970,6 @@ dummy_func( if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD_JIT && next_instr->op.code != ENTER_EXECUTOR) { - if (tstate->interp->jit_state.code_buffer == NULL) { - tstate->interp->jit_state.code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); - if (tstate->interp->jit_state.code_buffer == NULL) { - // Don't error, just go to next instruction. - DISPATCH(); - } - } /* Back up over EXTENDED_ARGs so executor is inserted at the correct place */ _Py_CODEUNIT *insert_exec_at = this_instr; while (oparg > 255) { @@ -5673,24 +5666,25 @@ dummy_func( } // Super instructions. Instruction deopted. There's a mismatch in what the stack expects // in the optimizer. So we have to reflect in the trace correctly. - if ((tstate->interp->jit_state.prev_state.instr->op.code == CALL_LIST_APPEND && + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + if ((_tstate->jit_state.prev_state.instr->op.code == CALL_LIST_APPEND && opcode == POP_TOP) || - (tstate->interp->jit_state.prev_state.instr->op.code == BINARY_OP_INPLACE_ADD_UNICODE && + (_tstate->jit_state.prev_state.instr->op.code == BINARY_OP_INPLACE_ADD_UNICODE && opcode == STORE_FAST)) { - tstate->interp->jit_state.prev_state.instr_is_super = true; + _tstate->jit_state.prev_state.instr_is_super = true; } else { - tstate->interp->jit_state.prev_state.instr = next_instr; + _tstate->jit_state.prev_state.instr = next_instr; } - tstate->interp->jit_state.prev_state.specialize_counter = 0; + _tstate->jit_state.prev_state.specialize_counter = 0; PyCodeObject *prev_code = (PyCodeObject *)Py_NewRef(PyStackRef_AsPyObjectBorrow(frame->f_executable)); - if (tstate->interp->jit_state.prev_state.instr_code != prev_code) { - Py_SETREF(tstate->interp->jit_state.prev_state.instr_code, prev_code); + if (_tstate->jit_state.prev_state.instr_code != prev_code) { + Py_SETREF(_tstate->jit_state.prev_state.instr_code, prev_code); } - tstate->interp->jit_state.prev_state.instr_frame = frame; - tstate->interp->jit_state.prev_state.instr_oparg = oparg; - tstate->interp->jit_state.prev_state.instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); + _tstate->jit_state.prev_state.instr_frame = frame; + _tstate->jit_state.prev_state.instr_oparg = oparg; + _tstate->jit_state.prev_state.instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); DISPATCH_GOTO_NON_TRACING(); #else Py_FatalError("JIT label executed in non-jit build."); diff --git a/Python/ceval.c b/Python/ceval.c index ce6493fdd85284..e6424503e88988 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1000,16 +1000,17 @@ bail_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame) if (!_PyErr_Occurred(tstate) && !_is_sys_tracing) { err = _PyOptimizer_Optimize(frame, tstate); } + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; // Deal with backoffs - _PyExitData *exit = tstate->interp->jit_state.initial_state.exit; + _PyExitData *exit = _tstate->jit_state.initial_state.exit; if (exit == NULL) { // We hold a strong reference to the code object, so the instruction won't be freed. if (err <= 0) { - _Py_BackoffCounter counter = tstate->interp->jit_state.initial_state.jump_backward_instr[1].counter; - tstate->interp->jit_state.initial_state.jump_backward_instr[1].counter = restart_backoff_counter(counter); + _Py_BackoffCounter counter = _tstate->jit_state.initial_state.jump_backward_instr[1].counter; + _tstate->jit_state.initial_state.jump_backward_instr[1].counter = restart_backoff_counter(counter); } else { - tstate->interp->jit_state.initial_state.jump_backward_instr[1].counter = initial_jump_backoff_counter(); + _tstate->jit_state.initial_state.jump_backward_instr[1].counter = initial_jump_backoff_counter(); } } else { @@ -1017,7 +1018,7 @@ bail_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame) // to be valid to access. if (err <= 0) { // Some opcodes will forever be unchanged. Don't ever bother specializing for them ever again. - if (tstate->interp->jit_state.prev_state.instr->op.code == INTERPRETER_EXIT) { + if (_tstate->jit_state.prev_state.instr->op.code == INTERPRETER_EXIT) { exit->temperature = initial_unreachable_backoff_counter(); } else { diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 219874ac240bd6..c1062e9d091d3e 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -134,7 +134,7 @@ #if (_Py_TAIL_CALL_INTERP || USE_COMPUTED_GOTOS) && _Py_TIER2 # define IS_JIT_TRACING() (DISPATCH_TABLE_VAR == TRACING_DISPATCH_TABLE) -# define IS_JIT_TRACING_MAKING_PROGRESS() (IS_JIT_TRACING() && tstate->interp->jit_state.prev_state.specialize_counter < MAX_SPECIALIZATION_TRIES) +# define IS_JIT_TRACING_MAKING_PROGRESS() (IS_JIT_TRACING() && ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter < MAX_SPECIALIZATION_TRIES) # define ENTER_TRACING() \ DISPATCH_TABLE_VAR = TRACING_DISPATCH_TABLE; # define LEAVE_TRACING() \ @@ -402,7 +402,7 @@ do { \ JUMP_TO_LABEL(error); \ } \ if (keep_tracing_bit) { \ - assert(tstate->interp->jit_state.prev_state.code_curr_size == 2); \ + assert(((_PyThreadStateImpl *)tstate)->jit_state.prev_state.code_curr_size == 2); \ ENTER_TRACING(); \ DISPATCH_NON_TRACING(); \ } \ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 24c4271c88fab2..a45138e6e6c5b9 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -45,7 +45,7 @@ _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); stack_pointer = _PyFrame_GetStackPointer(frame); #if _Py_TIER2 - tstate->interp->jit_state.prev_state.specialize_counter++; + ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++; #endif DISPATCH_SAME_OPARG(); } @@ -1539,7 +1539,7 @@ _Py_Specialize_Call(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); stack_pointer = _PyFrame_GetStackPointer(frame); #if _Py_TIER2 - tstate->interp->jit_state.prev_state.specialize_counter++; + ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++; #endif DISPATCH_SAME_OPARG(); } @@ -2832,7 +2832,7 @@ _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); stack_pointer = _PyFrame_GetStackPointer(frame); #if _Py_TIER2 - tstate->interp->jit_state.prev_state.specialize_counter++; + ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++; #endif DISPATCH_SAME_OPARG(); } @@ -4678,7 +4678,7 @@ _Py_Specialize_CompareOp(left, right, next_instr, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); #if _Py_TIER2 - tstate->interp->jit_state.prev_state.specialize_counter++; + ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++; #endif DISPATCH_SAME_OPARG(); } @@ -4926,7 +4926,7 @@ _Py_Specialize_ContainsOp(right, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); #if _Py_TIER2 - tstate->interp->jit_state.prev_state.specialize_counter++; + ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++; #endif DISPATCH_SAME_OPARG(); } @@ -5665,7 +5665,7 @@ _Py_Specialize_ForIter(iter, null_or_index, next_instr, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); #if _Py_TIER2 - tstate->interp->jit_state.prev_state.specialize_counter++; + ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++; #endif DISPATCH_SAME_OPARG(); } @@ -7647,7 +7647,7 @@ FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, desired); next_instr = this_instr; #if _Py_TIER2 - tstate->interp->jit_state.prev_state.specialize_counter++; + ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++; #endif DISPATCH_SAME_OPARG(); } @@ -7703,11 +7703,12 @@ if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD_JIT && next_instr->op.code != ENTER_EXECUTOR) { - if (tstate->interp->jit_state.code_buffer == NULL) { + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + if (_tstate->jit_state.code_buffer == NULL) { _PyFrame_SetStackPointer(frame, stack_pointer); - tstate->interp->jit_state.code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); + _tstate->jit_state.code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); stack_pointer = _PyFrame_GetStackPointer(frame); - if (tstate->interp->jit_state.code_buffer == NULL) { + if (_tstate->jit_state.code_buffer == NULL) { DISPATCH(); } } @@ -7880,7 +7881,7 @@ _Py_Specialize_LoadAttr(owner, next_instr, name); stack_pointer = _PyFrame_GetStackPointer(frame); #if _Py_TIER2 - tstate->interp->jit_state.prev_state.specialize_counter++; + ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++; #endif DISPATCH_SAME_OPARG(); } @@ -9188,7 +9189,7 @@ _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); stack_pointer = _PyFrame_GetStackPointer(frame); #if _Py_TIER2 - tstate->interp->jit_state.prev_state.specialize_counter++; + ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++; #endif DISPATCH_SAME_OPARG(); } @@ -9511,7 +9512,7 @@ _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); stack_pointer = _PyFrame_GetStackPointer(frame); #if _Py_TIER2 - tstate->interp->jit_state.prev_state.specialize_counter++; + ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++; #endif DISPATCH_SAME_OPARG(); } @@ -10499,7 +10500,7 @@ _Py_Specialize_Send(receiver, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); #if _Py_TIER2 - tstate->interp->jit_state.prev_state.specialize_counter++; + ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++; #endif DISPATCH_SAME_OPARG(); } @@ -10801,7 +10802,7 @@ _Py_Specialize_StoreAttr(owner, next_instr, name); stack_pointer = _PyFrame_GetStackPointer(frame); #if _Py_TIER2 - tstate->interp->jit_state.prev_state.specialize_counter++; + ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++; #endif DISPATCH_SAME_OPARG(); } @@ -11301,7 +11302,7 @@ _Py_Specialize_StoreSubscr(container, sub, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); #if _Py_TIER2 - tstate->interp->jit_state.prev_state.specialize_counter++; + ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++; #endif DISPATCH_SAME_OPARG(); } @@ -11513,7 +11514,7 @@ _Py_Specialize_ToBool(value, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); #if _Py_TIER2 - tstate->interp->jit_state.prev_state.specialize_counter++; + ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++; #endif DISPATCH_SAME_OPARG(); } @@ -11896,7 +11897,7 @@ _Py_Specialize_UnpackSequence(seq, next_instr, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); #if _Py_TIER2 - tstate->interp->jit_state.prev_state.specialize_counter++; + ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++; #endif DISPATCH_SAME_OPARG(); } @@ -12360,25 +12361,26 @@ JUMP_TO_LABEL(error); } DISPATCH_GOTO_NON_TRACING(); } - if ((tstate->interp->jit_state.prev_state.instr->op.code == CALL_LIST_APPEND && + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + if ((_tstate->jit_state.prev_state.instr->op.code == CALL_LIST_APPEND && opcode == POP_TOP) || - (tstate->interp->jit_state.prev_state.instr->op.code == BINARY_OP_INPLACE_ADD_UNICODE && + (_tstate->jit_state.prev_state.instr->op.code == BINARY_OP_INPLACE_ADD_UNICODE && opcode == STORE_FAST)) { - tstate->interp->jit_state.prev_state.instr_is_super = true; + _tstate->jit_state.prev_state.instr_is_super = true; } else { - tstate->interp->jit_state.prev_state.instr = next_instr; + _tstate->jit_state.prev_state.instr = next_instr; } - tstate->interp->jit_state.prev_state.specialize_counter = 0; + _tstate->jit_state.prev_state.specialize_counter = 0; PyCodeObject *prev_code = (PyCodeObject *)Py_NewRef(PyStackRef_AsPyObjectBorrow(frame->f_executable)); - if (tstate->interp->jit_state.prev_state.instr_code != prev_code) { + if (_tstate->jit_state.prev_state.instr_code != prev_code) { _PyFrame_SetStackPointer(frame, stack_pointer); - Py_SETREF(tstate->interp->jit_state.prev_state.instr_code, prev_code); + Py_SETREF(_tstate->jit_state.prev_state.instr_code, prev_code); stack_pointer = _PyFrame_GetStackPointer(frame); } - tstate->interp->jit_state.prev_state.instr_frame = frame; - tstate->interp->jit_state.prev_state.instr_oparg = oparg; - tstate->interp->jit_state.prev_state.instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); + _tstate->jit_state.prev_state.instr_frame = frame; + _tstate->jit_state.prev_state.instr_oparg = oparg; + _tstate->jit_state.prev_state.instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); DISPATCH_GOTO_NON_TRACING(); #else Py_FatalError("JIT label executed in non-jit build."); diff --git a/Python/optimizer.c b/Python/optimizer.c index 1e858ca3689ad1..c38f28e224a8b6 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -118,14 +118,15 @@ Py_NO_INLINE int _PyOptimizer_Optimize( _PyInterpreterFrame *frame, PyThreadState *tstate) { + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + int chain_depth = _tstate->jit_state.initial_state.chain_depth; PyInterpreterState *interp = _PyInterpreterState_GET(); - int chain_depth = tstate->interp->jit_state.initial_state.chain_depth; assert(interp->jit); assert(!interp->compiling); - assert(tstate->interp->jit_state.initial_state.stack_depth >= 0); + assert(_tstate->jit_state.initial_state.stack_depth >= 0); #ifndef Py_GIL_DISABLED // Trace got stomped on by another thread. - if (tstate->interp->jit_state.initial_state.func == NULL) { + if (_tstate->jit_state.initial_state.func == NULL) { return 0; } interp->compiling = true; @@ -135,14 +136,14 @@ _PyOptimizer_Optimize( // this is true, since a deopt won't infinitely re-enter the executor: chain_depth %= MAX_CHAIN_DEPTH; bool progress_needed = chain_depth == 0; - PyCodeObject *code = (PyCodeObject *)tstate->interp->jit_state.initial_state.code; - _Py_CODEUNIT *start = tstate->interp->jit_state.initial_state.start_instr; + PyCodeObject *code = (PyCodeObject *)_tstate->jit_state.initial_state.code; + _Py_CODEUNIT *start = _tstate->jit_state.initial_state.start_instr; if (progress_needed && !has_space_for_executor(code, start)) { interp->compiling = false; return 0; } // One of our dependencies while tracing was invalidated. Not worth compiling. - if (!tstate->interp->jit_state.prev_state.dependencies_still_valid) { + if (!_tstate->jit_state.prev_state.dependencies_still_valid) { interp->compiling = false; return 0; } @@ -171,7 +172,7 @@ _PyOptimizer_Optimize( else { executor->vm_data.code = NULL; } - _PyExitData *exit = tstate->interp->jit_state.initial_state.exit; + _PyExitData *exit = _tstate->jit_state.initial_state.exit; if (exit != NULL) { exit->executor = executor; } @@ -561,27 +562,21 @@ _PyJit_translate_single_bytecode_to_trace( lltrace = *python_lltrace - '0'; // TODO: Parse an int and all that } #endif - - PyCodeObject *old_code = tstate->interp->jit_state.prev_state.instr_code; - // Something else finalized the trace. This can happen in multi-threaded scenarios as our trace - // addition from bytecode execution to here is not atomic. - // Though in GIL builds, the GIL protects the rest. - if (old_code == NULL) { - return 0; - } - bool progress_needed = (tstate->interp->jit_state.initial_state.chain_depth % MAX_CHAIN_DEPTH) == 0; - _PyBloomFilter *dependencies = &tstate->interp->jit_state.prev_state.dependencies; + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + PyCodeObject *old_code = _tstate->jit_state.prev_state.instr_code; + bool progress_needed = (_tstate->jit_state.initial_state.chain_depth % MAX_CHAIN_DEPTH) == 0; + _PyBloomFilter *dependencies = &_tstate->jit_state.prev_state.dependencies; _Py_BloomFilter_Add(dependencies, old_code); - int trace_length = tstate->interp->jit_state.prev_state.code_curr_size; - _PyUOpInstruction *trace = tstate->interp->jit_state.code_buffer; - int max_length = tstate->interp->jit_state.prev_state.code_max_size; + int trace_length = _tstate->jit_state.prev_state.code_curr_size; + _PyUOpInstruction *trace = _tstate->jit_state.code_buffer; + int max_length = _tstate->jit_state.prev_state.code_max_size; int is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL); if (is_sys_tracing) { goto full; } - _Py_CODEUNIT *this_instr = tstate->interp->jit_state.prev_state.instr; + _Py_CODEUNIT *this_instr = _tstate->jit_state.prev_state.instr; _Py_CODEUNIT *target_instr = this_instr; uint32_t target = 0; @@ -591,10 +586,10 @@ _PyJit_translate_single_bytecode_to_trace( // Rewind EXTENDED_ARG so that we see the whole thing. // We must point to the first EXTENDED_ARG when deopting. - int oparg = tstate->interp->jit_state.prev_state.instr_oparg; + int oparg = _tstate->jit_state.prev_state.instr_oparg; int opcode = this_instr->op.code; // Failed specialization many times. Deopt! - if (tstate->interp->jit_state.prev_state.specialize_counter >= MAX_SPECIALIZATION_TRIES) { + if (_tstate->jit_state.prev_state.specialize_counter >= MAX_SPECIALIZATION_TRIES) { opcode = _PyOpcode_Deopt[opcode]; } int rewind_oparg = oparg; @@ -603,7 +598,7 @@ _PyJit_translate_single_bytecode_to_trace( target--; } - int old_stack_level = tstate->interp->jit_state.prev_state.instr_stacklevel; + int old_stack_level = _tstate->jit_state.prev_state.instr_stacklevel; // Strange control-flow bool has_dynamic_jump_taken = OPCODE_HAS_UNPREDICTABLE_JUMP(opcode) && @@ -611,7 +606,7 @@ _PyJit_translate_single_bytecode_to_trace( /* Special case the first instruction, * so that we can guarantee forward progress */ - if (progress_needed && tstate->interp->jit_state.prev_state.code_curr_size <= 3) { + if (progress_needed && _tstate->jit_state.prev_state.code_curr_size <= 3) { if (OPCODE_HAS_EXIT(opcode) || OPCODE_HAS_DEOPT(opcode)) { opcode = _PyOpcode_Deopt[opcode]; } @@ -633,8 +628,8 @@ _PyJit_translate_single_bytecode_to_trace( #endif // Skip over super instructions. - if (tstate->interp->jit_state.prev_state.instr_is_super) { - tstate->interp->jit_state.prev_state.instr_is_super = false; + if (_tstate->jit_state.prev_state.instr_is_super) { + _tstate->jit_state.prev_state.instr_is_super = false; return 1; } @@ -642,13 +637,13 @@ _PyJit_translate_single_bytecode_to_trace( goto full; } - if (!tstate->interp->jit_state.prev_state.dependencies_still_valid) { + if (!_tstate->jit_state.prev_state.dependencies_still_valid) { goto done; } // This happens when a recursive call happens that we can't trace. Such as Python -> C -> Python calls // If we haven't guarded the IP, then it's untraceable. - if (frame != tstate->interp->jit_state.prev_state.instr_frame && !needs_guard_ip) { + if (frame != _tstate->jit_state.prev_state.instr_frame && !needs_guard_ip) { DPRINTF(2, "Unsupported: unguardable jump taken\n"); goto unsupported; } @@ -748,9 +743,9 @@ _PyJit_translate_single_bytecode_to_trace( _Py_FALLTHROUGH; case JUMP_BACKWARD_NO_INTERRUPT: { - if ((next_instr != tstate->interp->jit_state.initial_state.close_loop_instr) && - (next_instr != tstate->interp->jit_state.initial_state.start_instr) && - tstate->interp->jit_state.prev_state.code_curr_size > 5 && + if ((next_instr != _tstate->jit_state.initial_state.close_loop_instr) && + (next_instr != _tstate->jit_state.initial_state.start_instr) && + _tstate->jit_state.prev_state.code_curr_size > 5 && // These are coroutines, and we want to unroll those usually. opcode != JUMP_BACKWARD_NO_INTERRUPT) { // We encountered a JUMP_BACKWARD but not to the top of our own loop. @@ -761,7 +756,7 @@ _PyJit_translate_single_bytecode_to_trace( ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); trace[trace_length-1].operand1 = true; // is_control_flow DPRINTF(2, "JUMP_BACKWARD not to top ends trace %p %p %p\n", next_instr, - tstate->interp->jit_state.initial_state.close_loop_instr, tstate->interp->jit_state.initial_state.start_instr); + _tstate->jit_state.initial_state.close_loop_instr, _tstate->jit_state.initial_state.start_instr); goto done; } break; @@ -916,9 +911,9 @@ _PyJit_translate_single_bytecode_to_trace( } } // Loop back to the start - int is_first_instr = tstate->interp->jit_state.initial_state.close_loop_instr == next_instr || - tstate->interp->jit_state.initial_state.start_instr == next_instr; - if (is_first_instr && tstate->interp->jit_state.prev_state.code_curr_size > 5) { + int is_first_instr = _tstate->jit_state.initial_state.close_loop_instr == next_instr || + _tstate->jit_state.initial_state.start_instr == next_instr; + if (is_first_instr && _tstate->jit_state.prev_state.code_curr_size > 5) { if (needs_guard_ip) { ADD_TO_TRACE(_SET_IP, 0, (uintptr_t)next_instr, 0); } @@ -926,27 +921,27 @@ _PyJit_translate_single_bytecode_to_trace( goto done; } DPRINTF(2, "Trace continuing\n"); - tstate->interp->jit_state.prev_state.code_curr_size = trace_length; - tstate->interp->jit_state.prev_state.code_max_size = max_length; + _tstate->jit_state.prev_state.code_curr_size = trace_length; + _tstate->jit_state.prev_state.code_max_size = max_length; return 1; done: DPRINTF(2, "Trace done\n"); - tstate->interp->jit_state.prev_state.code_curr_size = trace_length; - tstate->interp->jit_state.prev_state.code_max_size = max_length; + _tstate->jit_state.prev_state.code_curr_size = trace_length; + _tstate->jit_state.prev_state.code_max_size = max_length; return 0; full: DPRINTF(2, "Trace full\n"); - if (!is_terminator(&tstate->interp->jit_state.code_buffer[trace_length-1])) { + if (!is_terminator(&_tstate->jit_state.code_buffer[trace_length-1])) { // Undo the last few instructions. - trace_length = tstate->interp->jit_state.prev_state.code_curr_size; - max_length = tstate->interp->jit_state.prev_state.code_max_size; + trace_length = _tstate->jit_state.prev_state.code_curr_size; + max_length = _tstate->jit_state.prev_state.code_max_size; // We previously reversed one. max_length += 1; ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); trace[trace_length-1].operand1 = true; // is_control_flow } - tstate->interp->jit_state.prev_state.code_curr_size = trace_length; - tstate->interp->jit_state.prev_state.code_max_size = max_length; + _tstate->jit_state.prev_state.code_curr_size = trace_length; + _tstate->jit_state.prev_state.code_max_size = max_length; return 0; } @@ -957,14 +952,22 @@ _PyJit_TryInitializeTracing( _Py_CODEUNIT *start_instr, _Py_CODEUNIT *close_loop_instr, int curr_stackdepth, int chain_depth, _PyExitData *exit, int oparg) { + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; // A recursive trace. // Don't trace into the inner call because it will stomp on the previous trace, causing endless retraces. - if (tstate->interp->jit_state.prev_state.code_curr_size > 2) { + if (_tstate->jit_state.prev_state.code_curr_size > 2) { return 0; } if (oparg > 0xFFFF) { return 0; } + if (_tstate->jit_state.code_buffer == NULL) { + _tstate->jit_state.code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); + if (_tstate->jit_state.code_buffer == NULL) { + // Don't error, just go to next instruction. + return 0; + } + } PyCodeObject *code = _PyFrame_GetCode(frame); #ifdef Py_DEBUG @@ -982,42 +985,43 @@ _PyJit_TryInitializeTracing( chain_depth); #endif - add_to_trace(tstate->interp->jit_state.code_buffer, 0, _START_EXECUTOR, 0, (uintptr_t)start_instr, INSTR_IP(start_instr, code)); - add_to_trace(tstate->interp->jit_state.code_buffer, 1, _MAKE_WARM, 0, 0, 0); - tstate->interp->jit_state.prev_state.code_curr_size = 2; - - tstate->interp->jit_state.prev_state.code_max_size = UOP_MAX_TRACE_LENGTH; - tstate->interp->jit_state.initial_state.start_instr = start_instr; - tstate->interp->jit_state.initial_state.close_loop_instr = close_loop_instr; - tstate->interp->jit_state.initial_state.code = (PyCodeObject *)Py_NewRef(code); - tstate->interp->jit_state.initial_state.func = (PyFunctionObject *)Py_XNewRef(PyStackRef_AsPyObjectBorrow(frame->f_funcobj)); - tstate->interp->jit_state.initial_state.exit = exit; - tstate->interp->jit_state.initial_state.stack_depth = curr_stackdepth; - tstate->interp->jit_state.initial_state.chain_depth = chain_depth; - tstate->interp->jit_state.prev_state.instr_frame = frame; - tstate->interp->jit_state.prev_state.dependencies_still_valid = true; - tstate->interp->jit_state.prev_state.specialize_counter = 0; - tstate->interp->jit_state.prev_state.instr_code = (PyCodeObject *)Py_NewRef(_PyFrame_GetCode(frame)); - tstate->interp->jit_state.prev_state.instr = curr_instr; - tstate->interp->jit_state.prev_state.instr_frame = frame; - tstate->interp->jit_state.prev_state.instr_oparg = oparg; - tstate->interp->jit_state.prev_state.instr_stacklevel = curr_stackdepth; - tstate->interp->jit_state.prev_state.instr_is_super = false; + add_to_trace(_tstate->jit_state.code_buffer, 0, _START_EXECUTOR, 0, (uintptr_t)start_instr, INSTR_IP(start_instr, code)); + add_to_trace(_tstate->jit_state.code_buffer, 1, _MAKE_WARM, 0, 0, 0); + _tstate->jit_state.prev_state.code_curr_size = 2; + + _tstate->jit_state.prev_state.code_max_size = UOP_MAX_TRACE_LENGTH; + _tstate->jit_state.initial_state.start_instr = start_instr; + _tstate->jit_state.initial_state.close_loop_instr = close_loop_instr; + _tstate->jit_state.initial_state.code = (PyCodeObject *)Py_NewRef(code); + _tstate->jit_state.initial_state.func = (PyFunctionObject *)Py_XNewRef(PyStackRef_AsPyObjectBorrow(frame->f_funcobj)); + _tstate->jit_state.initial_state.exit = exit; + _tstate->jit_state.initial_state.stack_depth = curr_stackdepth; + _tstate->jit_state.initial_state.chain_depth = chain_depth; + _tstate->jit_state.prev_state.instr_frame = frame; + _tstate->jit_state.prev_state.dependencies_still_valid = true; + _tstate->jit_state.prev_state.specialize_counter = 0; + _tstate->jit_state.prev_state.instr_code = (PyCodeObject *)Py_NewRef(_PyFrame_GetCode(frame)); + _tstate->jit_state.prev_state.instr = curr_instr; + _tstate->jit_state.prev_state.instr_frame = frame; + _tstate->jit_state.prev_state.instr_oparg = oparg; + _tstate->jit_state.prev_state.instr_stacklevel = curr_stackdepth; + _tstate->jit_state.prev_state.instr_is_super = false; assert(curr_instr->op.code == JUMP_BACKWARD_JIT || (exit != NULL)); - tstate->interp->jit_state.initial_state.jump_backward_instr = curr_instr; + _tstate->jit_state.initial_state.jump_backward_instr = curr_instr; assert(curr_instr->op.code == JUMP_BACKWARD_JIT || (exit != NULL)); - _Py_BloomFilter_Init(&tstate->interp->jit_state.prev_state.dependencies); + _Py_BloomFilter_Init(&_tstate->jit_state.prev_state.dependencies); return 1; } void _PyJit_FinalizeTracing(PyThreadState *tstate) { - Py_CLEAR(tstate->interp->jit_state.initial_state.code); - Py_CLEAR(tstate->interp->jit_state.initial_state.func); - Py_CLEAR(tstate->interp->jit_state.prev_state.instr_code); - tstate->interp->jit_state.prev_state.code_curr_size = 2; - tstate->interp->jit_state.prev_state.code_max_size = UOP_MAX_TRACE_LENGTH - 1; + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + Py_CLEAR(_tstate->jit_state.initial_state.code); + Py_CLEAR(_tstate->jit_state.initial_state.func); + Py_CLEAR(_tstate->jit_state.prev_state.instr_code); + _tstate->jit_state.prev_state.code_curr_size = 2; + _tstate->jit_state.prev_state.code_max_size = UOP_MAX_TRACE_LENGTH - 1; } @@ -1327,6 +1331,7 @@ uop_optimize( _PyExecutorObject **exec_ptr, bool progress_needed) { + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; // Note: the executor has a slightly different set of dependencies than the tracer. // For example: the tracer depends on function and code objects. // The executor may only depend on the code object. @@ -1337,17 +1342,16 @@ uop_optimize( // It is the optimizer's responsibility to add the dependencies it requires on its own. _PyBloomFilter new_dependencies; _Py_BloomFilter_Init(&new_dependencies); - _Py_BloomFilter_Add(&new_dependencies, tstate->interp->jit_state.initial_state.code); - PyInterpreterState *interp = _PyInterpreterState_GET(); - _PyUOpInstruction *buffer = interp->jit_state.code_buffer; + _Py_BloomFilter_Add(&new_dependencies, _tstate->jit_state.initial_state.code); + _PyUOpInstruction *buffer = _tstate->jit_state.code_buffer; OPT_STAT_INC(attempts); char *env_var = Py_GETENV("PYTHON_UOPS_OPTIMIZE"); bool is_noopt = true; if (env_var == NULL || *env_var == '\0' || *env_var > '0') { is_noopt = false; } - int curr_stackentries = tstate->interp->jit_state.initial_state.stack_depth; - int length = interp->jit_state.prev_state.code_curr_size; + int curr_stackentries = _tstate->jit_state.initial_state.stack_depth; + int length = _tstate->jit_state.prev_state.code_curr_size; // Trace too short, don't bother. if (length <= 5) { return 0; @@ -1356,7 +1360,7 @@ uop_optimize( assert(length < UOP_MAX_TRACE_LENGTH); OPT_STAT_INC(traces_created); if (!is_noopt) { - length = _Py_uop_analyze_and_optimize(tstate->interp->jit_state.initial_state.func, buffer, + length = _Py_uop_analyze_and_optimize(_tstate->jit_state.initial_state.func, buffer, length, curr_stackentries, &new_dependencies); if (length <= 0) { @@ -1382,7 +1386,7 @@ uop_optimize( length = prepare_for_execution(buffer, length); assert(length <= UOP_MAX_TRACE_LENGTH); _PyExecutorObject *executor = make_executor_from_uops( - buffer, length, &new_dependencies, tstate->interp->jit_state.initial_state.chain_depth); + buffer, length, &new_dependencies, _tstate->jit_state.initial_state.chain_depth); if (executor == NULL) { return -1; } @@ -1724,10 +1728,10 @@ _PyJit_Tracer_InvalidateDependency(PyThreadState *tstate, void *obj) _PyBloomFilter obj_filter; _Py_BloomFilter_Init(&obj_filter); _Py_BloomFilter_Add(&obj_filter, obj); - - if (bloom_filter_may_contain(&tstate->interp->jit_state.prev_state.dependencies, &obj_filter)) + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + if (bloom_filter_may_contain(&_tstate->jit_state.prev_state.dependencies, &obj_filter)) { - tstate->interp->jit_state.prev_state.dependencies_still_valid = false; + _tstate->jit_state.prev_state.dependencies_still_valid = false; } } /* Invalidate all executors */ diff --git a/Python/pystate.c b/Python/pystate.c index 36d62ecae2ee0b..a6a54c2f94a7dc 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -545,9 +545,6 @@ init_interpreter(PyInterpreterState *interp, _Py_brc_init_state(interp); #endif -#ifdef _Py_TIER2 - interp->jit_state.code_buffer = NULL; -#endif llist_init(&interp->mem_free_queue.head); llist_init(&interp->asyncio_tasks_head); interp->asyncio_tasks_lock = (PyMutex){0}; @@ -797,10 +794,6 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) #ifdef _Py_TIER2 _Py_ClearExecutorDeletionList(interp); - if (interp->jit_state.code_buffer != NULL) { - _PyObject_VirtualFree(interp->jit_state.code_buffer, UOP_BUFFER_SIZE); - interp->jit_state.code_buffer = NULL; - } #endif _PyAST_Fini(interp); _PyAtExit_Fini(interp); @@ -1495,6 +1488,9 @@ init_threadstate(_PyThreadStateImpl *_tstate, _tstate->asyncio_running_loop = NULL; _tstate->asyncio_running_task = NULL; +#ifdef _Py_TIER2 + _tstate->jit_state.code_buffer = NULL; +#endif tstate->delete_later = NULL; llist_init(&_tstate->mem_free_queue); @@ -1794,6 +1790,14 @@ tstate_delete_common(PyThreadState *tstate, int release_gil) assert(tstate_impl->refcounts.values == NULL); #endif +#if _Py_TIER2 + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; + if (_tstate->jit_state.code_buffer != NULL) { + _PyObject_VirtualFree(_tstate->jit_state.code_buffer, UOP_BUFFER_SIZE); + _tstate->jit_state.code_buffer = NULL; + } +#endif + HEAD_UNLOCK(runtime); // XXX Unbind in PyThreadState_Clear(), or earlier diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 4ede31992a7dfb..7e4f1bd5c88c86 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -162,7 +162,7 @@ def dispatch_same_oparg( assert "specializing" in uop.annotations, uop.name self.out.start_line() self.emit("#if _Py_TIER2\n") - self.emit("tstate->interp->jit_state.prev_state.specialize_counter++;\n") + self.emit("((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++;\n") self.emit("#endif\n") self.emit(tkn) emit_to(self.out, tkn_iter, "SEMI") From 547f5872e020e2fd4abfa86e9d4b5162d4d3b511 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 7 Nov 2025 22:59:14 +0000 Subject: [PATCH 173/190] Update optimizer.c --- Python/optimizer.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Python/optimizer.c b/Python/optimizer.c index c38f28e224a8b6..f1d0ca811c7e00 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -711,7 +711,10 @@ _PyJit_translate_single_bytecode_to_trace( max_length--; } - RESERVE_RAW(expansion->nuops + needs_guard_ip + 3 + (!OPCODE_HAS_NO_SAVE_IP(opcode)), "uop and various checks"); + // _GUARD_IP leads to an exit. + max_length -= needs_guard_ip; + + RESERVE_RAW(expansion->nuops + needs_guard_ip + 2 + (!OPCODE_HAS_NO_SAVE_IP(opcode)), "uop and various checks"); ADD_TO_TRACE(_CHECK_VALIDITY, 0, 0, target); From a55d766aceb1dd3da982b182d4c5bef00f72f561 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 7 Nov 2025 23:05:59 +0000 Subject: [PATCH 174/190] Update generated_cases.c.h --- Python/generated_cases.c.h | 9 --------- 1 file changed, 9 deletions(-) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index a45138e6e6c5b9..284d90869aea80 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -7703,15 +7703,6 @@ if (!IS_JIT_TRACING() && backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD_JIT && next_instr->op.code != ENTER_EXECUTOR) { - _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; - if (_tstate->jit_state.code_buffer == NULL) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _tstate->jit_state.code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (_tstate->jit_state.code_buffer == NULL) { - DISPATCH(); - } - } _Py_CODEUNIT *insert_exec_at = this_instr; while (oparg > 255) { oparg >>= 8; From 278bbe62f289ff1d80dbe7d962d68395e1af3888 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 8 Nov 2025 01:09:25 +0000 Subject: [PATCH 175/190] Support __init__ in the optimizer --- Python/optimizer_analysis.c | 19 ++++++++++++++----- Python/optimizer_bytecodes.c | 20 +++++++++++++++++--- Python/optimizer_cases.c.h | 23 ++++++++++++++++++++--- Python/optimizer_symbols.c | 1 - 4 files changed, 51 insertions(+), 12 deletions(-) diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index a24906cfdce570..a1f8f40ed763f1 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -142,8 +142,10 @@ incorrect_keys(PyObject *obj, uint32_t version) #define STACK_LEVEL() ((int)(stack_pointer - ctx->frame->stack)) #define STACK_SIZE() ((int)(ctx->frame->stack_len)) +#define CURRENT_FRAME_IS_INIT_SHIM() (ctx->frame->code == ((PyCodeObject *)&_Py_InitCleanup)) + #define WITHIN_STACK_BOUNDS() \ - (STACK_LEVEL() >= 0 && STACK_LEVEL() <= STACK_SIZE()) + (CURRENT_FRAME_IS_INIT_SHIM() || (STACK_LEVEL() >= 0 && STACK_LEVEL() <= STACK_SIZE())) #define GETLOCAL(idx) ((ctx->frame->locals[idx])) @@ -316,13 +318,18 @@ optimize_uops( ctx->frame = frame; _PyUOpInstruction *this_instr = NULL; + JitOptRef *stack_pointer = ctx->frame->stack_pointer; + for (int i = 0; !ctx->done; i++) { assert(i < trace_len); this_instr = &trace[i]; int oparg = this_instr->oparg; opcode = this_instr->opcode; - JitOptRef *stack_pointer = ctx->frame->stack_pointer; + + if (!CURRENT_FRAME_IS_INIT_SHIM()) { + stack_pointer = ctx->frame->stack_pointer; + } #ifdef Py_DEBUG if (get_lltrace() >= 3) { @@ -341,9 +348,11 @@ optimize_uops( Py_UNREACHABLE(); } assert(ctx->frame != NULL); - DPRINTF(3, " stack_level %d\n", STACK_LEVEL()); - ctx->frame->stack_pointer = stack_pointer; - assert(STACK_LEVEL() >= 0); + if (!CURRENT_FRAME_IS_INIT_SHIM()) { + DPRINTF(3, " stack_level %d\n", STACK_LEVEL()); + ctx->frame->stack_pointer = stack_pointer; + assert(STACK_LEVEL() >= 0); + } } if (ctx->out_of_space) { DPRINTF(3, "\n"); diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index fdc4e49aaf4445..16bb8b85a39081 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -764,8 +764,20 @@ dummy_func(void) { } op(_CREATE_INIT_FRAME, (init, self, args[oparg] -- init_frame)) { - init_frame = PyJitRef_NULL; - ctx->done = true; + _Py_UOpsAbstractFrame *old_frame = ctx->frame; + _Py_UOpsAbstractFrame *shim = frame_new(ctx, (PyCodeObject *)&_Py_InitCleanup, 0, NULL, 0); + if (shim == NULL) { + break; + } + /* Push self onto stack of shim */ + shim->stack[0] = self; + shim->stack_pointer++; + assert((int)(shim->stack_pointer - shim->stack) == 1); + ctx->frame = shim; + ctx->curr_frame_depth++; + assert((this_instr + 1)->opcode == _PUSH_FRAME); + PyCodeObject *co = get_code_with_logging((this_instr + 1)); + init_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, args-1, oparg+1)); } op(_RETURN_VALUE, (retval -- res)) { @@ -863,7 +875,9 @@ dummy_func(void) { op(_PUSH_FRAME, (new_frame -- )) { SYNC_SP(); - ctx->frame->stack_pointer = stack_pointer; + if (!CURRENT_FRAME_IS_INIT_SHIM()) { + ctx->frame->stack_pointer = stack_pointer; + } ctx->frame = (_Py_UOpsAbstractFrame *)PyJitRef_Unwrap(new_frame); ctx->curr_frame_depth++; stack_pointer = ctx->frame->stack_pointer; diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 397502cec784f0..74c824e7bc905b 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -2622,7 +2622,9 @@ new_frame = stack_pointer[-1]; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); - ctx->frame->stack_pointer = stack_pointer; + if (!CURRENT_FRAME_IS_INIT_SHIM()) { + ctx->frame->stack_pointer = stack_pointer; + } ctx->frame = (_Py_UOpsAbstractFrame *)PyJitRef_Unwrap(new_frame); ctx->curr_frame_depth++; stack_pointer = ctx->frame->stack_pointer; @@ -2768,9 +2770,24 @@ } case _CREATE_INIT_FRAME: { + JitOptRef *args; + JitOptRef self; JitOptRef init_frame; - init_frame = PyJitRef_NULL; - ctx->done = true; + args = &stack_pointer[-oparg]; + self = stack_pointer[-1 - oparg]; + _Py_UOpsAbstractFrame *old_frame = ctx->frame; + _Py_UOpsAbstractFrame *shim = frame_new(ctx, (PyCodeObject *)&_Py_InitCleanup, 0, NULL, 0); + if (shim == NULL) { + break; + } + shim->stack[0] = self; + shim->stack_pointer++; + assert((int)(shim->stack_pointer - shim->stack) == 1); + ctx->frame = shim; + ctx->curr_frame_depth++; + assert((this_instr + 1)->opcode == _PUSH_FRAME); + PyCodeObject *co = get_code_with_logging((this_instr + 1)); + init_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, args-1, oparg+1)); stack_pointer[-2 - oparg] = init_frame; stack_pointer += -1 - oparg; assert(WITHIN_STACK_BOUNDS()); diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index f16c6b0659f86b..728e24cc5b2a09 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -823,7 +823,6 @@ _Py_uop_frame_new( return NULL; } _Py_UOpsAbstractFrame *frame = &ctx->frames[ctx->curr_frame_depth]; - frame->code = co; frame->stack_len = co->co_stacksize; frame->locals_len = co->co_nlocalsplus; From f547880dc785dd6a02a33c0a789f53fd9245cf1f Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 8 Nov 2025 02:20:14 +0000 Subject: [PATCH 176/190] Fix a few perf regressions due to tracing thru optimizer --- Python/optimizer.c | 4 +++- Python/optimizer_bytecodes.c | 24 +++++++++++++++--------- Python/optimizer_cases.c.h | 22 ++++++++++++++-------- 3 files changed, 32 insertions(+), 18 deletions(-) diff --git a/Python/optimizer.c b/Python/optimizer.c index f1d0ca811c7e00..e73753ca830558 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -865,7 +865,9 @@ _PyJit_translate_single_bytecode_to_trace( operand = 0; if (frame->owner < FRAME_OWNED_BY_INTERPRETER) { - if (new_func != NULL) { + // Don't add nested code objects to the dependency. + // It causes endless re-traces. + if (new_func != NULL && !(new_code->co_flags & CO_NESTED)) { operand = (uintptr_t)new_func; DPRINTF(2, "Adding %p func to op\n", (void *)operand); _Py_BloomFilter_Add(dependencies, new_func); diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 16bb8b85a39081..a8b58dbaa1e684 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -752,8 +752,14 @@ dummy_func(void) { } op(_PY_FRAME_KW, (callable, self_or_null, args[oparg], kwnames -- new_frame)) { - new_frame = PyJitRef_NULL; - ctx->done = true; + assert((this_instr + 2)->opcode == _PUSH_FRAME); + PyCodeObject *co = get_code_with_logging((this_instr + 2)); + if (co == NULL) { + ctx->done = true; + break; + } + + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); } op(_CHECK_AND_ALLOCATE_OBJECT, (type_version/2, callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { @@ -882,16 +888,16 @@ dummy_func(void) { ctx->curr_frame_depth++; stack_pointer = ctx->frame->stack_pointer; uint64_t operand = this_instr->operand0; - if (operand == 0 || (operand & 1)) { - // It's either a code object or NULL + if (operand == 0) { ctx->done = true; break; } - PyFunctionObject *func = (PyFunctionObject *)operand; - PyCodeObject *co = (PyCodeObject *)func->func_code; - _Py_BloomFilter_Add(dependencies, co); - assert(PyFunction_Check(func)); - ctx->frame->func = func; + if (!(operand & 1)) { + PyFunctionObject *func = (PyFunctionObject *)operand; + PyCodeObject *co = (PyCodeObject *)func->func_code; + _Py_BloomFilter_Add(dependencies, co); + ctx->frame->func = func; + } } op(_UNPACK_SEQUENCE, (seq -- values[oparg], top[0])) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 74c824e7bc905b..879758e3e4fb6d 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -2629,15 +2629,16 @@ ctx->curr_frame_depth++; stack_pointer = ctx->frame->stack_pointer; uint64_t operand = this_instr->operand0; - if (operand == 0 || (operand & 1)) { + if (operand == 0) { ctx->done = true; break; } - PyFunctionObject *func = (PyFunctionObject *)operand; - PyCodeObject *co = (PyCodeObject *)func->func_code; - _Py_BloomFilter_Add(dependencies, co); - assert(PyFunction_Check(func)); - ctx->frame->func = func; + if (!(operand & 1)) { + PyFunctionObject *func = (PyFunctionObject *)operand; + PyCodeObject *co = (PyCodeObject *)func->func_code; + _Py_BloomFilter_Add(dependencies, co); + ctx->frame->func = func; + } break; } @@ -2972,8 +2973,13 @@ case _PY_FRAME_KW: { JitOptRef new_frame; - new_frame = PyJitRef_NULL; - ctx->done = true; + assert((this_instr + 2)->opcode == _PUSH_FRAME); + PyCodeObject *co = get_code_with_logging((this_instr + 2)); + if (co == NULL) { + ctx->done = true; + break; + } + new_frame = PyJitRef_Wrap((JitOptSymbol *)frame_new(ctx, co, 0, NULL, 0)); stack_pointer[-3 - oparg] = new_frame; stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); From 7e2bc1d53befeb98616d7f9df34a51d1b3eb02ba Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Sat, 8 Nov 2025 16:49:31 +0000 Subject: [PATCH 177/190] Some fixups --- Include/internal/pycore_optimizer.h | 4 +--- Python/optimizer.c | 6 ++++-- Python/optimizer_bytecodes.c | 15 ++++++++++++++- Python/optimizer_cases.c.h | 14 +++++++++++++- 4 files changed, 32 insertions(+), 7 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index eea5608621e9e7..e4219ebc86ae29 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -88,8 +88,6 @@ PyAPI_FUNC(void) _Py_Executors_InvalidateCold(PyInterpreterState *interp); // This value is arbitrary and was not optimized. #define JIT_CLEANUP_THRESHOLD 1000 -#define TRACE_STACK_SIZE 5 - int _Py_uop_analyze_and_optimize( PyFunctionObject *func, _PyUOpInstruction *trace, int trace_len, int curr_stackentries, @@ -125,7 +123,7 @@ static inline uint16_t uop_get_error_target(const _PyUOpInstruction *inst) #define TY_ARENA_SIZE (UOP_MAX_TRACE_LENGTH * 5) // Need extras for root frame and for overflow frame (see TRACE_STACK_PUSH()) -#define MAX_ABSTRACT_FRAME_DEPTH (TRACE_STACK_SIZE + 2) +#define MAX_ABSTRACT_FRAME_DEPTH (16) // The maximum number of side exits that we can take before requiring forward // progress (and inserting a new ENTER_EXECUTOR instruction). In practice, this diff --git a/Python/optimizer.c b/Python/optimizer.c index e73753ca830558..5a7de32f4607d9 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -616,7 +616,7 @@ _PyJit_translate_single_bytecode_to_trace( bool needs_guard_ip = OPCODE_HAS_NEEDS_GUARD_IP(opcode); if (has_dynamic_jump_taken && !needs_guard_ip) { - DPRINTF(2, "Unsupported: dynamic jump taken\n"); + DPRINTF(2, "Unsupported: dynamic jump taken %s\n", _PyOpcode_OpName[opcode]); goto unsupported; } DPRINTF(2, "%p %d: %s(%d) %d %d\n", old_code, target, _PyOpcode_OpName[opcode], oparg, needs_guard_ip, old_stack_level); @@ -749,6 +749,8 @@ _PyJit_translate_single_bytecode_to_trace( if ((next_instr != _tstate->jit_state.initial_state.close_loop_instr) && (next_instr != _tstate->jit_state.initial_state.start_instr) && _tstate->jit_state.prev_state.code_curr_size > 5 && + // For side exits, we don't want to terminate them early. + _tstate->jit_state.initial_state.exit == NULL && // These are coroutines, and we want to unroll those usually. opcode != JUMP_BACKWARD_NO_INTERRUPT) { // We encountered a JUMP_BACKWARD but not to the top of our own loop. @@ -867,7 +869,7 @@ _PyJit_translate_single_bytecode_to_trace( if (frame->owner < FRAME_OWNED_BY_INTERPRETER) { // Don't add nested code objects to the dependency. // It causes endless re-traces. - if (new_func != NULL && !(new_code->co_flags & CO_NESTED)) { + if (new_func != NULL && !Py_IsNone((PyObject*)new_func) && !(new_code->co_flags & CO_NESTED)) { operand = (uintptr_t)new_func; DPRINTF(2, "Adding %p func to op\n", (void *)operand); _Py_BloomFilter_Add(dependencies, new_func); diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index a8b58dbaa1e684..eca5c0b69696a5 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -770,7 +770,7 @@ dummy_func(void) { } op(_CREATE_INIT_FRAME, (init, self, args[oparg] -- init_frame)) { - _Py_UOpsAbstractFrame *old_frame = ctx->frame; + ctx->frame->stack_pointer = stack_pointer - oparg - 2; _Py_UOpsAbstractFrame *shim = frame_new(ctx, (PyCodeObject *)&_Py_InitCleanup, 0, NULL, 0); if (shim == NULL) { break; @@ -799,6 +799,13 @@ dummy_func(void) { } _Py_BloomFilter_Add(dependencies, returning_code); int returning_stacklevel = this_instr->operand1; + if (ctx->curr_frame_depth >= 2) { + PyCodeObject *expected_code = ctx->frames[ctx->curr_frame_depth - 2].code; + if (expected_code == returning_code) { + assert((this_instr + 1)->opcode == _GUARD_IP_RETURN_VALUE); + REPLACE_OP((this_instr + 1), _NOP, 0, 0); + } + } if (frame_pop(ctx, returning_code, returning_stacklevel)) { break; } @@ -898,6 +905,12 @@ dummy_func(void) { _Py_BloomFilter_Add(dependencies, co); ctx->frame->func = func; } + // Fixed calls don't need IP guards. + if ((this_instr-1)->opcode == _SAVE_RETURN_OFFSET || + (this_instr-1)->opcode == _CREATE_INIT_FRAME) { + assert((this_instr+1)->opcode == _GUARD_IP__PUSH_FRAME); + REPLACE_OP(this_instr+1, _NOP, 0, 0); + } } op(_UNPACK_SEQUENCE, (seq -- values[oparg], top[0])) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 879758e3e4fb6d..765303cdeb9909 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1124,6 +1124,13 @@ } _Py_BloomFilter_Add(dependencies, returning_code); int returning_stacklevel = this_instr->operand1; + if (ctx->curr_frame_depth >= 2) { + PyCodeObject *expected_code = ctx->frames[ctx->curr_frame_depth - 2].code; + if (expected_code == returning_code) { + assert((this_instr + 1)->opcode == _GUARD_IP_RETURN_VALUE); + REPLACE_OP((this_instr + 1), _NOP, 0, 0); + } + } if (frame_pop(ctx, returning_code, returning_stacklevel)) { break; } @@ -2639,6 +2646,11 @@ _Py_BloomFilter_Add(dependencies, co); ctx->frame->func = func; } + if ((this_instr-1)->opcode == _SAVE_RETURN_OFFSET || + (this_instr-1)->opcode == _CREATE_INIT_FRAME) { + assert((this_instr+1)->opcode == _GUARD_IP__PUSH_FRAME); + REPLACE_OP(this_instr+1, _NOP, 0, 0); + } break; } @@ -2776,7 +2788,7 @@ JitOptRef init_frame; args = &stack_pointer[-oparg]; self = stack_pointer[-1 - oparg]; - _Py_UOpsAbstractFrame *old_frame = ctx->frame; + ctx->frame->stack_pointer = stack_pointer - oparg - 2; _Py_UOpsAbstractFrame *shim = frame_new(ctx, (PyCodeObject *)&_Py_InitCleanup, 0, NULL, 0); if (shim == NULL) { break; From fa3e285d064c2e6748766e04ec6bcb71d0b68c65 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 10 Nov 2025 17:02:27 +0000 Subject: [PATCH 178/190] Clean up labels --- Include/internal/pycore_opcode_metadata.h | 4 +- Include/internal/pycore_optimizer.h | 2 +- Python/bytecodes.c | 62 +++++++-------- Python/ceval.c | 8 +- Python/generated_cases.c.h | 91 +++++++++++------------ Python/opcode_targets.h | 1 + Python/optimizer.c | 27 ++++--- 7 files changed, 91 insertions(+), 104 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 825321739c40c5..548627dc7982ec 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1150,7 +1150,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [END_ASYNC_FOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [END_FOR] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG }, [END_SEND] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_PURE_FLAG }, - [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [FORMAT_SIMPLE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1189,7 +1189,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, - [INTERPRETER_EXIT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [INTERPRETER_EXIT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [IS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_BACKWARD_JIT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index e4219ebc86ae29..653285a2c6b79b 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -362,7 +362,7 @@ PyAPI_FUNC(int) _PyDumpExecutors(FILE *out); extern void _Py_ClearExecutorDeletionList(PyInterpreterState *interp); #endif -int _PyJit_translate_single_bytecode_to_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr); +int _PyJit_translate_single_bytecode_to_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *next_instr, bool stop_tracing); int _PyJit_TryInitializeTracing(PyThreadState *tstate, _PyInterpreterFrame *frame, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 283a18424715b6..5bfd9ea8a5a030 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1218,33 +1218,19 @@ dummy_func( tstate->current_frame = frame->previous; assert(!_PyErr_Occurred(tstate)); PyObject *result = PyStackRef_AsPyObjectSteal(retval); - if (IS_JIT_TRACING()) { -#if _Py_TIER2 - _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL); - LEAVE_TRACING(); - int err = bail_tracing_and_jit(tstate, frame); - if (err < 0) { - Py_DECREF(result); - ERROR_IF(true); - } - return result; -#endif - } - else { #if !_Py_TAIL_CALL_INTERP - assert(frame == &entry.frame); + assert(frame == &entry.frame); #endif #ifdef _Py_TIER2 - _PyStackRef executor = frame->localsplus[0]; - assert(tstate->current_executor == NULL); - if (!PyStackRef_IsNull(executor)) { - tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor); - PyStackRef_CLOSE(executor); - } -#endif - LLTRACE_RESUME_FRAME(); - return result; + _PyStackRef executor = frame->localsplus[0]; + assert(tstate->current_executor == NULL); + if (!PyStackRef_IsNull(executor)) { + tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor); + PyStackRef_CLOSE(executor); } +#endif + LLTRACE_RESUME_FRAME(); + return result; } // The stack effect here is a bit misleading. @@ -3028,10 +3014,8 @@ dummy_func( tier1 inst(ENTER_EXECUTOR, (--)) { #ifdef _Py_TIER2 if (IS_JIT_TRACING()) { - _PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr); - LEAVE_TRACING(); - int err = bail_tracing_and_jit(tstate, frame); - ERROR_IF(err < 0); + next_instr = this_instr; + goto stop_tracing; } PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; @@ -5657,7 +5641,10 @@ dummy_func( #if _Py_TIER2 assert(IS_JIT_TRACING()); int opcode = next_instr->op.code; - int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr); + bool stop_tracing = (opcode == WITH_EXCEPT_START || + opcode == RERAISE || opcode == CLEANUP_THROW || + opcode == PUSH_EXC_INFO || opcode == INTERPRETER_EXIT); + int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing); if (full) { LEAVE_TRACING(); int err = bail_tracing_and_jit(tstate, frame); @@ -5677,9 +5664,9 @@ dummy_func( _tstate->jit_state.prev_state.instr = next_instr; } _tstate->jit_state.prev_state.specialize_counter = 0; - PyCodeObject *prev_code = (PyCodeObject *)Py_NewRef(PyStackRef_AsPyObjectBorrow(frame->f_executable)); - if (_tstate->jit_state.prev_state.instr_code != prev_code) { - Py_SETREF(_tstate->jit_state.prev_state.instr_code, prev_code); + PyObject *prev_code = PyStackRef_AsPyObjectBorrow(frame->f_executable); + if (_tstate->jit_state.prev_state.instr_code != (PyCodeObject *)prev_code) { + Py_SETREF(_tstate->jit_state.prev_state.instr_code, (PyCodeObject*)Py_NewRef((prev_code))); } _tstate->jit_state.prev_state.instr_frame = frame; @@ -5691,6 +5678,19 @@ dummy_func( #endif } + label(stop_tracing) { +#if _Py_TIER2 + assert(IS_JIT_TRACING()); + _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL, true); + LEAVE_TRACING(); + int err = bail_tracing_and_jit(tstate, frame); + ERROR_IF(err < 0); + DISPATCH_GOTO_NON_TRACING(); +#else + Py_FatalError("JIT label executed in non-jit build."); +#endif + } + // END BYTECODES // diff --git a/Python/ceval.c b/Python/ceval.c index e6424503e88988..aea353603b545f 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1017,13 +1017,7 @@ bail_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame) // Likewise, we hold a strong reference to the executor containing this exit, so the exit is guaranteed // to be valid to access. if (err <= 0) { - // Some opcodes will forever be unchanged. Don't ever bother specializing for them ever again. - if (_tstate->jit_state.prev_state.instr->op.code == INTERPRETER_EXIT) { - exit->temperature = initial_unreachable_backoff_counter(); - } - else { - exit->temperature = restart_backoff_counter(exit->temperature); - } + exit->temperature = restart_backoff_counter(exit->temperature); } else { exit->temperature = initial_temperature_backoff_counter(); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 284d90869aea80..29b294f0f6e9cb 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5492,16 +5492,8 @@ opcode = ENTER_EXECUTOR; #ifdef _Py_TIER2 if (IS_JIT_TRACING()) { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr); - stack_pointer = _PyFrame_GetStackPointer(frame); - LEAVE_TRACING(); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = bail_tracing_and_jit(tstate, frame); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - JUMP_TO_LABEL(error); - } + next_instr = this_instr; + JUMP_TO_LABEL(stop_tracing); } PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; @@ -7552,46 +7544,24 @@ tstate->current_frame = frame->previous; assert(!_PyErr_Occurred(tstate)); PyObject *result = PyStackRef_AsPyObjectSteal(retval); - if (IS_JIT_TRACING()) { - #if _Py_TIER2 + #if !_Py_TAIL_CALL_INTERP + assert(frame == &entry.frame); + #endif + #ifdef _Py_TIER2 + _PyStackRef executor = frame->localsplus[0]; + assert(tstate->current_executor == NULL); + if (!PyStackRef_IsNull(executor)) { + tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); - _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL); - stack_pointer = _PyFrame_GetStackPointer(frame); - LEAVE_TRACING(); - _PyFrame_SetStackPointer(frame, stack_pointer); - int err = bail_tracing_and_jit(tstate, frame); + PyStackRef_CLOSE(executor); stack_pointer = _PyFrame_GetStackPointer(frame); - if (err < 0) { - _PyFrame_SetStackPointer(frame, stack_pointer); - Py_DECREF(result); - stack_pointer = _PyFrame_GetStackPointer(frame); - JUMP_TO_LABEL(error); - } - return result; - #endif - } - else { - #if !_Py_TAIL_CALL_INTERP - assert(frame == &entry.frame); - #endif - #ifdef _Py_TIER2 - _PyStackRef executor = frame->localsplus[0]; - assert(tstate->current_executor == NULL); - if (!PyStackRef_IsNull(executor)) { - tstate->current_executor = PyStackRef_AsPyObjectBorrow(executor); - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(executor); - stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer += 1; - } - #endif - LLTRACE_RESUME_FRAME(); - return result; + stack_pointer += 1; } + #endif + LLTRACE_RESUME_FRAME(); + return result; } TARGET(IS_OP) { @@ -12339,8 +12309,11 @@ JUMP_TO_LABEL(error); #if _Py_TIER2 assert(IS_JIT_TRACING()); int opcode = next_instr->op.code; + bool stop_tracing = (opcode == WITH_EXCEPT_START || + opcode == RERAISE || opcode == CLEANUP_THROW || + opcode == PUSH_EXC_INFO || opcode == INTERPRETER_EXIT); _PyFrame_SetStackPointer(frame, stack_pointer); - int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr); + int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing); stack_pointer = _PyFrame_GetStackPointer(frame); if (full) { LEAVE_TRACING(); @@ -12363,10 +12336,10 @@ JUMP_TO_LABEL(error); _tstate->jit_state.prev_state.instr = next_instr; } _tstate->jit_state.prev_state.specialize_counter = 0; - PyCodeObject *prev_code = (PyCodeObject *)Py_NewRef(PyStackRef_AsPyObjectBorrow(frame->f_executable)); - if (_tstate->jit_state.prev_state.instr_code != prev_code) { + PyObject *prev_code = PyStackRef_AsPyObjectBorrow(frame->f_executable); + if (_tstate->jit_state.prev_state.instr_code != (PyCodeObject *)prev_code) { _PyFrame_SetStackPointer(frame, stack_pointer); - Py_SETREF(_tstate->jit_state.prev_state.instr_code, prev_code); + Py_SETREF(_tstate->jit_state.prev_state.instr_code, (PyCodeObject*)Py_NewRef((prev_code))); stack_pointer = _PyFrame_GetStackPointer(frame); } _tstate->jit_state.prev_state.instr_frame = frame; @@ -12378,5 +12351,25 @@ JUMP_TO_LABEL(error); #endif } + LABEL(stop_tracing) + { + #if _Py_TIER2 + assert(IS_JIT_TRACING()); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL, true); + stack_pointer = _PyFrame_GetStackPointer(frame); + LEAVE_TRACING(); + _PyFrame_SetStackPointer(frame, stack_pointer); + int err = bail_tracing_and_jit(tstate, frame); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (err < 0) { + JUMP_TO_LABEL(error); + } + DISPATCH_GOTO_NON_TRACING(); + #else + Py_FatalError("JIT label executed in non-jit build."); + #endif + } + /* END LABELS */ #undef TIER_ONE diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 288e45ce8bc843..1b9196503b570b 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -529,6 +529,7 @@ Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exception_unwind(TAIL_CALL_PARAM Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_exit_unwind(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_start_frame(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_record_previous_inst(TAIL_CALL_PARAMS); +Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_stop_tracing(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP(TAIL_CALL_PARAMS); Py_PRESERVE_NONE_CC static PyObject *_TAIL_CALL_BINARY_OP_ADD_FLOAT(TAIL_CALL_PARAMS); diff --git a/Python/optimizer.c b/Python/optimizer.c index 5a7de32f4607d9..64e57f26094d4f 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -552,7 +552,8 @@ int _PyJit_translate_single_bytecode_to_trace( PyThreadState *tstate, _PyInterpreterFrame *frame, - _Py_CODEUNIT *next_instr) + _Py_CODEUNIT *next_instr, + bool stop_tracing) { #ifdef Py_DEBUG @@ -571,11 +572,6 @@ _PyJit_translate_single_bytecode_to_trace( _PyUOpInstruction *trace = _tstate->jit_state.code_buffer; int max_length = _tstate->jit_state.prev_state.code_max_size; - int is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL); - if (is_sys_tracing) { - goto full; - } - _Py_CODEUNIT *this_instr = _tstate->jit_state.prev_state.instr; _Py_CODEUNIT *target_instr = this_instr; uint32_t target = 0; @@ -619,6 +615,17 @@ _PyJit_translate_single_bytecode_to_trace( DPRINTF(2, "Unsupported: dynamic jump taken %s\n", _PyOpcode_OpName[opcode]); goto unsupported; } + + int is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL); + if (is_sys_tracing) { + goto full; + } + + if (stop_tracing) { + ADD_TO_TRACE(_DEOPT, 0, 0, target); + goto done; + } + DPRINTF(2, "%p %d: %s(%d) %d %d\n", old_code, target, _PyOpcode_OpName[opcode], oparg, needs_guard_ip, old_stack_level); #ifdef Py_DEBUG @@ -655,11 +662,6 @@ _PyJit_translate_single_bytecode_to_trace( // TODO (gh-140277): The constituent use one extra stack slot. So we need to check for headroom. if (opcode == BINARY_OP_SUBSCR_GETITEM && old_stack_level + 1 > old_code->co_stacksize) { - goto unsupported; - } - - if (opcode == WITH_EXCEPT_START || opcode == RERAISE || opcode == CLEANUP_THROW || opcode == PUSH_EXC_INFO) { - DPRINTF(2, "Unsupported: strange control-flow\n"); unsupported: { // Rewind to previous instruction and replace with _EXIT_TRACE. @@ -773,9 +775,6 @@ _PyJit_translate_single_bytecode_to_trace( * start with RESUME_CHECK */ ADD_TO_TRACE(_TIER2_RESUME_CHECK, 0, 0, target); break; - case INTERPRETER_EXIT: - ADD_TO_TRACE(_DEOPT, 0, 0, target); - goto done;; default: { const struct opcode_macro_expansion *expansion = &_PyOpcode_macro_expansion[opcode]; From 251e19e3a0593325d4ff13b7265660f14354ef44 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 10 Nov 2025 17:49:35 +0000 Subject: [PATCH 179/190] fix TC --- Python/bytecodes.c | 1 + Python/generated_cases.c.h | 1 + 2 files changed, 2 insertions(+) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 5bfd9ea8a5a030..0469e0c96080ac 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5681,6 +5681,7 @@ dummy_func( label(stop_tracing) { #if _Py_TIER2 assert(IS_JIT_TRACING()); + int opcode = next_instr->op.code; _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL, true); LEAVE_TRACING(); int err = bail_tracing_and_jit(tstate, frame); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 29b294f0f6e9cb..f2f562a134cf10 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -12355,6 +12355,7 @@ JUMP_TO_LABEL(error); { #if _Py_TIER2 assert(IS_JIT_TRACING()); + int opcode = next_instr->op.code; _PyFrame_SetStackPointer(frame, stack_pointer); _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL, true); stack_pointer = _PyFrame_GetStackPointer(frame); From 08ec6005efa1120d1a9bcd6b92e540d47fb4c1fe Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 10 Nov 2025 23:23:50 +0000 Subject: [PATCH 180/190] Remove specialize_counter --- Include/internal/pycore_backoff.h | 16 +++---- Include/internal/pycore_tstate.h | 1 - Python/bytecodes.c | 8 ++-- Python/ceval_macros.h | 4 +- Python/executor_cases.c.h | 4 +- Python/generated_cases.c.h | 49 ++-------------------- Python/optimizer.c | 26 +++++++----- Python/optimizer_analysis.c | 5 ++- Python/optimizer_bytecodes.c | 1 - Python/optimizer_cases.c.h | 1 - Python/specialize.c | 8 +++- Tools/cases_generator/analyzer.py | 1 + Tools/cases_generator/generators_common.py | 21 ---------- 13 files changed, 44 insertions(+), 101 deletions(-) diff --git a/Include/internal/pycore_backoff.h b/Include/internal/pycore_backoff.h index 3762257f3173d9..71066f1bd9f19b 100644 --- a/Include/internal/pycore_backoff.h +++ b/Include/internal/pycore_backoff.h @@ -95,6 +95,14 @@ backoff_counter_triggers(_Py_BackoffCounter counter) return counter.value_and_backoff < UNREACHABLE_BACKOFF; } +static inline _Py_BackoffCounter +trigger_backoff_counter(void) +{ + _Py_BackoffCounter result; + result.value_and_backoff = 0; + return result; +} + // Initial JUMP_BACKWARD counter. // Must be larger than ADAPTIVE_COOLDOWN_VALUE, otherwise when JIT code is // invalidated we may construct a new trace before the bytecode has properly @@ -134,14 +142,6 @@ initial_unreachable_backoff_counter(void) return make_backoff_counter(0, UNREACHABLE_BACKOFF); } -// Required to not get stuck in infinite specialization loops due to specialization failure. -// We use 2 here as there are a few scenarios: -// 1. Freshly specialized from unspecialized, in which case the counter will be 1. -// 2. Re-specialized from deopt, in which case the counter will be 1. -// 3. Deopt -> Specialize -> Deopt -> Specialize, in which case the counter will be 2. -// We do not want the 3rd case. -#define MAX_SPECIALIZATION_TRIES 2 - #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index 04041b273d7560..d2407df43c1a4e 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -41,7 +41,6 @@ typedef struct _PyJitTracerPreviousState { int code_curr_size; int instr_oparg; int instr_stacklevel; - int specialize_counter; _Py_CODEUNIT *instr; PyCodeObject *instr_code; // Strong struct _PyInterpreterFrame *instr_frame; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 0469e0c96080ac..c2d4725ccd458a 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5249,7 +5249,7 @@ dummy_func( ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame)) + exit->target; OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - if (frame->lltrace >= 2) { + if (frame->lltrace >= 3) { printf("SIDE EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); printf(", exit %tu, temp %d, target %d -> %s, is_control_flow %d]\n", @@ -5267,7 +5267,7 @@ dummy_func( _PyExitData *exit = (_PyExitData *)exit_p; _Py_CODEUNIT *target = frame->instr_ptr; OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - if (frame->lltrace >= 2) { + if (frame->lltrace >= 3) { printf("DYNAMIC EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); printf(", exit %tu, temp %d, target %d -> %s]\n", @@ -5663,7 +5663,6 @@ dummy_func( else { _tstate->jit_state.prev_state.instr = next_instr; } - _tstate->jit_state.prev_state.specialize_counter = 0; PyObject *prev_code = PyStackRef_AsPyObjectBorrow(frame->f_executable); if (_tstate->jit_state.prev_state.instr_code != (PyCodeObject *)prev_code) { Py_SETREF(_tstate->jit_state.prev_state.instr_code, (PyCodeObject*)Py_NewRef((prev_code))); @@ -5672,6 +5671,9 @@ dummy_func( _tstate->jit_state.prev_state.instr_frame = frame; _tstate->jit_state.prev_state.instr_oparg = oparg; _tstate->jit_state.prev_state.instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + (&next_instr[1])->counter = trigger_backoff_counter(); + } DISPATCH_GOTO_NON_TRACING(); #else Py_FatalError("JIT label executed in non-jit build."); diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index c1062e9d091d3e..a812c234cb2e1b 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -134,14 +134,12 @@ #if (_Py_TAIL_CALL_INTERP || USE_COMPUTED_GOTOS) && _Py_TIER2 # define IS_JIT_TRACING() (DISPATCH_TABLE_VAR == TRACING_DISPATCH_TABLE) -# define IS_JIT_TRACING_MAKING_PROGRESS() (IS_JIT_TRACING() && ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter < MAX_SPECIALIZATION_TRIES) # define ENTER_TRACING() \ DISPATCH_TABLE_VAR = TRACING_DISPATCH_TABLE; # define LEAVE_TRACING() \ DISPATCH_TABLE_VAR = DISPATCH_TABLE; #else # define IS_JIT_TRACING() (0) -# define IS_JIT_TRACING_MAKING_PROGRESS() (0) # define ENTER_TRACING() # define LEAVE_TRACING() #endif @@ -308,7 +306,7 @@ GETITEM(PyObject *v, Py_ssize_t i) { * which is always an integral type. */ // Force re-specialization when tracing a side exit to get good side exits. #define ADAPTIVE_COUNTER_TRIGGERS(COUNTER) \ - backoff_counter_triggers(forge_backoff_counter((COUNTER))) || IS_JIT_TRACING_MAKING_PROGRESS() + backoff_counter_triggers(forge_backoff_counter((COUNTER))) #define ADVANCE_ADAPTIVE_COUNTER(COUNTER) \ do { \ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 84ce8f862a9d2c..7ba2e9d0d92999 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7114,7 +7114,7 @@ ? _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS_PTR : _PyFrame_GetBytecode(frame)) + exit->target; OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - if (frame->lltrace >= 2) { + if (frame->lltrace >= 3) { _PyFrame_SetStackPointer(frame, stack_pointer); printf("SIDE EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); @@ -7136,7 +7136,7 @@ _PyExitData *exit = (_PyExitData *)exit_p; _Py_CODEUNIT *target = frame->instr_ptr; OPT_HIST(trace_uop_execution_counter, trace_run_length_hist); - if (frame->lltrace >= 2) { + if (frame->lltrace >= 3) { _PyFrame_SetStackPointer(frame, stack_pointer); printf("DYNAMIC EXIT: [UOp "); _PyUOpPrint(&next_uop[-1]); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index f2f562a134cf10..241d2fd71e6692 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -44,9 +44,6 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); stack_pointer = _PyFrame_GetStackPointer(frame); - #if _Py_TIER2 - ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++; - #endif DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(BINARY_OP); @@ -1538,9 +1535,6 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_Call(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); stack_pointer = _PyFrame_GetStackPointer(frame); - #if _Py_TIER2 - ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++; - #endif DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CALL); @@ -2831,9 +2825,6 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_CallKw(callable, next_instr, oparg + !PyStackRef_IsNull(self_or_null)); stack_pointer = _PyFrame_GetStackPointer(frame); - #if _Py_TIER2 - ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++; - #endif DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CALL_KW); @@ -4677,9 +4668,6 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_CompareOp(left, right, next_instr, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - #if _Py_TIER2 - ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++; - #endif DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(COMPARE_OP); @@ -4925,9 +4913,6 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_ContainsOp(right, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - #if _Py_TIER2 - ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++; - #endif DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(CONTAINS_OP); @@ -5656,9 +5641,6 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_ForIter(iter, null_or_index, next_instr, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - #if _Py_TIER2 - ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++; - #endif DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(FOR_ITER); @@ -7616,9 +7598,6 @@ uint8_t desired = tstate->interp->jit ? JUMP_BACKWARD_JIT : JUMP_BACKWARD_NO_JIT; FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, desired); next_instr = this_instr; - #if _Py_TIER2 - ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++; - #endif DISPATCH_SAME_OPARG(); } #endif @@ -7841,9 +7820,6 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_LoadAttr(owner, next_instr, name); stack_pointer = _PyFrame_GetStackPointer(frame); - #if _Py_TIER2 - ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++; - #endif DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(LOAD_ATTR); @@ -9149,9 +9125,6 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); stack_pointer = _PyFrame_GetStackPointer(frame); - #if _Py_TIER2 - ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++; - #endif DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(LOAD_GLOBAL); @@ -9472,9 +9445,6 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_LoadSuperAttr(global_super_st, class_st, next_instr, load_method); stack_pointer = _PyFrame_GetStackPointer(frame); - #if _Py_TIER2 - ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++; - #endif DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR); @@ -10460,9 +10430,6 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_Send(receiver, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - #if _Py_TIER2 - ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++; - #endif DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(SEND); @@ -10762,9 +10729,6 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_StoreAttr(owner, next_instr, name); stack_pointer = _PyFrame_GetStackPointer(frame); - #if _Py_TIER2 - ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++; - #endif DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(STORE_ATTR); @@ -11262,9 +11226,6 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_StoreSubscr(container, sub, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - #if _Py_TIER2 - ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++; - #endif DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(STORE_SUBSCR); @@ -11474,9 +11435,6 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_ToBool(value, next_instr); stack_pointer = _PyFrame_GetStackPointer(frame); - #if _Py_TIER2 - ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++; - #endif DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(TO_BOOL); @@ -11857,9 +11815,6 @@ _PyFrame_SetStackPointer(frame, stack_pointer); _Py_Specialize_UnpackSequence(seq, next_instr, oparg); stack_pointer = _PyFrame_GetStackPointer(frame); - #if _Py_TIER2 - ((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++; - #endif DISPATCH_SAME_OPARG(); } OPCODE_DEFERRED_INC(UNPACK_SEQUENCE); @@ -12335,7 +12290,6 @@ JUMP_TO_LABEL(error); else { _tstate->jit_state.prev_state.instr = next_instr; } - _tstate->jit_state.prev_state.specialize_counter = 0; PyObject *prev_code = PyStackRef_AsPyObjectBorrow(frame->f_executable); if (_tstate->jit_state.prev_state.instr_code != (PyCodeObject *)prev_code) { _PyFrame_SetStackPointer(frame, stack_pointer); @@ -12345,6 +12299,9 @@ JUMP_TO_LABEL(error); _tstate->jit_state.prev_state.instr_frame = frame; _tstate->jit_state.prev_state.instr_oparg = oparg; _tstate->jit_state.prev_state.instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + (&next_instr[1])->counter = trigger_backoff_counter(); + } DISPATCH_GOTO_NON_TRACING(); #else Py_FatalError("JIT label executed in non-jit build."); diff --git a/Python/optimizer.c b/Python/optimizer.c index 64e57f26094d4f..c4da65c19ff961 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -129,6 +129,7 @@ _PyOptimizer_Optimize( if (_tstate->jit_state.initial_state.func == NULL) { return 0; } + assert(_tstate->jit_state.initial_state.func != NULL); interp->compiling = true; // The first executor in a chain and the MAX_CHAIN_DEPTH'th executor *must* // make progress in order to avoid infinite loops or excessively-long @@ -584,10 +585,6 @@ _PyJit_translate_single_bytecode_to_trace( // We must point to the first EXTENDED_ARG when deopting. int oparg = _tstate->jit_state.prev_state.instr_oparg; int opcode = this_instr->op.code; - // Failed specialization many times. Deopt! - if (_tstate->jit_state.prev_state.specialize_counter >= MAX_SPECIALIZATION_TRIES) { - opcode = _PyOpcode_Deopt[opcode]; - } int rewind_oparg = oparg; while (rewind_oparg > 255) { rewind_oparg >>= 8; @@ -736,6 +733,7 @@ _PyJit_translate_single_bytecode_to_trace( _Py_CODEUNIT *computed_jump_instr = computed_next_instr_without_modifiers + oparg; assert(next_instr == computed_next_instr || next_instr == computed_jump_instr); int jump_happened = computed_jump_instr == next_instr; + assert(jump_happened == (target_instr[1].cache & 1)); uint32_t uopcode = BRANCH_TO_GUARD[opcode - POP_JUMP_IF_FALSE][jump_happened]; ADD_TO_TRACE(uopcode, 0, 0, INSTR_IP(jump_happened ? computed_next_instr : computed_jump_instr, old_code)); break; @@ -974,7 +972,10 @@ _PyJit_TryInitializeTracing( return 0; } } - + PyObject *func = PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + if (func == NULL) { + return 0; + } PyCodeObject *code = _PyFrame_GetCode(frame); #ifdef Py_DEBUG char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); @@ -999,13 +1000,12 @@ _PyJit_TryInitializeTracing( _tstate->jit_state.initial_state.start_instr = start_instr; _tstate->jit_state.initial_state.close_loop_instr = close_loop_instr; _tstate->jit_state.initial_state.code = (PyCodeObject *)Py_NewRef(code); - _tstate->jit_state.initial_state.func = (PyFunctionObject *)Py_XNewRef(PyStackRef_AsPyObjectBorrow(frame->f_funcobj)); + _tstate->jit_state.initial_state.func = (PyFunctionObject *)Py_NewRef(func); _tstate->jit_state.initial_state.exit = exit; _tstate->jit_state.initial_state.stack_depth = curr_stackdepth; _tstate->jit_state.initial_state.chain_depth = chain_depth; _tstate->jit_state.prev_state.instr_frame = frame; _tstate->jit_state.prev_state.dependencies_still_valid = true; - _tstate->jit_state.prev_state.specialize_counter = 0; _tstate->jit_state.prev_state.instr_code = (PyCodeObject *)Py_NewRef(_PyFrame_GetCode(frame)); _tstate->jit_state.prev_state.instr = curr_instr; _tstate->jit_state.prev_state.instr_frame = frame; @@ -1014,7 +1014,10 @@ _PyJit_TryInitializeTracing( _tstate->jit_state.prev_state.instr_is_super = false; assert(curr_instr->op.code == JUMP_BACKWARD_JIT || (exit != NULL)); _tstate->jit_state.initial_state.jump_backward_instr = curr_instr; - assert(curr_instr->op.code == JUMP_BACKWARD_JIT || (exit != NULL)); + + if (_PyOpcode_Caches[_PyOpcode_Deopt[close_loop_instr->op.code]]) { + close_loop_instr[1].counter = trigger_backoff_counter(); + } _Py_BloomFilter_Init(&_tstate->jit_state.prev_state.dependencies); return 1; } @@ -1366,9 +1369,10 @@ uop_optimize( assert(length < UOP_MAX_TRACE_LENGTH); OPT_STAT_INC(traces_created); if (!is_noopt) { - length = _Py_uop_analyze_and_optimize(_tstate->jit_state.initial_state.func, buffer, - length, - curr_stackentries, &new_dependencies); + length = _Py_uop_analyze_and_optimize( + _tstate->jit_state.initial_state.func, + buffer,length, + curr_stackentries, &new_dependencies); if (length <= 0) { return length; } diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index a1f8f40ed763f1..ded542ecc42c6d 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -360,9 +360,10 @@ optimize_uops( } if (ctx->contradiction) { // Attempted to push a "bottom" (contradiction) symbol onto the stack. - // This means that the abstract interpreter has hit unreachable code. + // This means that the abstract interpreter has optimized to trace + // to an unreachable estate. // We *could* generate an _EXIT_TRACE or _FATAL_ERROR here, but hitting - // bottom indicates type instability, so we are probably better off + // bottom usually indicates an optimizer bug, so we are probably better off // retrying later. DPRINTF(3, "\n"); DPRINTF(1, "Hit bottom in abstract interpreter\n"); diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index eca5c0b69696a5..b12ee02557d924 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -342,7 +342,6 @@ dummy_func(void) { int already_bool = optimize_to_bool(this_instr, ctx, value, &value); if (!already_bool) { sym_set_type(value, &PyBool_Type); - value = sym_new_truthiness(ctx, value, true); } } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 765303cdeb9909..ce5332814ced5b 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -280,7 +280,6 @@ int already_bool = optimize_to_bool(this_instr, ctx, value, &value); if (!already_bool) { sym_set_type(value, &PyBool_Type); - value = sym_new_truthiness(ctx, value, true); } stack_pointer[-1] = value; break; diff --git a/Python/specialize.c b/Python/specialize.c index a1c5dedd61563b..5b44432b8b7258 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2041,8 +2041,12 @@ specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr, bool pop = (next.op.code == POP_TOP); int oparg = instr->op.arg; if ((PyObject *)descr == list_append && oparg == 1 && pop) { - specialize(instr, CALL_LIST_APPEND); - return 0; + PyThreadState *tstate = PyThreadState_GET(); + PyObject *self = PyStackRef_AsPyObjectBorrow(tstate->current_frame->stackpointer[-2]); + if (PyList_CheckExact(self)) { + specialize(instr, CALL_LIST_APPEND); + return 0; + } } specialize(instr, CALL_METHOD_DESCRIPTOR_O); return 0; diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 532d5df14f4136..d39013db4f7fd6 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -702,6 +702,7 @@ def has_error_without_pop(op: parser.CodeDef) -> bool: "_PyJit_TryInitializeTracing", "_Py_unset_eval_breaker_bit", "_Py_set_eval_breaker_bit", + "trigger_backoff_counter", ) diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 7e4f1bd5c88c86..0b5f764ec52b45 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -129,7 +129,6 @@ def __init__(self, out: CWriter, labels: dict[str, Label], cannot_escape: bool = "DISPATCH": self.dispatch, "INSTRUCTION_SIZE": self.instruction_size, "stack_pointer": self.stack_pointer, - "DISPATCH_SAME_OPARG": self.dispatch_same_oparg, } self.out = out self.labels = labels @@ -150,26 +149,6 @@ def dispatch( self.emit(tkn) return False - def dispatch_same_oparg( - self, - tkn: Token, - tkn_iter: TokenIterator, - uop: CodeSection, - storage: Storage, - inst: Instruction | None, - ) -> bool: - assert isinstance(uop, Uop) - assert "specializing" in uop.annotations, uop.name - self.out.start_line() - self.emit("#if _Py_TIER2\n") - self.emit("((_PyThreadStateImpl *)tstate)->jit_state.prev_state.specialize_counter++;\n") - self.emit("#endif\n") - self.emit(tkn) - emit_to(self.out, tkn_iter, "SEMI") - self.emit(";\n") - self.out.start_line() - return False - def deopt_if( self, tkn: Token, From f7c26d4baa575acc0baecde44ce754092aea3fe8 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 10 Nov 2025 23:31:41 +0000 Subject: [PATCH 181/190] rename jit_state to jit_tracer_state --- Include/internal/pycore_tstate.h | 2 +- Python/bytecodes.c | 18 ++-- Python/ceval.c | 8 +- Python/ceval_macros.h | 2 +- Python/generated_cases.c.h | 18 ++-- Python/jit.c | 10 +- Python/optimizer.c | 156 +++++++++++++++---------------- Python/pystate.c | 8 +- 8 files changed, 111 insertions(+), 111 deletions(-) diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index d2407df43c1a4e..1a0041cb85a07c 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -109,7 +109,7 @@ typedef struct _PyThreadStateImpl { Py_ssize_t reftotal; // this thread's total refcount operations #endif #if _Py_TIER2 - _PyJitTracerState jit_state; + _PyJitTracerState jit_tracer_state; #endif } _PyThreadStateImpl; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index c2d4725ccd458a..6a4d278f6ed78a 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5654,23 +5654,23 @@ dummy_func( // Super instructions. Instruction deopted. There's a mismatch in what the stack expects // in the optimizer. So we have to reflect in the trace correctly. _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; - if ((_tstate->jit_state.prev_state.instr->op.code == CALL_LIST_APPEND && + if ((_tstate->jit_tracer_state.prev_state.instr->op.code == CALL_LIST_APPEND && opcode == POP_TOP) || - (_tstate->jit_state.prev_state.instr->op.code == BINARY_OP_INPLACE_ADD_UNICODE && + (_tstate->jit_tracer_state.prev_state.instr->op.code == BINARY_OP_INPLACE_ADD_UNICODE && opcode == STORE_FAST)) { - _tstate->jit_state.prev_state.instr_is_super = true; + _tstate->jit_tracer_state.prev_state.instr_is_super = true; } else { - _tstate->jit_state.prev_state.instr = next_instr; + _tstate->jit_tracer_state.prev_state.instr = next_instr; } PyObject *prev_code = PyStackRef_AsPyObjectBorrow(frame->f_executable); - if (_tstate->jit_state.prev_state.instr_code != (PyCodeObject *)prev_code) { - Py_SETREF(_tstate->jit_state.prev_state.instr_code, (PyCodeObject*)Py_NewRef((prev_code))); + if (_tstate->jit_tracer_state.prev_state.instr_code != (PyCodeObject *)prev_code) { + Py_SETREF(_tstate->jit_tracer_state.prev_state.instr_code, (PyCodeObject*)Py_NewRef((prev_code))); } - _tstate->jit_state.prev_state.instr_frame = frame; - _tstate->jit_state.prev_state.instr_oparg = oparg; - _tstate->jit_state.prev_state.instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); + _tstate->jit_tracer_state.prev_state.instr_frame = frame; + _tstate->jit_tracer_state.prev_state.instr_oparg = oparg; + _tstate->jit_tracer_state.prev_state.instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { (&next_instr[1])->counter = trigger_backoff_counter(); } diff --git a/Python/ceval.c b/Python/ceval.c index aea353603b545f..4af03796effefa 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1002,15 +1002,15 @@ bail_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame) } _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; // Deal with backoffs - _PyExitData *exit = _tstate->jit_state.initial_state.exit; + _PyExitData *exit = _tstate->jit_tracer_state.initial_state.exit; if (exit == NULL) { // We hold a strong reference to the code object, so the instruction won't be freed. if (err <= 0) { - _Py_BackoffCounter counter = _tstate->jit_state.initial_state.jump_backward_instr[1].counter; - _tstate->jit_state.initial_state.jump_backward_instr[1].counter = restart_backoff_counter(counter); + _Py_BackoffCounter counter = _tstate->jit_tracer_state.initial_state.jump_backward_instr[1].counter; + _tstate->jit_tracer_state.initial_state.jump_backward_instr[1].counter = restart_backoff_counter(counter); } else { - _tstate->jit_state.initial_state.jump_backward_instr[1].counter = initial_jump_backoff_counter(); + _tstate->jit_tracer_state.initial_state.jump_backward_instr[1].counter = initial_jump_backoff_counter(); } } else { diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index a812c234cb2e1b..6ea8b724abe38a 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -400,7 +400,7 @@ do { \ JUMP_TO_LABEL(error); \ } \ if (keep_tracing_bit) { \ - assert(((_PyThreadStateImpl *)tstate)->jit_state.prev_state.code_curr_size == 2); \ + assert(((_PyThreadStateImpl *)tstate)->jit_tracer_state.prev_state.code_curr_size == 2); \ ENTER_TRACING(); \ DISPATCH_NON_TRACING(); \ } \ diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 241d2fd71e6692..a90a532a785e8d 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -12281,24 +12281,24 @@ JUMP_TO_LABEL(error); DISPATCH_GOTO_NON_TRACING(); } _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; - if ((_tstate->jit_state.prev_state.instr->op.code == CALL_LIST_APPEND && + if ((_tstate->jit_tracer_state.prev_state.instr->op.code == CALL_LIST_APPEND && opcode == POP_TOP) || - (_tstate->jit_state.prev_state.instr->op.code == BINARY_OP_INPLACE_ADD_UNICODE && + (_tstate->jit_tracer_state.prev_state.instr->op.code == BINARY_OP_INPLACE_ADD_UNICODE && opcode == STORE_FAST)) { - _tstate->jit_state.prev_state.instr_is_super = true; + _tstate->jit_tracer_state.prev_state.instr_is_super = true; } else { - _tstate->jit_state.prev_state.instr = next_instr; + _tstate->jit_tracer_state.prev_state.instr = next_instr; } PyObject *prev_code = PyStackRef_AsPyObjectBorrow(frame->f_executable); - if (_tstate->jit_state.prev_state.instr_code != (PyCodeObject *)prev_code) { + if (_tstate->jit_tracer_state.prev_state.instr_code != (PyCodeObject *)prev_code) { _PyFrame_SetStackPointer(frame, stack_pointer); - Py_SETREF(_tstate->jit_state.prev_state.instr_code, (PyCodeObject*)Py_NewRef((prev_code))); + Py_SETREF(_tstate->jit_tracer_state.prev_state.instr_code, (PyCodeObject*)Py_NewRef((prev_code))); stack_pointer = _PyFrame_GetStackPointer(frame); } - _tstate->jit_state.prev_state.instr_frame = frame; - _tstate->jit_state.prev_state.instr_oparg = oparg; - _tstate->jit_state.prev_state.instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); + _tstate->jit_tracer_state.prev_state.instr_frame = frame; + _tstate->jit_tracer_state.prev_state.instr_oparg = oparg; + _tstate->jit_tracer_state.prev_state.instr_stacklevel = PyStackRef_IsNone(frame->f_executable) ? 2 : STACK_LEVEL(); if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { (&next_instr[1])->counter = trigger_backoff_counter(); } diff --git a/Python/jit.c b/Python/jit.c index d2d5b16b69a9b6..c62cec2fcb1a97 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -139,7 +139,7 @@ typedef struct { typedef struct { trampoline_state trampolines; uintptr_t instruction_starts[UOP_MAX_TRACE_LENGTH]; -} jit_state; +} jit_tracer_state; // Warning! AArch64 requires you to get your hands dirty. These are your gloves: @@ -443,7 +443,7 @@ patch_x86_64_32rx(unsigned char *location, uint64_t value) patch_32r(location, value); } -void patch_aarch64_trampoline(unsigned char *location, int ordinal, jit_state *state); +void patch_aarch64_trampoline(unsigned char *location, int ordinal, jit_tracer_state *state); #include "jit_stencils.h" @@ -458,7 +458,7 @@ void patch_aarch64_trampoline(unsigned char *location, int ordinal, jit_state *s // Generate and patch AArch64 trampolines. The symbols to jump to are stored // in the jit_stencils.h in the symbols_map. void -patch_aarch64_trampoline(unsigned char *location, int ordinal, jit_state *state) +patch_aarch64_trampoline(unsigned char *location, int ordinal, jit_tracer_state *state) { uint64_t value = (uintptr_t)symbols_map[ordinal]; @@ -518,7 +518,7 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction trace[], siz // Loop once to find the total compiled size: size_t code_size = 0; size_t data_size = 0; - jit_state state = {0}; + jit_tracer_state state = {0}; for (size_t i = 0; i < length; i++) { const _PyUOpInstruction *instruction = &trace[i]; group = &stencil_groups[instruction->opcode]; @@ -596,7 +596,7 @@ compile_trampoline(void) const StencilGroup *group; size_t code_size = 0; size_t data_size = 0; - jit_state state = {0}; + jit_tracer_state state = {0}; group = &trampoline; code_size += group->code_size; data_size += group->data_size; diff --git a/Python/optimizer.c b/Python/optimizer.c index c4da65c19ff961..cee0f29efbfb60 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -119,17 +119,17 @@ _PyOptimizer_Optimize( _PyInterpreterFrame *frame, PyThreadState *tstate) { _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; - int chain_depth = _tstate->jit_state.initial_state.chain_depth; + int chain_depth = _tstate->jit_tracer_state.initial_state.chain_depth; PyInterpreterState *interp = _PyInterpreterState_GET(); assert(interp->jit); assert(!interp->compiling); - assert(_tstate->jit_state.initial_state.stack_depth >= 0); + assert(_tstate->jit_tracer_state.initial_state.stack_depth >= 0); #ifndef Py_GIL_DISABLED // Trace got stomped on by another thread. - if (_tstate->jit_state.initial_state.func == NULL) { + if (_tstate->jit_tracer_state.initial_state.func == NULL) { return 0; } - assert(_tstate->jit_state.initial_state.func != NULL); + assert(_tstate->jit_tracer_state.initial_state.func != NULL); interp->compiling = true; // The first executor in a chain and the MAX_CHAIN_DEPTH'th executor *must* // make progress in order to avoid infinite loops or excessively-long @@ -137,14 +137,14 @@ _PyOptimizer_Optimize( // this is true, since a deopt won't infinitely re-enter the executor: chain_depth %= MAX_CHAIN_DEPTH; bool progress_needed = chain_depth == 0; - PyCodeObject *code = (PyCodeObject *)_tstate->jit_state.initial_state.code; - _Py_CODEUNIT *start = _tstate->jit_state.initial_state.start_instr; + PyCodeObject *code = (PyCodeObject *)_tstate->jit_tracer_state.initial_state.code; + _Py_CODEUNIT *start = _tstate->jit_tracer_state.initial_state.start_instr; if (progress_needed && !has_space_for_executor(code, start)) { interp->compiling = false; return 0; } // One of our dependencies while tracing was invalidated. Not worth compiling. - if (!_tstate->jit_state.prev_state.dependencies_still_valid) { + if (!_tstate->jit_tracer_state.prev_state.dependencies_still_valid) { interp->compiling = false; return 0; } @@ -173,7 +173,7 @@ _PyOptimizer_Optimize( else { executor->vm_data.code = NULL; } - _PyExitData *exit = _tstate->jit_state.initial_state.exit; + _PyExitData *exit = _tstate->jit_tracer_state.initial_state.exit; if (exit != NULL) { exit->executor = executor; } @@ -565,15 +565,15 @@ _PyJit_translate_single_bytecode_to_trace( } #endif _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; - PyCodeObject *old_code = _tstate->jit_state.prev_state.instr_code; - bool progress_needed = (_tstate->jit_state.initial_state.chain_depth % MAX_CHAIN_DEPTH) == 0; - _PyBloomFilter *dependencies = &_tstate->jit_state.prev_state.dependencies; + PyCodeObject *old_code = _tstate->jit_tracer_state.prev_state.instr_code; + bool progress_needed = (_tstate->jit_tracer_state.initial_state.chain_depth % MAX_CHAIN_DEPTH) == 0; + _PyBloomFilter *dependencies = &_tstate->jit_tracer_state.prev_state.dependencies; _Py_BloomFilter_Add(dependencies, old_code); - int trace_length = _tstate->jit_state.prev_state.code_curr_size; - _PyUOpInstruction *trace = _tstate->jit_state.code_buffer; - int max_length = _tstate->jit_state.prev_state.code_max_size; + int trace_length = _tstate->jit_tracer_state.prev_state.code_curr_size; + _PyUOpInstruction *trace = _tstate->jit_tracer_state.code_buffer; + int max_length = _tstate->jit_tracer_state.prev_state.code_max_size; - _Py_CODEUNIT *this_instr = _tstate->jit_state.prev_state.instr; + _Py_CODEUNIT *this_instr = _tstate->jit_tracer_state.prev_state.instr; _Py_CODEUNIT *target_instr = this_instr; uint32_t target = 0; @@ -583,7 +583,7 @@ _PyJit_translate_single_bytecode_to_trace( // Rewind EXTENDED_ARG so that we see the whole thing. // We must point to the first EXTENDED_ARG when deopting. - int oparg = _tstate->jit_state.prev_state.instr_oparg; + int oparg = _tstate->jit_tracer_state.prev_state.instr_oparg; int opcode = this_instr->op.code; int rewind_oparg = oparg; while (rewind_oparg > 255) { @@ -591,7 +591,7 @@ _PyJit_translate_single_bytecode_to_trace( target--; } - int old_stack_level = _tstate->jit_state.prev_state.instr_stacklevel; + int old_stack_level = _tstate->jit_tracer_state.prev_state.instr_stacklevel; // Strange control-flow bool has_dynamic_jump_taken = OPCODE_HAS_UNPREDICTABLE_JUMP(opcode) && @@ -599,7 +599,7 @@ _PyJit_translate_single_bytecode_to_trace( /* Special case the first instruction, * so that we can guarantee forward progress */ - if (progress_needed && _tstate->jit_state.prev_state.code_curr_size <= 3) { + if (progress_needed && _tstate->jit_tracer_state.prev_state.code_curr_size <= 3) { if (OPCODE_HAS_EXIT(opcode) || OPCODE_HAS_DEOPT(opcode)) { opcode = _PyOpcode_Deopt[opcode]; } @@ -632,8 +632,8 @@ _PyJit_translate_single_bytecode_to_trace( #endif // Skip over super instructions. - if (_tstate->jit_state.prev_state.instr_is_super) { - _tstate->jit_state.prev_state.instr_is_super = false; + if (_tstate->jit_tracer_state.prev_state.instr_is_super) { + _tstate->jit_tracer_state.prev_state.instr_is_super = false; return 1; } @@ -641,13 +641,13 @@ _PyJit_translate_single_bytecode_to_trace( goto full; } - if (!_tstate->jit_state.prev_state.dependencies_still_valid) { + if (!_tstate->jit_tracer_state.prev_state.dependencies_still_valid) { goto done; } // This happens when a recursive call happens that we can't trace. Such as Python -> C -> Python calls // If we haven't guarded the IP, then it's untraceable. - if (frame != _tstate->jit_state.prev_state.instr_frame && !needs_guard_ip) { + if (frame != _tstate->jit_tracer_state.prev_state.instr_frame && !needs_guard_ip) { DPRINTF(2, "Unsupported: unguardable jump taken\n"); goto unsupported; } @@ -746,11 +746,11 @@ _PyJit_translate_single_bytecode_to_trace( _Py_FALLTHROUGH; case JUMP_BACKWARD_NO_INTERRUPT: { - if ((next_instr != _tstate->jit_state.initial_state.close_loop_instr) && - (next_instr != _tstate->jit_state.initial_state.start_instr) && - _tstate->jit_state.prev_state.code_curr_size > 5 && + if ((next_instr != _tstate->jit_tracer_state.initial_state.close_loop_instr) && + (next_instr != _tstate->jit_tracer_state.initial_state.start_instr) && + _tstate->jit_tracer_state.prev_state.code_curr_size > 5 && // For side exits, we don't want to terminate them early. - _tstate->jit_state.initial_state.exit == NULL && + _tstate->jit_tracer_state.initial_state.exit == NULL && // These are coroutines, and we want to unroll those usually. opcode != JUMP_BACKWARD_NO_INTERRUPT) { // We encountered a JUMP_BACKWARD but not to the top of our own loop. @@ -761,7 +761,7 @@ _PyJit_translate_single_bytecode_to_trace( ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); trace[trace_length-1].operand1 = true; // is_control_flow DPRINTF(2, "JUMP_BACKWARD not to top ends trace %p %p %p\n", next_instr, - _tstate->jit_state.initial_state.close_loop_instr, _tstate->jit_state.initial_state.start_instr); + _tstate->jit_tracer_state.initial_state.close_loop_instr, _tstate->jit_tracer_state.initial_state.start_instr); goto done; } break; @@ -915,9 +915,9 @@ _PyJit_translate_single_bytecode_to_trace( } } // Loop back to the start - int is_first_instr = _tstate->jit_state.initial_state.close_loop_instr == next_instr || - _tstate->jit_state.initial_state.start_instr == next_instr; - if (is_first_instr && _tstate->jit_state.prev_state.code_curr_size > 5) { + int is_first_instr = _tstate->jit_tracer_state.initial_state.close_loop_instr == next_instr || + _tstate->jit_tracer_state.initial_state.start_instr == next_instr; + if (is_first_instr && _tstate->jit_tracer_state.prev_state.code_curr_size > 5) { if (needs_guard_ip) { ADD_TO_TRACE(_SET_IP, 0, (uintptr_t)next_instr, 0); } @@ -925,27 +925,27 @@ _PyJit_translate_single_bytecode_to_trace( goto done; } DPRINTF(2, "Trace continuing\n"); - _tstate->jit_state.prev_state.code_curr_size = trace_length; - _tstate->jit_state.prev_state.code_max_size = max_length; + _tstate->jit_tracer_state.prev_state.code_curr_size = trace_length; + _tstate->jit_tracer_state.prev_state.code_max_size = max_length; return 1; done: DPRINTF(2, "Trace done\n"); - _tstate->jit_state.prev_state.code_curr_size = trace_length; - _tstate->jit_state.prev_state.code_max_size = max_length; + _tstate->jit_tracer_state.prev_state.code_curr_size = trace_length; + _tstate->jit_tracer_state.prev_state.code_max_size = max_length; return 0; full: DPRINTF(2, "Trace full\n"); - if (!is_terminator(&_tstate->jit_state.code_buffer[trace_length-1])) { + if (!is_terminator(&_tstate->jit_tracer_state.code_buffer[trace_length-1])) { // Undo the last few instructions. - trace_length = _tstate->jit_state.prev_state.code_curr_size; - max_length = _tstate->jit_state.prev_state.code_max_size; + trace_length = _tstate->jit_tracer_state.prev_state.code_curr_size; + max_length = _tstate->jit_tracer_state.prev_state.code_max_size; // We previously reversed one. max_length += 1; ADD_TO_TRACE(_EXIT_TRACE, 0, 0, target); trace[trace_length-1].operand1 = true; // is_control_flow } - _tstate->jit_state.prev_state.code_curr_size = trace_length; - _tstate->jit_state.prev_state.code_max_size = max_length; + _tstate->jit_tracer_state.prev_state.code_curr_size = trace_length; + _tstate->jit_tracer_state.prev_state.code_max_size = max_length; return 0; } @@ -959,15 +959,15 @@ _PyJit_TryInitializeTracing( _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; // A recursive trace. // Don't trace into the inner call because it will stomp on the previous trace, causing endless retraces. - if (_tstate->jit_state.prev_state.code_curr_size > 2) { + if (_tstate->jit_tracer_state.prev_state.code_curr_size > 2) { return 0; } if (oparg > 0xFFFF) { return 0; } - if (_tstate->jit_state.code_buffer == NULL) { - _tstate->jit_state.code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); - if (_tstate->jit_state.code_buffer == NULL) { + if (_tstate->jit_tracer_state.code_buffer == NULL) { + _tstate->jit_tracer_state.code_buffer = (_PyUOpInstruction *)_PyObject_VirtualAlloc(UOP_BUFFER_SIZE); + if (_tstate->jit_tracer_state.code_buffer == NULL) { // Don't error, just go to next instruction. return 0; } @@ -992,33 +992,33 @@ _PyJit_TryInitializeTracing( chain_depth); #endif - add_to_trace(_tstate->jit_state.code_buffer, 0, _START_EXECUTOR, 0, (uintptr_t)start_instr, INSTR_IP(start_instr, code)); - add_to_trace(_tstate->jit_state.code_buffer, 1, _MAKE_WARM, 0, 0, 0); - _tstate->jit_state.prev_state.code_curr_size = 2; - - _tstate->jit_state.prev_state.code_max_size = UOP_MAX_TRACE_LENGTH; - _tstate->jit_state.initial_state.start_instr = start_instr; - _tstate->jit_state.initial_state.close_loop_instr = close_loop_instr; - _tstate->jit_state.initial_state.code = (PyCodeObject *)Py_NewRef(code); - _tstate->jit_state.initial_state.func = (PyFunctionObject *)Py_NewRef(func); - _tstate->jit_state.initial_state.exit = exit; - _tstate->jit_state.initial_state.stack_depth = curr_stackdepth; - _tstate->jit_state.initial_state.chain_depth = chain_depth; - _tstate->jit_state.prev_state.instr_frame = frame; - _tstate->jit_state.prev_state.dependencies_still_valid = true; - _tstate->jit_state.prev_state.instr_code = (PyCodeObject *)Py_NewRef(_PyFrame_GetCode(frame)); - _tstate->jit_state.prev_state.instr = curr_instr; - _tstate->jit_state.prev_state.instr_frame = frame; - _tstate->jit_state.prev_state.instr_oparg = oparg; - _tstate->jit_state.prev_state.instr_stacklevel = curr_stackdepth; - _tstate->jit_state.prev_state.instr_is_super = false; + add_to_trace(_tstate->jit_tracer_state.code_buffer, 0, _START_EXECUTOR, 0, (uintptr_t)start_instr, INSTR_IP(start_instr, code)); + add_to_trace(_tstate->jit_tracer_state.code_buffer, 1, _MAKE_WARM, 0, 0, 0); + _tstate->jit_tracer_state.prev_state.code_curr_size = 2; + + _tstate->jit_tracer_state.prev_state.code_max_size = UOP_MAX_TRACE_LENGTH; + _tstate->jit_tracer_state.initial_state.start_instr = start_instr; + _tstate->jit_tracer_state.initial_state.close_loop_instr = close_loop_instr; + _tstate->jit_tracer_state.initial_state.code = (PyCodeObject *)Py_NewRef(code); + _tstate->jit_tracer_state.initial_state.func = (PyFunctionObject *)Py_NewRef(func); + _tstate->jit_tracer_state.initial_state.exit = exit; + _tstate->jit_tracer_state.initial_state.stack_depth = curr_stackdepth; + _tstate->jit_tracer_state.initial_state.chain_depth = chain_depth; + _tstate->jit_tracer_state.prev_state.instr_frame = frame; + _tstate->jit_tracer_state.prev_state.dependencies_still_valid = true; + _tstate->jit_tracer_state.prev_state.instr_code = (PyCodeObject *)Py_NewRef(_PyFrame_GetCode(frame)); + _tstate->jit_tracer_state.prev_state.instr = curr_instr; + _tstate->jit_tracer_state.prev_state.instr_frame = frame; + _tstate->jit_tracer_state.prev_state.instr_oparg = oparg; + _tstate->jit_tracer_state.prev_state.instr_stacklevel = curr_stackdepth; + _tstate->jit_tracer_state.prev_state.instr_is_super = false; assert(curr_instr->op.code == JUMP_BACKWARD_JIT || (exit != NULL)); - _tstate->jit_state.initial_state.jump_backward_instr = curr_instr; + _tstate->jit_tracer_state.initial_state.jump_backward_instr = curr_instr; if (_PyOpcode_Caches[_PyOpcode_Deopt[close_loop_instr->op.code]]) { close_loop_instr[1].counter = trigger_backoff_counter(); } - _Py_BloomFilter_Init(&_tstate->jit_state.prev_state.dependencies); + _Py_BloomFilter_Init(&_tstate->jit_tracer_state.prev_state.dependencies); return 1; } @@ -1026,11 +1026,11 @@ void _PyJit_FinalizeTracing(PyThreadState *tstate) { _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; - Py_CLEAR(_tstate->jit_state.initial_state.code); - Py_CLEAR(_tstate->jit_state.initial_state.func); - Py_CLEAR(_tstate->jit_state.prev_state.instr_code); - _tstate->jit_state.prev_state.code_curr_size = 2; - _tstate->jit_state.prev_state.code_max_size = UOP_MAX_TRACE_LENGTH - 1; + Py_CLEAR(_tstate->jit_tracer_state.initial_state.code); + Py_CLEAR(_tstate->jit_tracer_state.initial_state.func); + Py_CLEAR(_tstate->jit_tracer_state.prev_state.instr_code); + _tstate->jit_tracer_state.prev_state.code_curr_size = 2; + _tstate->jit_tracer_state.prev_state.code_max_size = UOP_MAX_TRACE_LENGTH - 1; } @@ -1351,16 +1351,16 @@ uop_optimize( // It is the optimizer's responsibility to add the dependencies it requires on its own. _PyBloomFilter new_dependencies; _Py_BloomFilter_Init(&new_dependencies); - _Py_BloomFilter_Add(&new_dependencies, _tstate->jit_state.initial_state.code); - _PyUOpInstruction *buffer = _tstate->jit_state.code_buffer; + _Py_BloomFilter_Add(&new_dependencies, _tstate->jit_tracer_state.initial_state.code); + _PyUOpInstruction *buffer = _tstate->jit_tracer_state.code_buffer; OPT_STAT_INC(attempts); char *env_var = Py_GETENV("PYTHON_UOPS_OPTIMIZE"); bool is_noopt = true; if (env_var == NULL || *env_var == '\0' || *env_var > '0') { is_noopt = false; } - int curr_stackentries = _tstate->jit_state.initial_state.stack_depth; - int length = _tstate->jit_state.prev_state.code_curr_size; + int curr_stackentries = _tstate->jit_tracer_state.initial_state.stack_depth; + int length = _tstate->jit_tracer_state.prev_state.code_curr_size; // Trace too short, don't bother. if (length <= 5) { return 0; @@ -1370,7 +1370,7 @@ uop_optimize( OPT_STAT_INC(traces_created); if (!is_noopt) { length = _Py_uop_analyze_and_optimize( - _tstate->jit_state.initial_state.func, + _tstate->jit_tracer_state.initial_state.func, buffer,length, curr_stackentries, &new_dependencies); if (length <= 0) { @@ -1396,7 +1396,7 @@ uop_optimize( length = prepare_for_execution(buffer, length); assert(length <= UOP_MAX_TRACE_LENGTH); _PyExecutorObject *executor = make_executor_from_uops( - buffer, length, &new_dependencies, _tstate->jit_state.initial_state.chain_depth); + buffer, length, &new_dependencies, _tstate->jit_tracer_state.initial_state.chain_depth); if (executor == NULL) { return -1; } @@ -1739,9 +1739,9 @@ _PyJit_Tracer_InvalidateDependency(PyThreadState *tstate, void *obj) _Py_BloomFilter_Init(&obj_filter); _Py_BloomFilter_Add(&obj_filter, obj); _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; - if (bloom_filter_may_contain(&_tstate->jit_state.prev_state.dependencies, &obj_filter)) + if (bloom_filter_may_contain(&_tstate->jit_tracer_state.prev_state.dependencies, &obj_filter)) { - _tstate->jit_state.prev_state.dependencies_still_valid = false; + _tstate->jit_tracer_state.prev_state.dependencies_still_valid = false; } } /* Invalidate all executors */ diff --git a/Python/pystate.c b/Python/pystate.c index a6a54c2f94a7dc..1de54a2562c904 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1489,7 +1489,7 @@ init_threadstate(_PyThreadStateImpl *_tstate, _tstate->asyncio_running_task = NULL; #ifdef _Py_TIER2 - _tstate->jit_state.code_buffer = NULL; + _tstate->jit_tracer_state.code_buffer = NULL; #endif tstate->delete_later = NULL; @@ -1792,9 +1792,9 @@ tstate_delete_common(PyThreadState *tstate, int release_gil) #if _Py_TIER2 _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; - if (_tstate->jit_state.code_buffer != NULL) { - _PyObject_VirtualFree(_tstate->jit_state.code_buffer, UOP_BUFFER_SIZE); - _tstate->jit_state.code_buffer = NULL; + if (_tstate->jit_tracer_state.code_buffer != NULL) { + _PyObject_VirtualFree(_tstate->jit_tracer_state.code_buffer, UOP_BUFFER_SIZE); + _tstate->jit_tracer_state.code_buffer = NULL; } #endif From ae1d6fe73794b8db83dedcba615fa56c55b11c30 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 11 Nov 2025 15:10:46 +0000 Subject: [PATCH 182/190] Restore a test, address review --- Include/internal/pycore_interp_structs.h | 1 - Lib/test/test_capi/test_opt.py | 1 - Python/optimizer.c | 4 ---- 3 files changed, 6 deletions(-) diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 39023923c7e2b7..516a4c3595e3fa 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -755,7 +755,6 @@ struct _Py_unique_id_pool { typedef _Py_CODEUNIT *(*_PyJitEntryFuncPtr)(struct _PyExecutorObject *exec, _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate); - /* PyInterpreterState holds the global state for one of the runtime's interpreters. Typically the initial (main) interpreter is the only one. diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 10b308c0aafab5..6c866c5833aff7 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -139,7 +139,6 @@ def testfunc(x): self.assertIn("_JUMP_TO_TOP", uops) self.assertIn("_LOAD_FAST_BORROW_0", uops) - @unittest.skip("gh-139109 WIP") def test_extended_arg(self): "Check EXTENDED_ARG handling in superblock creation" ns = {} diff --git a/Python/optimizer.c b/Python/optimizer.c index cee0f29efbfb60..8443476dfa95d1 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -125,10 +125,6 @@ _PyOptimizer_Optimize( assert(!interp->compiling); assert(_tstate->jit_tracer_state.initial_state.stack_depth >= 0); #ifndef Py_GIL_DISABLED - // Trace got stomped on by another thread. - if (_tstate->jit_tracer_state.initial_state.func == NULL) { - return 0; - } assert(_tstate->jit_tracer_state.initial_state.func != NULL); interp->compiling = true; // The first executor in a chain and the MAX_CHAIN_DEPTH'th executor *must* From 0658f1b683cacae67cb70059f3ac2a093a8a9435 Mon Sep 17 00:00:00 2001 From: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Date: Tue, 11 Nov 2025 15:12:12 +0000 Subject: [PATCH 183/190] remove CALL_LIST_APPEND fix --- Python/specialize.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Python/specialize.c b/Python/specialize.c index 5b44432b8b7258..a1c5dedd61563b 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2041,12 +2041,8 @@ specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr, bool pop = (next.op.code == POP_TOP); int oparg = instr->op.arg; if ((PyObject *)descr == list_append && oparg == 1 && pop) { - PyThreadState *tstate = PyThreadState_GET(); - PyObject *self = PyStackRef_AsPyObjectBorrow(tstate->current_frame->stackpointer[-2]); - if (PyList_CheckExact(self)) { - specialize(instr, CALL_LIST_APPEND); - return 0; - } + specialize(instr, CALL_LIST_APPEND); + return 0; } specialize(instr, CALL_METHOD_DESCRIPTOR_O); return 0; From f8a764ad51b22f3eecd31d020b81dce2f8f17ffe Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Wed, 12 Nov 2025 13:30:33 +0000 Subject: [PATCH 184/190] fix JIT builds --- Python/jit.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Python/jit.c b/Python/jit.c index c62cec2fcb1a97..d2d5b16b69a9b6 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -139,7 +139,7 @@ typedef struct { typedef struct { trampoline_state trampolines; uintptr_t instruction_starts[UOP_MAX_TRACE_LENGTH]; -} jit_tracer_state; +} jit_state; // Warning! AArch64 requires you to get your hands dirty. These are your gloves: @@ -443,7 +443,7 @@ patch_x86_64_32rx(unsigned char *location, uint64_t value) patch_32r(location, value); } -void patch_aarch64_trampoline(unsigned char *location, int ordinal, jit_tracer_state *state); +void patch_aarch64_trampoline(unsigned char *location, int ordinal, jit_state *state); #include "jit_stencils.h" @@ -458,7 +458,7 @@ void patch_aarch64_trampoline(unsigned char *location, int ordinal, jit_tracer_s // Generate and patch AArch64 trampolines. The symbols to jump to are stored // in the jit_stencils.h in the symbols_map. void -patch_aarch64_trampoline(unsigned char *location, int ordinal, jit_tracer_state *state) +patch_aarch64_trampoline(unsigned char *location, int ordinal, jit_state *state) { uint64_t value = (uintptr_t)symbols_map[ordinal]; @@ -518,7 +518,7 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction trace[], siz // Loop once to find the total compiled size: size_t code_size = 0; size_t data_size = 0; - jit_tracer_state state = {0}; + jit_state state = {0}; for (size_t i = 0; i < length; i++) { const _PyUOpInstruction *instruction = &trace[i]; group = &stencil_groups[instruction->opcode]; @@ -596,7 +596,7 @@ compile_trampoline(void) const StencilGroup *group; size_t code_size = 0; size_t data_size = 0; - jit_tracer_state state = {0}; + jit_state state = {0}; group = &trampoline; code_size += group->code_size; data_size += group->data_size; From b72ab8bb66814f8eedf5c73b5610134a68456323 Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Wed, 12 Nov 2025 18:42:08 +0000 Subject: [PATCH 185/190] Address review --- Python/bytecodes.c | 20 ++++++++++---------- Python/ceval.c | 4 ++-- Python/generated_cases.c.h | 4 ++-- Tools/cases_generator/tier2_generator.py | 4 ++-- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 6a4d278f6ed78a..2c798855a71f55 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5462,33 +5462,33 @@ dummy_func( } tier2 op(_GUARD_IP__PUSH_FRAME, (ip/4 --)) { - _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF(_PUSH_FRAME); + _Py_CODEUNIT *target = frame->instr_ptr + IP_OFFSET_OF(_PUSH_FRAME); if (target != (_Py_CODEUNIT *)ip) { - frame->instr_ptr += OFFSET_OF(_PUSH_FRAME); + frame->instr_ptr += IP_OFFSET_OF(_PUSH_FRAME); EXIT_IF(true); } } tier2 op(_GUARD_IP_YIELD_VALUE, (ip/4 --)) { - _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF(YIELD_VALUE); + _Py_CODEUNIT *target = frame->instr_ptr + IP_OFFSET_OF(YIELD_VALUE); if (target != (_Py_CODEUNIT *)ip) { - frame->instr_ptr += OFFSET_OF(YIELD_VALUE); + frame->instr_ptr += IP_OFFSET_OF(YIELD_VALUE); EXIT_IF(true); } } tier2 op(_GUARD_IP_RETURN_VALUE, (ip/4 --)) { - _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF(RETURN_VALUE); + _Py_CODEUNIT *target = frame->instr_ptr + IP_OFFSET_OF(RETURN_VALUE); if (target != (_Py_CODEUNIT *)ip) { - frame->instr_ptr += OFFSET_OF(RETURN_VALUE); + frame->instr_ptr += IP_OFFSET_OF(RETURN_VALUE); EXIT_IF(true); } } tier2 op(_GUARD_IP_RETURN_GENERATOR, (ip/4 --)) { - _Py_CODEUNIT *target = frame->instr_ptr + OFFSET_OF(RETURN_GENERATOR); + _Py_CODEUNIT *target = frame->instr_ptr + IP_OFFSET_OF(RETURN_GENERATOR); if (target != (_Py_CODEUNIT *)ip) { - frame->instr_ptr += OFFSET_OF(RETURN_GENERATOR); + frame->instr_ptr += IP_OFFSET_OF(RETURN_GENERATOR); EXIT_IF(true); } } @@ -5647,7 +5647,7 @@ dummy_func( int full = !_PyJit_translate_single_bytecode_to_trace(tstate, frame, next_instr, stop_tracing); if (full) { LEAVE_TRACING(); - int err = bail_tracing_and_jit(tstate, frame); + int err = stop_tracing_and_jit(tstate, frame); ERROR_IF(err < 0); DISPATCH_GOTO_NON_TRACING(); } @@ -5686,7 +5686,7 @@ dummy_func( int opcode = next_instr->op.code; _PyJit_translate_single_bytecode_to_trace(tstate, frame, NULL, true); LEAVE_TRACING(); - int err = bail_tracing_and_jit(tstate, frame); + int err = stop_tracing_and_jit(tstate, frame); ERROR_IF(err < 0); DISPATCH_GOTO_NON_TRACING(); #else diff --git a/Python/ceval.c b/Python/ceval.c index 4af03796effefa..06ceb7e828c4c5 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -993,7 +993,7 @@ _PyObjectArray_Free(PyObject **array, PyObject **scratch) #if _Py_TIER2 // 0 for success, -1 for error. static int -bail_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame) +stop_tracing_and_jit(PyThreadState *tstate, _PyInterpreterFrame *frame) { int _is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL); int err = 0; @@ -1245,7 +1245,7 @@ _PyTier2Interpreter( uopcode = next_uop->opcode; #ifdef Py_DEBUG if (frame->lltrace >= 3) { - // dump_stack(frame, stack_pointer); + dump_stack(frame, stack_pointer); if (next_uop->opcode == _START_EXECUTOR) { printf("%4d uop: ", 0); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index a90a532a785e8d..a984da6dc912a2 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -12273,7 +12273,7 @@ JUMP_TO_LABEL(error); if (full) { LEAVE_TRACING(); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = bail_tracing_and_jit(tstate, frame); + int err = stop_tracing_and_jit(tstate, frame); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { JUMP_TO_LABEL(error); @@ -12318,7 +12318,7 @@ JUMP_TO_LABEL(error); stack_pointer = _PyFrame_GetStackPointer(frame); LEAVE_TRACING(); _PyFrame_SetStackPointer(frame, stack_pointer); - int err = bail_tracing_and_jit(tstate, frame); + int err = stop_tracing_and_jit(tstate, frame); stack_pointer = _PyFrame_GetStackPointer(frame); if (err < 0) { JUMP_TO_LABEL(error); diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index 8a93b6277155ef..ac3e6b94afe49e 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -63,7 +63,7 @@ class Tier2Emitter(Emitter): def __init__(self, out: CWriter, labels: dict[str, Label]): super().__init__(out, labels) self._replacers["oparg"] = self.oparg - self._replacers["OFFSET_OF"] = self.offset_of + self._replacers["IP_OFFSET_OF"] = self.ip_offset_of def goto_error(self, offset: int, storage: Storage) -> str: # To do: Add jump targets for popping values. @@ -135,7 +135,7 @@ def oparg( self.out.emit_at(uop.name[-1], tkn) return True - def offset_of( + def ip_offset_of( self, tkn: Token, tkn_iter: TokenIterator, From fa45ae85dd85d9a41e400fb0046f12577f817262 Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Thu, 13 Nov 2025 14:37:21 +0000 Subject: [PATCH 186/190] add stats, document --- Include/cpython/pystats.h | 2 ++ Python/optimizer.c | 58 ++++++++++++++++++++----------------- Python/optimizer_analysis.c | 1 + Python/optimizer_symbols.c | 1 + 4 files changed, 35 insertions(+), 27 deletions(-) diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index d0a925a3055485..1c94603c08b9b6 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -150,6 +150,8 @@ typedef struct _optimization_stats { uint64_t optimized_trace_length_hist[_Py_UOP_HIST_SIZE]; uint64_t optimizer_attempts; uint64_t optimizer_successes; + uint64_t optimizer_contradiction; + uint64_t optimizer_frame_overflow; uint64_t optimizer_failure_reason_no_memory; uint64_t remove_globals_builtins_changed; uint64_t remove_globals_incorrect_keys; diff --git a/Python/optimizer.c b/Python/optimizer.c index dab4df11cc560e..2f6c2eb8f05eb3 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -29,6 +29,16 @@ #define MAX_EXECUTORS_SIZE 256 +// Trace too short, no progress: +// _START_EXECUTOR +// _MAKE_WARM +// _CHECK_VALIDITY +// _SET_IP +// is 4-5 instructions. +#define CODE_SIZE_NO_PROGRESS 5 +// We start with _START_EXECUTOR, _MAKE_WARM +#define CODE_SIZE_EMPTY 2 + #define _PyExecutorObject_CAST(op) ((_PyExecutorObject *)(op)) static bool @@ -489,6 +499,14 @@ BRANCH_TO_GUARD[4][2] = { [POP_JUMP_IF_NOT_NONE - POP_JUMP_IF_FALSE][1] = _GUARD_IS_NOT_NONE_POP, }; +static const uint16_t +guard_ip_uop[MAX_UOP_ID + 1] = { + [_PUSH_FRAME] = _GUARD_IP__PUSH_FRAME, + [_RETURN_GENERATOR] = _GUARD_IP_RETURN_GENERATOR, + [_RETURN_VALUE] = _GUARD_IP_RETURN_VALUE, + [_YIELD_VALUE] = _GUARD_IP_YIELD_VALUE, +}; + #define CONFIDENCE_RANGE 1000 #define CONFIDENCE_CUTOFF 333 @@ -601,7 +619,7 @@ _PyJit_translate_single_bytecode_to_trace( /* Special case the first instruction, * so that we can guarantee forward progress */ - if (progress_needed && _tstate->jit_tracer_state.prev_state.code_curr_size <= 3) { + if (progress_needed && _tstate->jit_tracer_state.prev_state.code_curr_size <= CODE_SIZE_NO_PROGRESS) { if (OPCODE_HAS_EXIT(opcode) || OPCODE_HAS_DEOPT(opcode)) { opcode = _PyOpcode_Deopt[opcode]; } @@ -750,7 +768,7 @@ _PyJit_translate_single_bytecode_to_trace( { if ((next_instr != _tstate->jit_tracer_state.initial_state.close_loop_instr) && (next_instr != _tstate->jit_tracer_state.initial_state.start_instr) && - _tstate->jit_tracer_state.prev_state.code_curr_size > 5 && + _tstate->jit_tracer_state.prev_state.code_curr_size > CODE_SIZE_NO_PROGRESS && // For side exits, we don't want to terminate them early. _tstate->jit_tracer_state.initial_state.exit == NULL && // These are coroutines, and we want to unroll those usually. @@ -898,28 +916,17 @@ _PyJit_translate_single_bytecode_to_trace( } // End switch (opcode) if (needs_guard_ip) { - switch (trace[trace_length-1].opcode) { - case _PUSH_FRAME: - ADD_TO_TRACE(_GUARD_IP__PUSH_FRAME, 0, (uintptr_t)next_instr, 0); - break; - case _RETURN_GENERATOR: - ADD_TO_TRACE(_GUARD_IP_RETURN_GENERATOR, 0, (uintptr_t)next_instr, 0); - break; - case _RETURN_VALUE: - ADD_TO_TRACE(_GUARD_IP_RETURN_VALUE, 0, (uintptr_t)next_instr, 0); - break; - case _YIELD_VALUE: - ADD_TO_TRACE(_GUARD_IP_YIELD_VALUE, 0, (uintptr_t)next_instr, 0); - break; - default: - DPRINTF(1, "Unknown uop needing guard ip %s\n", _PyOpcode_uop_name[trace[trace_length-1].opcode]); - Py_UNREACHABLE(); + uint16_t guard_ip = guard_ip_uop[trace[trace_length-1].opcode]; + if (guard_ip == 0) { + DPRINTF(1, "Unknown uop needing guard ip %s\n", _PyOpcode_uop_name[trace[trace_length-1].opcode]); + Py_UNREACHABLE(); } + ADD_TO_TRACE(guard_ip, 0, (uintptr_t)next_instr, 0); } // Loop back to the start int is_first_instr = _tstate->jit_tracer_state.initial_state.close_loop_instr == next_instr || _tstate->jit_tracer_state.initial_state.start_instr == next_instr; - if (is_first_instr && _tstate->jit_tracer_state.prev_state.code_curr_size > 5) { + if (is_first_instr && _tstate->jit_tracer_state.prev_state.code_curr_size > CODE_SIZE_NO_PROGRESS) { if (needs_guard_ip) { ADD_TO_TRACE(_SET_IP, 0, (uintptr_t)next_instr, 0); } @@ -961,7 +968,7 @@ _PyJit_TryInitializeTracing( _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; // A recursive trace. // Don't trace into the inner call because it will stomp on the previous trace, causing endless retraces. - if (_tstate->jit_tracer_state.prev_state.code_curr_size > 2) { + if (_tstate->jit_tracer_state.prev_state.code_curr_size > CODE_SIZE_NO_PROGRESS) { return 0; } if (oparg > 0xFFFF) { @@ -996,7 +1003,7 @@ _PyJit_TryInitializeTracing( add_to_trace(_tstate->jit_tracer_state.code_buffer, 0, _START_EXECUTOR, 0, (uintptr_t)start_instr, INSTR_IP(start_instr, code)); add_to_trace(_tstate->jit_tracer_state.code_buffer, 1, _MAKE_WARM, 0, 0, 0); - _tstate->jit_tracer_state.prev_state.code_curr_size = 2; + _tstate->jit_tracer_state.prev_state.code_curr_size = CODE_SIZE_EMPTY; _tstate->jit_tracer_state.prev_state.code_max_size = UOP_MAX_TRACE_LENGTH; _tstate->jit_tracer_state.initial_state.start_instr = start_instr; @@ -1031,7 +1038,7 @@ _PyJit_FinalizeTracing(PyThreadState *tstate) Py_CLEAR(_tstate->jit_tracer_state.initial_state.code); Py_CLEAR(_tstate->jit_tracer_state.initial_state.func); Py_CLEAR(_tstate->jit_tracer_state.prev_state.instr_code); - _tstate->jit_tracer_state.prev_state.code_curr_size = 2; + _tstate->jit_tracer_state.prev_state.code_curr_size = CODE_SIZE_EMPTY; _tstate->jit_tracer_state.prev_state.code_max_size = UOP_MAX_TRACE_LENGTH - 1; } @@ -1112,7 +1119,6 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) exit_op = _HANDLE_PENDING_AND_DEOPT; } int32_t jump_target = target; - bool unique_target = false; if ( opcode == _GUARD_IP__PUSH_FRAME || opcode == _GUARD_IP_RETURN_VALUE || @@ -1120,10 +1126,9 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) opcode == _GUARD_IP_RETURN_GENERATOR ) { exit_op = _DYNAMIC_EXIT; - unique_target = true; } bool is_control_flow = (opcode == _GUARD_IS_FALSE_POP || opcode == _GUARD_IS_TRUE_POP || is_for_iter_test[opcode]); - if (unique_target || jump_target != current_jump_target || current_exit_op != exit_op) { + if (jump_target != current_jump_target || current_exit_op != exit_op) { make_exit(&buffer[next_spare], exit_op, jump_target, is_control_flow); current_exit_op = exit_op; current_jump_target = jump_target; @@ -1363,8 +1368,7 @@ uop_optimize( } int curr_stackentries = _tstate->jit_tracer_state.initial_state.stack_depth; int length = _tstate->jit_tracer_state.prev_state.code_curr_size; - // Trace too short, don't bother. - if (length <= 5) { + if (length <= CODE_SIZE_NO_PROGRESS) { return 0; } assert(length > 0); diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index ded542ecc42c6d..8d7b734e17cb0b 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -368,6 +368,7 @@ optimize_uops( DPRINTF(3, "\n"); DPRINTF(1, "Hit bottom in abstract interpreter\n"); _Py_uop_abstractcontext_fini(ctx); + OPT_STAT_INC(optimizer_contradiction); return 0; } diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 728e24cc5b2a09..8a71eff465e5a3 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -820,6 +820,7 @@ _Py_uop_frame_new( if (ctx->curr_frame_depth >= MAX_ABSTRACT_FRAME_DEPTH) { ctx->done = true; ctx->out_of_space = true; + OPT_STAT_INC(optimizer_frame_overflow); return NULL; } _Py_UOpsAbstractFrame *frame = &ctx->frames[ctx->curr_frame_depth]; From 308ccc053a5e4b12345f0fe730cef364ef69f56b Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Thu, 13 Nov 2025 16:10:03 +0000 Subject: [PATCH 187/190] Fix off-by-one --- Python/optimizer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/optimizer.c b/Python/optimizer.c index 2f6c2eb8f05eb3..184e7aa951d237 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -619,7 +619,7 @@ _PyJit_translate_single_bytecode_to_trace( /* Special case the first instruction, * so that we can guarantee forward progress */ - if (progress_needed && _tstate->jit_tracer_state.prev_state.code_curr_size <= CODE_SIZE_NO_PROGRESS) { + if (progress_needed && _tstate->jit_tracer_state.prev_state.code_curr_size < CODE_SIZE_NO_PROGRESS) { if (OPCODE_HAS_EXIT(opcode) || OPCODE_HAS_DEOPT(opcode)) { opcode = _PyOpcode_Deopt[opcode]; } From 33cf28721a2e589b8db5aea21c13258ce5376439 Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Thu, 13 Nov 2025 16:13:53 +0000 Subject: [PATCH 188/190] address review --- Python/optimizer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/optimizer.c b/Python/optimizer.c index 184e7aa951d237..bb7ea57bb926b1 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -968,7 +968,7 @@ _PyJit_TryInitializeTracing( _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; // A recursive trace. // Don't trace into the inner call because it will stomp on the previous trace, causing endless retraces. - if (_tstate->jit_tracer_state.prev_state.code_curr_size > CODE_SIZE_NO_PROGRESS) { + if (_tstate->jit_tracer_state.prev_state.code_curr_size > CODE_SIZE_EMPTY) { return 0; } if (oparg > 0xFFFF) { From 3163d9b117f8968bc3b3d4101dad04fab1d250f5 Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Thu, 13 Nov 2025 17:11:18 +0000 Subject: [PATCH 189/190] Fix a bug in dependencies detected by CI --- Python/optimizer.c | 21 +++++++-------------- Python/optimizer_bytecodes.c | 5 ++--- Python/optimizer_cases.c.h | 3 --- 3 files changed, 9 insertions(+), 20 deletions(-) diff --git a/Python/optimizer.c b/Python/optimizer.c index bb7ea57bb926b1..124b3d365cb80c 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -588,7 +588,10 @@ _PyJit_translate_single_bytecode_to_trace( PyCodeObject *old_code = _tstate->jit_tracer_state.prev_state.instr_code; bool progress_needed = (_tstate->jit_tracer_state.initial_state.chain_depth % MAX_CHAIN_DEPTH) == 0; _PyBloomFilter *dependencies = &_tstate->jit_tracer_state.prev_state.dependencies; - _Py_BloomFilter_Add(dependencies, old_code); + // Can be NULL for the entry frame. + if (old_code != NULL) { + _Py_BloomFilter_Add(dependencies, old_code); + } int trace_length = _tstate->jit_tracer_state.prev_state.code_curr_size; _PyUOpInstruction *trace = _tstate->jit_tracer_state.code_buffer; int max_length = _tstate->jit_tracer_state.prev_state.code_max_size; @@ -1348,17 +1351,7 @@ uop_optimize( bool progress_needed) { _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; - // Note: the executor has a slightly different set of dependencies than the tracer. - // For example: the tracer depends on function and code objects. - // The executor may only depend on the code object. - // Furthermore, it may decide to cut the trace early, meaning it does not depend on the rest - // of the code objects in the trace. - // It is crucial we differentiate them for performance reasons. - // This prevents endless re-tracing for nested functions. - // It is the optimizer's responsibility to add the dependencies it requires on its own. - _PyBloomFilter new_dependencies; - _Py_BloomFilter_Init(&new_dependencies); - _Py_BloomFilter_Add(&new_dependencies, _tstate->jit_tracer_state.initial_state.code); + _PyBloomFilter *dependencies = &_tstate->jit_tracer_state.prev_state.dependencies; _PyUOpInstruction *buffer = _tstate->jit_tracer_state.code_buffer; OPT_STAT_INC(attempts); char *env_var = Py_GETENV("PYTHON_UOPS_OPTIMIZE"); @@ -1378,7 +1371,7 @@ uop_optimize( length = _Py_uop_analyze_and_optimize( _tstate->jit_tracer_state.initial_state.func, buffer,length, - curr_stackentries, &new_dependencies); + curr_stackentries, dependencies); if (length <= 0) { return length; } @@ -1402,7 +1395,7 @@ uop_optimize( length = prepare_for_execution(buffer, length); assert(length <= UOP_MAX_TRACE_LENGTH); _PyExecutorObject *executor = make_executor_from_uops( - buffer, length, &new_dependencies, _tstate->jit_tracer_state.initial_state.chain_depth); + buffer, length, dependencies, _tstate->jit_tracer_state.initial_state.chain_depth); if (executor == NULL) { return -1; } diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index b12ee02557d924..06fa8a4522a499 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -796,7 +796,6 @@ dummy_func(void) { ctx->done = true; break; } - _Py_BloomFilter_Add(dependencies, returning_code); int returning_stacklevel = this_instr->operand1; if (ctx->curr_frame_depth >= 2) { PyCodeObject *expected_code = ctx->frames[ctx->curr_frame_depth - 2].code; @@ -900,8 +899,8 @@ dummy_func(void) { } if (!(operand & 1)) { PyFunctionObject *func = (PyFunctionObject *)operand; - PyCodeObject *co = (PyCodeObject *)func->func_code; - _Py_BloomFilter_Add(dependencies, co); + // No need to re-add to dependencies here. Already + // handled by the tracer. ctx->frame->func = func; } // Fixed calls don't need IP guards. diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index ce5332814ced5b..01263fe8c7a78f 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1121,7 +1121,6 @@ ctx->done = true; break; } - _Py_BloomFilter_Add(dependencies, returning_code); int returning_stacklevel = this_instr->operand1; if (ctx->curr_frame_depth >= 2) { PyCodeObject *expected_code = ctx->frames[ctx->curr_frame_depth - 2].code; @@ -2641,8 +2640,6 @@ } if (!(operand & 1)) { PyFunctionObject *func = (PyFunctionObject *)operand; - PyCodeObject *co = (PyCodeObject *)func->func_code; - _Py_BloomFilter_Add(dependencies, co); ctx->frame->func = func; } if ((this_instr-1)->opcode == _SAVE_RETURN_OFFSET || From 658fd981d56d0be81ae077fd4523ec9d9a6957dc Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Thu, 13 Nov 2025 17:14:14 +0000 Subject: [PATCH 190/190] add code object only after we add to trace --- Python/optimizer.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Python/optimizer.c b/Python/optimizer.c index 124b3d365cb80c..65007a256d0c3b 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -588,10 +588,6 @@ _PyJit_translate_single_bytecode_to_trace( PyCodeObject *old_code = _tstate->jit_tracer_state.prev_state.instr_code; bool progress_needed = (_tstate->jit_tracer_state.initial_state.chain_depth % MAX_CHAIN_DEPTH) == 0; _PyBloomFilter *dependencies = &_tstate->jit_tracer_state.prev_state.dependencies; - // Can be NULL for the entry frame. - if (old_code != NULL) { - _Py_BloomFilter_Add(dependencies, old_code); - } int trace_length = _tstate->jit_tracer_state.prev_state.code_curr_size; _PyUOpInstruction *trace = _tstate->jit_tracer_state.code_buffer; int max_length = _tstate->jit_tracer_state.prev_state.code_max_size; @@ -744,6 +740,10 @@ _PyJit_translate_single_bytecode_to_trace( ADD_TO_TRACE(_SET_IP, 0, (uintptr_t)target_instr, target); } + // Can be NULL for the entry frame. + if (old_code != NULL) { + _Py_BloomFilter_Add(dependencies, old_code); + } switch (opcode) { case POP_JUMP_IF_NONE: