#include "Python.h"

#include "pycore_abstract.h" // _PyIndex_Check()

#include "pycore_backoff.h"

#include "pycore_cell.h" // PyCell_GetRef()

#include "pycore_code.h"

#include "pycore_emscripten_signal.h" // _Py_CHECK_EMSCRIPTEN_SIGNALS

#include "pycore_function.h"

#include "pycore_instruments.h"

#include "pycore_intrinsics.h"

#include "pycore_long.h" // _PyLong_GetZero()

#include "pycore_moduleobject.h" // PyModuleObject

#include "pycore_object.h" // _PyObject_GC_TRACK()

#include "pycore_opcode_metadata.h" // uop names

#include "pycore_opcode_utils.h" // MAKE_FUNCTION_*

#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_*

#include "pycore_pyerrors.h" // _PyErr_GetRaisedException()

#include "pycore_pystate.h" // _PyInterpreterState_GET()

#include "pycore_range.h" // _PyRangeIterObject

#include "pycore_setobject.h" // _PySet_NextEntry()

#include "pycore_sliceobject.h" // _PyBuildSlice_ConsumeRefs

#include "pycore_sysmodule.h" // _PySys_Audit()

#include "pycore_tuple.h" // _PyTuple_ITEMS()

#include "pycore_typeobject.h" // _PySuper_Lookup()

#include "pycore_dict.h"

#include "dictobject.h"

#include "pycore_frame.h"

#include "opcode.h"

#include "optimizer.h"

#include "pydtrace.h"

#include "setobject.h"

#define USE_COMPUTED_GOTOS 0

#include "ceval_macros.h"

#define GO_TO_INSTRUCTION(instname) ((void)0)

#define inst(name, ...) case name:

#define op(name, ...) /* NAME is ignored */

#define macro(name) static int MACRO_##name

#define super(name) static int SUPER_##name

#define family(name, ...) static int family_##name

#define pseudo(name) static int pseudo_##name

#define guard

#define override

#define specializing

#define split

#define replicate(TIMES)

static PyObject *value, *value1, *value2, *left, *right, *res, *sum, *prod, *sub;

static PyObject *container, *start, *stop, *v, *lhs, *rhs, *res2;

static PyObject *list, *tuple, *dict, *owner, *set, *str, *tup, *map, *keys;

static PyObject *exit_func, *lasti, *val, *retval, *obj, *iter, *exhausted;

static PyObject *aiter, *awaitable, *iterable, *w, *exc_value, *bc, *locals;

static PyObject *orig, *excs, *update, *b, *fromlist, *level, *from;

static PyObject **pieces, **values;

static size_t jump;

static uint16_t invert, counter, index, hint;

#define unused 0 // Used in a macro def, can't be static

static uint32_t type_version;

static _PyExecutorObject *current_executor;

static PyObject *

dummy_func(

)

{

// Dummy labels.

pop_1_error:

// Dummy locals.

PyObject *dummy;

_Py_CODEUNIT *this_instr;

PyObject *attr;

PyObject *attrs;

PyObject *bottom;

PyObject *callable;

PyObject *callargs;

PyObject *codeobj;

PyObject *cond;

PyObject *descr;

_PyInterpreterFrame entry_frame;

PyObject *exc;

PyObject *exit;

PyObject *fget;

PyObject *fmt_spec;

PyObject *func;

uint32_t func_version;

PyObject *getattribute;

PyObject *kwargs;

PyObject *kwdefaults;

PyObject *len_o;

PyObject *match;

PyObject *match_type;

PyObject *method;

PyObject *mgr;

Py_ssize_t min_args;

PyObject *names;

PyObject *new_exc;

PyObject *next;

PyObject *none;

PyObject *null;

PyObject *prev_exc;

PyObject *receiver;

PyObject *rest;

int result;

PyObject *self;

PyObject *seq;

PyObject *slice;

PyObject *step;

PyObject *subject;

PyObject *top;

PyObject *type;

PyObject *typevars;

PyObject *val0;

PyObject *val1;

int values_or_none;

switch (opcode) {

pure inst(NOP, (--)) {

}

family(RESUME, 0) = {

RESUME_CHECK,

};

tier1 inst(RESUME, (--)) {

assert(frame == tstate->current_frame);

if (tstate->tracing == 0) {

uintptr_t global_version =

_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) &

~_PY_EVAL_EVENTS_MASK;

PyCodeObject* code = _PyFrame_GetCode(frame);

uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(code->_co_instrumentation_version);

assert((code_version & 255) == 0);

if (code_version != global_version) {

int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp);

ERROR_IF(err, error);

next_instr = this_instr;

DISPATCH();

}

assert(this_instr->op.code == RESUME ||

this_instr->op.code == RESUME_CHECK ||

this_instr->op.code == INSTRUMENTED_RESUME ||

this_instr->op.code == ENTER_EXECUTOR);

if (this_instr->op.code == RESUME) {

#if ENABLE_SPECIALIZATION

FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK);

#endif /* ENABLE_SPECIALIZATION */

}

}

if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) {

CHECK_EVAL_BREAKER();

}

}

inst(RESUME_CHECK, (--)) {

#if defined(__EMSCRIPTEN__)

DEOPT_IF(_Py_emscripten_signal_clock == 0);

_Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING;

uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker);

uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version);

assert((version & _PY_EVAL_EVENTS_MASK) == 0);

DEOPT_IF(eval_breaker != version);

}

inst(INSTRUMENTED_RESUME, (--)) {

uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK;

uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version);

if (code_version != global_version && tstate->tracing == 0) {

if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) {

ERROR_NO_POP();

}

next_instr = this_instr;

}

else {

if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) {

CHECK_EVAL_BREAKER();

}

_PyFrame_SetStackPointer(frame, stack_pointer);

int err = _Py_call_instrumentation(

tstate, oparg > 0, frame, this_instr);

stack_pointer = _PyFrame_GetStackPointer(frame);

ERROR_IF(err, error);

if (frame->instr_ptr != this_instr) {

/* Instrumentation has jumped */

next_instr = frame->instr_ptr;

DISPATCH();

}

}

}

pseudo(LOAD_CLOSURE, (-- unused)) = {

LOAD_FAST,

};

inst(LOAD_FAST_CHECK, (-- value)) {

value = GETLOCAL(oparg);

if (value == NULL) {

_PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError,

UNBOUNDLOCAL_ERROR_MSG,

PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)

);

ERROR_IF(1, error);

}

Py_INCREF(value);

}

replicate(8) pure inst(LOAD_FAST, (-- value)) {

value = GETLOCAL(oparg);

assert(value != NULL);

Py_INCREF(value);

}

inst(LOAD_FAST_AND_CLEAR, (-- value)) {

value = GETLOCAL(oparg);

// do not use SETLOCAL here, it decrefs the old value

GETLOCAL(oparg) = NULL;

}

inst(LOAD_FAST_LOAD_FAST, ( -- value1, value2)) {

uint32_t oparg1 = oparg >> 4;

uint32_t oparg2 = oparg & 15;

value1 = GETLOCAL(oparg1);

value2 = GETLOCAL(oparg2);

Py_INCREF(value1);

Py_INCREF(value2);

}

pure inst(LOAD_CONST, (-- value)) {

value = GETITEM(FRAME_CO_CONSTS, oparg);

Py_INCREF(value);

}

replicate(8) inst(STORE_FAST, (value --)) {

SETLOCAL(oparg, value);

}

pseudo(STORE_FAST_MAYBE_NULL, (unused --)) = {

STORE_FAST,

};

inst(STORE_FAST_LOAD_FAST, (value1 -- value2)) {

uint32_t oparg1 = oparg >> 4;

uint32_t oparg2 = oparg & 15;

SETLOCAL(oparg1, value1);

value2 = GETLOCAL(oparg2);

Py_INCREF(value2);

}

inst(STORE_FAST_STORE_FAST, (value2, value1 --)) {

uint32_t oparg1 = oparg >> 4;

uint32_t oparg2 = oparg & 15;

SETLOCAL(oparg1, value1);

SETLOCAL(oparg2, value2);

}

pure inst(POP_TOP, (value --)) {

DECREF_INPUTS();

}

pure inst(PUSH_NULL, (-- res)) {

res = NULL;

}

macro(END_FOR) = POP_TOP;

tier1 inst(INSTRUMENTED_END_FOR, (receiver, value -- receiver)) {

/* Need to create a fake StopIteration error here,

if (PyGen_Check(receiver)) {

SAVE_SP();

int err = monitor_stop_iteration(tstate, frame, this_instr, value);

LOAD_SP();

if (err) {

ERROR_NO_POP();

}

}

DECREF_INPUTS();

}

pure inst(END_SEND, (receiver, value -- value)) {

Py_DECREF(receiver);

}

tier1 inst(INSTRUMENTED_END_SEND, (receiver, value -- value)) {

if (PyGen_Check(receiver) || PyCoro_CheckExact(receiver)) {

SAVE_SP();

int err = monitor_stop_iteration(tstate, frame, this_instr, value);

LOAD_SP();

if (err) {

ERROR_NO_POP();

}

}

Py_DECREF(receiver);

}

inst(UNARY_NEGATIVE, (value -- res)) {

SAVE_SP();

res = PyNumber_Negative(value);

LOAD_SP();

DECREF_INPUTS();

ERROR_IF(res == NULL, error);

}

pure inst(UNARY_NOT, (value -- res)) {

assert(PyBool_Check(value));

res = Py_IsFalse(value) ? Py_True : Py_False;

}

family(TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL) = {

TO_BOOL_ALWAYS_TRUE,

TO_BOOL_BOOL,

TO_BOOL_INT,

TO_BOOL_LIST,

TO_BOOL_NONE,

TO_BOOL_STR,

};

specializing op(_SPECIALIZE_TO_BOOL, (counter/1, value -- value)) {

#if ENABLE_SPECIALIZATION

if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {

next_instr = this_instr;

SAVE_SP();

_Py_Specialize_ToBool(value, next_instr);

LOAD_SP();

DISPATCH_SAME_OPARG();

}

STAT_INC(TO_BOOL, deferred);

ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);

#endif /* ENABLE_SPECIALIZATION */

}

op(_TO_BOOL, (value -- res)) {

SAVE_SP();

int err = PyObject_IsTrue(value);

LOAD_SP();

DECREF_INPUTS();

ERROR_IF(err < 0, error);

res = err ? Py_True : Py_False;

}

macro(TO_BOOL) = _SPECIALIZE_TO_BOOL + unused/2 + _TO_BOOL;

inst(TO_BOOL_BOOL, (unused/1, unused/2, value -- value)) {

EXIT_IF(!PyBool_Check(value));

STAT_INC(TO_BOOL, hit);

}

inst(TO_BOOL_INT, (unused/1, unused/2, value -- res)) {

EXIT_IF(!PyLong_CheckExact(value));

STAT_INC(TO_BOOL, hit);

if (_PyLong_IsZero((PyLongObject *)value)) {

assert(_Py_IsImmortal(value));

res = Py_False;

}

else {

DECREF_INPUTS();

res = Py_True;

}

}

inst(TO_BOOL_LIST, (unused/1, unused/2, value -- res)) {

EXIT_IF(!PyList_CheckExact(value));

STAT_INC(TO_BOOL, hit);

res = Py_SIZE(value) ? Py_True : Py_False;

DECREF_INPUTS();

}

inst(TO_BOOL_NONE, (unused/1, unused/2, value -- res)) {

// This one is a bit weird, because we expect *some* failures:

EXIT_IF(!Py_IsNone(value));

STAT_INC(TO_BOOL, hit);

res = Py_False;

}

inst(TO_BOOL_STR, (unused/1, unused/2, value -- res)) {

EXIT_IF(!PyUnicode_CheckExact(value));

STAT_INC(TO_BOOL, hit);

if (value == &_Py_STR(empty)) {

assert(_Py_IsImmortal(value));

res = Py_False;

}

else {

assert(Py_SIZE(value));

DECREF_INPUTS();

res = Py_True;

}

}

op(_REPLACE_WITH_TRUE, (value -- res)) {

Py_DECREF(value);

res = Py_True;

}

macro(TO_BOOL_ALWAYS_TRUE) =

unused/1 +

_GUARD_TYPE_VERSION +

_REPLACE_WITH_TRUE;

inst(UNARY_INVERT, (value -- res)) {

SAVE_SP();

res = PyNumber_Invert(value);

LOAD_SP();

DECREF_INPUTS();

ERROR_IF(res == NULL, error);

}

family(BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP) = {

BINARY_OP_MULTIPLY_INT,

BINARY_OP_ADD_INT,

BINARY_OP_SUBTRACT_INT,

BINARY_OP_MULTIPLY_FLOAT,

BINARY_OP_ADD_FLOAT,

BINARY_OP_SUBTRACT_FLOAT,

BINARY_OP_ADD_UNICODE,

// BINARY_OP_INPLACE_ADD_UNICODE, // See comments at that opcode.

};

op(_GUARD_BOTH_INT, (left, right -- left, right)) {

EXIT_IF(!PyLong_CheckExact(left));

EXIT_IF(!PyLong_CheckExact(right));

}

op(_GUARD_NOS_INT, (left, unused -- left, unused)) {

EXIT_IF(!PyLong_CheckExact(left));

}

op(_GUARD_TOS_INT, (value -- value)) {

EXIT_IF(!PyLong_CheckExact(value));

}

pure op(_BINARY_OP_MULTIPLY_INT, (left, right -- res)) {

STAT_INC(BINARY_OP, hit);

res = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right);

_Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);

_Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);

ERROR_IF(res == NULL, error);

}

pure op(_BINARY_OP_ADD_INT, (left, right -- res)) {

STAT_INC(BINARY_OP, hit);

res = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right);

_Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);

_Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);

ERROR_IF(res == NULL, error);

}

pure op(_BINARY_OP_SUBTRACT_INT, (left, right -- res)) {

STAT_INC(BINARY_OP, hit);

res = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right);

_Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);

_Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);

ERROR_IF(res == NULL, error);

}

macro(BINARY_OP_MULTIPLY_INT) =

_GUARD_BOTH_INT + unused/1 + _BINARY_OP_MULTIPLY_INT;

macro(BINARY_OP_ADD_INT) =

_GUARD_BOTH_INT + unused/1 + _BINARY_OP_ADD_INT;

macro(BINARY_OP_SUBTRACT_INT) =

_GUARD_BOTH_INT + unused/1 + _BINARY_OP_SUBTRACT_INT;

op(_GUARD_BOTH_FLOAT, (left, right -- left, right)) {

EXIT_IF(!PyFloat_CheckExact(left));

EXIT_IF(!PyFloat_CheckExact(right));

}

op(_GUARD_NOS_FLOAT, (left, unused -- left, unused)) {

EXIT_IF(!PyFloat_CheckExact(left));

}

op(_GUARD_TOS_FLOAT, (value -- value)) {

EXIT_IF(!PyFloat_CheckExact(value));

}

pure op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res)) {

STAT_INC(BINARY_OP, hit);

double dres =

((PyFloatObject *)left)->ob_fval *

((PyFloatObject *)right)->ob_fval;

DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res);

}

pure op(_BINARY_OP_ADD_FLOAT, (left, right -- res)) {

STAT_INC(BINARY_OP, hit);

double dres =

((PyFloatObject *)left)->ob_fval +

((PyFloatObject *)right)->ob_fval;

DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res);

}

pure op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res)) {

STAT_INC(BINARY_OP, hit);

double dres =

((PyFloatObject *)left)->ob_fval -

((PyFloatObject *)right)->ob_fval;

DECREF_INPUTS_AND_REUSE_FLOAT(left, right, dres, res);

}

macro(BINARY_OP_MULTIPLY_FLOAT) =

_GUARD_BOTH_FLOAT + unused/1 + _BINARY_OP_MULTIPLY_FLOAT;

macro(BINARY_OP_ADD_FLOAT) =

_GUARD_BOTH_FLOAT + unused/1 + _BINARY_OP_ADD_FLOAT;

macro(BINARY_OP_SUBTRACT_FLOAT) =

_GUARD_BOTH_FLOAT + unused/1 + _BINARY_OP_SUBTRACT_FLOAT;

op(_GUARD_BOTH_UNICODE, (left, right -- left, right)) {

EXIT_IF(!PyUnicode_CheckExact(left));

EXIT_IF(!PyUnicode_CheckExact(right));

}

pure op(_BINARY_OP_ADD_UNICODE, (left, right -- res)) {

STAT_INC(BINARY_OP, hit);

res = PyUnicode_Concat(left, right);

_Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc);

_Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc);

ERROR_IF(res == NULL, error);

}

macro(BINARY_OP_ADD_UNICODE) =

_GUARD_BOTH_UNICODE + unused/1 + _BINARY_OP_ADD_UNICODE;

// This is a subtle one. It's a super-instruction for

// BINARY_OP_ADD_UNICODE followed by STORE_FAST

// where the store goes into the left argument.

// So the inputs are the same as for all BINARY_OP

// specializations, but there is no output.

// At the end we just skip over the STORE_FAST.

tier1 op(_BINARY_OP_INPLACE_ADD_UNICODE, (left, right --)) {

assert(next_instr->op.code == STORE_FAST);

PyObject **target_local = &GETLOCAL(next_instr->op.arg);

DEOPT_IF(*target_local != left);

STAT_INC(BINARY_OP, hit);

/* Handle `left = left + right` or `left += right` for str.

assert(Py_REFCNT(left) >= 2);

_Py_DECREF_NO_DEALLOC(left);

SAVE_SP();

PyUnicode_Append(target_local, right);

LOAD_SP();

_Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc);

ERROR_IF(*target_local == NULL, error);

// The STORE_FAST is already done.

assert(next_instr->op.code == STORE_FAST);

SKIP_OVER(1);

}

macro(BINARY_OP_INPLACE_ADD_UNICODE) =

_GUARD_BOTH_UNICODE + unused/1 + _BINARY_OP_INPLACE_ADD_UNICODE;

family(BINARY_SUBSCR, INLINE_CACHE_ENTRIES_BINARY_SUBSCR) = {

BINARY_SUBSCR_DICT,

BINARY_SUBSCR_GETITEM,

BINARY_SUBSCR_LIST_INT,

BINARY_SUBSCR_STR_INT,

BINARY_SUBSCR_TUPLE_INT,

};

specializing op(_SPECIALIZE_BINARY_SUBSCR, (counter/1, container, sub -- container, sub)) {

#if ENABLE_SPECIALIZATION

if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {

next_instr = this_instr;

SAVE_SP();

_Py_Specialize_BinarySubscr(container, sub, next_instr);

LOAD_SP();

DISPATCH_SAME_OPARG();

}

STAT_INC(BINARY_SUBSCR, deferred);

ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);

#endif /* ENABLE_SPECIALIZATION */

}

op(_BINARY_SUBSCR, (container, sub -- res)) {

SAVE_SP();

res = PyObject_GetItem(container, sub);

LOAD_SP();

DECREF_INPUTS();

ERROR_IF(res == NULL, error);

}

macro(BINARY_SUBSCR) = _SPECIALIZE_BINARY_SUBSCR + _BINARY_SUBSCR;

inst(BINARY_SLICE, (container, start, stop -- res)) {

SAVE_SP();

PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop);

LOAD_SP();

// Can't use ERROR_IF() here, because we haven't

// DECREF'ed container yet, and we still own slice.

if (slice == NULL) {

res = NULL;

}

else {

SAVE_SP();

res = PyObject_GetItem(container, slice);

LOAD_SP();

Py_DECREF(slice);

}

Py_DECREF(container);

ERROR_IF(res == NULL, error);

}

inst(STORE_SLICE, (v, container, start, stop -- )) {

PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop);

int err;

if (slice == NULL) {

err = 1;

}

else {

SAVE_SP();

err = PyObject_SetItem(container, slice, v);

LOAD_SP();

Py_DECREF(slice);

}

Py_DECREF(v);

Py_DECREF(container);

ERROR_IF(err, error);

}

inst(BINARY_SUBSCR_LIST_INT, (unused/1, list, sub -- res)) {

DEOPT_IF(!PyLong_CheckExact(sub));

DEOPT_IF(!PyList_CheckExact(list));

// Deopt unless 0 <= sub < PyList_Size(list)

DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub));

Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];

DEOPT_IF(index >= PyList_GET_SIZE(list));

STAT_INC(BINARY_SUBSCR, hit);

res = PyList_GET_ITEM(list, index);

assert(res != NULL);

Py_INCREF(res);

_Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);

Py_DECREF(list);

}

inst(BINARY_SUBSCR_STR_INT, (unused/1, str, sub -- res)) {

DEOPT_IF(!PyLong_CheckExact(sub));

DEOPT_IF(!PyUnicode_CheckExact(str));

DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub));

Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];

DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index);

// Specialize for reading an ASCII character from any string:

Py_UCS4 c = PyUnicode_READ_CHAR(str, index);

DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c);

STAT_INC(BINARY_SUBSCR, hit);

res = (PyObject*)&_Py_SINGLETON(strings).ascii[c];

_Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);

Py_DECREF(str);

}

inst(BINARY_SUBSCR_TUPLE_INT, (unused/1, tuple, sub -- res)) {

DEOPT_IF(!PyLong_CheckExact(sub));

DEOPT_IF(!PyTuple_CheckExact(tuple));

// Deopt unless 0 <= sub < PyTuple_Size(list)

DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub));

Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];

DEOPT_IF(index >= PyTuple_GET_SIZE(tuple));

STAT_INC(BINARY_SUBSCR, hit);

res = PyTuple_GET_ITEM(tuple, index);

assert(res != NULL);

Py_INCREF(res);

_Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);

Py_DECREF(tuple);

}

inst(BINARY_SUBSCR_DICT, (unused/1, dict, sub -- res)) {

DEOPT_IF(!PyDict_CheckExact(dict));

STAT_INC(BINARY_SUBSCR, hit);

SAVE_SP();

int rc = PyDict_GetItemRef(dict, sub, &res);

LOAD_SP();

if (rc == 0) {

_PyErr_SetKeyError(sub);

}

DECREF_INPUTS();

ERROR_IF(rc <= 0, error); // not found or error

}

inst(BINARY_SUBSCR_GETITEM, (unused/1, container, sub -- unused)) {

DEOPT_IF(tstate->interp->eval_frame);

PyTypeObject *tp = Py_TYPE(container);

DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE));

PyHeapTypeObject *ht = (PyHeapTypeObject *)tp;

PyObject *cached = ht->_spec_cache.getitem;

DEOPT_IF(cached == NULL);

assert(PyFunction_Check(cached));

PyFunctionObject *getitem = (PyFunctionObject *)cached;

uint32_t cached_version = ht->_spec_cache.getitem_version;

DEOPT_IF(getitem->func_version != cached_version);

PyCodeObject *code = (PyCodeObject *)getitem->func_code;

assert(code->co_argcount == 2);

DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize));

STAT_INC(BINARY_SUBSCR, hit);

Py_INCREF(getitem);

_PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2);

STACK_SHRINK(2);

new_frame->localsplus[0] = container;

new_frame->localsplus[1] = sub;

frame->return_offset = (uint16_t)(next_instr - this_instr);

DISPATCH_INLINED(new_frame);

}

inst(LIST_APPEND, (list, unused[oparg-1], v -- list, unused[oparg-1])) {

ERROR_IF(_PyList_AppendTakeRef((PyListObject *)list, v) < 0, error);

}

inst(SET_ADD, (set, unused[oparg-1], v -- set, unused[oparg-1])) {

SAVE_SP();

int err = PySet_Add(set, v);

LOAD_SP();

DECREF_INPUTS();

ERROR_IF(err, error);

}

family(STORE_SUBSCR, INLINE_CACHE_ENTRIES_STORE_SUBSCR) = {

STORE_SUBSCR_DICT,

STORE_SUBSCR_LIST_INT,

};

specializing op(_SPECIALIZE_STORE_SUBSCR, (counter/1, container, sub -- container, sub)) {

#if ENABLE_SPECIALIZATION

if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {

next_instr = this_instr;

SAVE_SP();

_Py_Specialize_StoreSubscr(container, sub, next_instr);

LOAD_SP();

DISPATCH_SAME_OPARG();

}

STAT_INC(STORE_SUBSCR, deferred);

ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);

#endif /* ENABLE_SPECIALIZATION */

}

op(_STORE_SUBSCR, (v, container, sub -- )) {

/* container[sub] = v */

SAVE_SP();

int err = PyObject_SetItem(container, sub, v);

LOAD_SP();

DECREF_INPUTS();

ERROR_IF(err, error);

}

macro(STORE_SUBSCR) = _SPECIALIZE_STORE_SUBSCR + _STORE_SUBSCR;

inst(STORE_SUBSCR_LIST_INT, (unused/1, value, list, sub -- )) {

DEOPT_IF(!PyLong_CheckExact(sub));

DEOPT_IF(!PyList_CheckExact(list));

// Ensure nonnegative, zero-or-one-digit ints.

DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub));

Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];

// Ensure index < len(list)

DEOPT_IF(index >= PyList_GET_SIZE(list));

STAT_INC(STORE_SUBSCR, hit);

PyObject *old_value = PyList_GET_ITEM(list, index);

PyList_SET_ITEM(list, index, value);

assert(old_value != NULL);

Py_DECREF(old_value);

_Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);

Py_DECREF(list);

}

inst(STORE_SUBSCR_DICT, (unused/1, value, dict, sub -- )) {

DEOPT_IF(!PyDict_CheckExact(dict));

STAT_INC(STORE_SUBSCR, hit);

SAVE_SP();

int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value);

LOAD_SP();

Py_DECREF(dict);

ERROR_IF(err, error);

}

inst(DELETE_SUBSCR, (container, sub --)) {

/* del container[sub] */

SAVE_SP();

int err = PyObject_DelItem(container, sub);

LOAD_SP();

DECREF_INPUTS();

ERROR_IF(err, error);

}

inst(CALL_INTRINSIC_1, (value -- res)) {

assert(oparg <= MAX_INTRINSIC_1);

SAVE_SP();

res = _PyIntrinsics_UnaryFunctions[oparg].func(tstate, value);

LOAD_SP();

DECREF_INPUTS();

ERROR_IF(res == NULL, error);

}

inst(CALL_INTRINSIC_2, (value2, value1 -- res)) {

assert(oparg <= MAX_INTRINSIC_2);

SAVE_SP();

res = _PyIntrinsics_BinaryFunctions[oparg].func(tstate, value2, value1);

LOAD_SP();

DECREF_INPUTS();

ERROR_IF(res == NULL, error);

}

tier1 inst(RAISE_VARARGS, (args[oparg] -- )) {

PyObject *cause = NULL, *exc = NULL;

SAVE_SP();

switch (oparg) {

case 2:

cause = args[1];

/* fall through */

case 1:

exc = args[0];

/* fall through */

case 0:

if (do_raise(tstate, exc, cause)) {

assert(oparg == 0);

monitor_reraise(tstate, frame, this_instr);

LOAD_SP();

goto exception_unwind;

}

break;

default:

_PyErr_SetString(tstate, PyExc_SystemError,

"bad RAISE_VARARGS oparg");

break;

}

LOAD_SP();

ERROR_IF(true, error);

}

tier1 inst(INTERPRETER_EXIT, (retval --)) {

assert(frame == &entry_frame);

assert(_PyFrame_IsIncomplete(frame));

/* Restore previous frame and return. */

SAVE_SP();

tstate->current_frame = frame->previous;

assert(!_PyErr_Occurred(tstate));

tstate->c_recursion_remaining += PY_EVAL_C_STACK_UNITS;

return retval;

}

// The stack effect here is ambiguous.

// We definitely pop the return value off the stack on entry.

// We also push it onto the stack on exit, but that's a

// different frame, and it's accounted for by _PUSH_FRAME.

op(_POP_FRAME, (retval --)) {

#if TIER_ONE

assert(frame != &entry_frame);

#endif

SYNC_SP();

_PyFrame_SetStackPointer(frame, stack_pointer);

assert(EMPTY());

_Py_LeaveRecursiveCallPy(tstate);

// GH-99729: We need to unlink the frame *before* clearing it:

_PyInterpreterFrame *dying = frame;

frame = tstate->current_frame = dying->previous;

_PyEval_FrameClearAndPop(tstate, dying);

_PyFrame_StackPush(frame, retval);

LOAD_SP();

LOAD_IP(frame->return_offset);

LLTRACE_RESUME_FRAME();

}

macro(RETURN_VALUE) =

_POP_FRAME;

inst(INSTRUMENTED_RETURN_VALUE, (retval --)) {

SAVE_SP();

int err = _Py_call_instrumentation_arg(

tstate, PY_MONITORING_EVENT_PY_RETURN,

frame, this_instr, retval);

LOAD_SP();

if (err) ERROR_NO_POP();

STACK_SHRINK(1);

assert(EMPTY());

_PyFrame_SetStackPointer(frame, stack_pointer);

_Py_LeaveRecursiveCallPy(tstate);

assert(frame != &entry_frame);

// GH-99729: We need to unlink the frame *before* clearing it:

_PyInterpreterFrame *dying = frame;

frame = tstate->current_frame = dying->previous;

_PyEval_FrameClearAndPop(tstate, dying);

_PyFrame_StackPush(frame, retval);

LOAD_IP(frame->return_offset);

goto resume_frame;

}

macro(RETURN_CONST) =

LOAD_CONST +

_POP_FRAME;

inst(INSTRUMENTED_RETURN_CONST, (--)) {

PyObject *retval = GETITEM(FRAME_CO_CONSTS, oparg);

SAVE_SP();

int err = _Py_call_instrumentation_arg(

tstate, PY_MONITORING_EVENT_PY_RETURN,

frame, this_instr, retval);

LOAD_SP();

if (err) ERROR_NO_POP();

Py_INCREF(retval);

assert(EMPTY());

_PyFrame_SetStackPointer(frame, stack_pointer);

_Py_LeaveRecursiveCallPy(tstate);

assert(frame != &entry_frame);

// GH-99729: We need to unlink the frame *before* clearing it:

_PyInterpreterFrame *dying = frame;

frame = tstate->current_frame = dying->previous;

_PyEval_FrameClearAndPop(tstate, dying);

_PyFrame_StackPush(frame, retval);

LOAD_IP(frame->return_offset);

goto resume_frame;

}

inst(GET_AITER, (obj -- iter)) {

unaryfunc getter = NULL;

PyTypeObject *type = Py_TYPE(obj);

if (type->tp_as_async != NULL) {

getter = type->tp_as_async->am_aiter;

}

if (getter == NULL) {

SAVE_SP();

_PyErr_Format(tstate, PyExc_TypeError,

"'async for' requires an object with "

"__aiter__ method, got %.100s",

type->tp_name);

LOAD_SP();

DECREF_INPUTS();

ERROR_IF(true, error);

}

SAVE_SP();

iter = (*getter)(obj);

LOAD_SP();

DECREF_INPUTS();

ERROR_IF(iter == NULL, error);

if (Py_TYPE(iter)->tp_as_async == NULL ||

Py_TYPE(iter)->tp_as_async->am_anext == NULL) {

_PyErr_Format(tstate, PyExc_TypeError,

"'async for' received an object from __aiter__ "

"that does not implement __anext__: %.100s",

Py_TYPE(iter)->tp_name);

Py_DECREF(iter);

ERROR_IF(true, error);

}

}

inst(GET_ANEXT, (aiter -- aiter, awaitable)) {

unaryfunc getter = NULL;

PyObject *next_iter = NULL;

PyTypeObject *type = Py_TYPE(aiter);

if (PyAsyncGen_CheckExact(aiter)) {

SAVE_SP();

awaitable = type->tp_as_async->am_anext(aiter);

LOAD_SP();

if (awaitable == NULL) {

ERROR_NO_POP();

}

} else {

if (type->tp_as_async != NULL){

getter = type->tp_as_async->am_anext;

}

if (getter != NULL) {

SAVE_SP();

next_iter = (*getter)(aiter);