bytecodes.c

// This file contains instruction definitions.
// It is read by generators stored in Tools/cases_generator/
// to generate Python/generated_cases.c.h and others.
// Note that there is some dummy C code at the top and bottom of the file
// to fool text editors like VS Code into believing this is valid C code.
// The actual instruction definitions start at // BEGIN BYTECODES //.
// See Tools/cases_generator/README.md for more information.

#include "Python.h"
#include "pycore_abstract.h"      // _PyIndex_Check()
#include "pycore_audit.h"         // _PySys_Audit()
#include "pycore_backoff.h"
#include "pycore_cell.h"          // PyCell_GetRef()
#include "pycore_ceval.h"
#include "pycore_code.h"
#include "pycore_emscripten_signal.h"  // _Py_CHECK_EMSCRIPTEN_SIGNALS
#include "pycore_function.h"
#include "pycore_instruments.h"
#include "pycore_interpolation.h" // _PyInterpolation_Build()
#include "pycore_intrinsics.h"
#include "pycore_long.h"          // _PyLong_GetZero()
#include "pycore_moduleobject.h"  // PyModuleObject
#include "pycore_object.h"        // _PyObject_GC_TRACK()
#include "pycore_opcode_metadata.h"  // uop names
#include "pycore_opcode_utils.h"  // MAKE_FUNCTION_*
#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_*
#include "pycore_pyerrors.h"      // _PyErr_GetRaisedException()
#include "pycore_pystate.h"       // _PyInterpreterState_GET()
#include "pycore_range.h"         // _PyRangeIterObject
#include "pycore_long.h"          // _PyLong_ExactDealloc()
#include "pycore_setobject.h"     // _PySet_NextEntry()
#include "pycore_sliceobject.h"   // _PyBuildSlice_ConsumeRefs
#include "pycore_stackref.h"
#include "pycore_template.h"      // _PyTemplate_Build()
#include "pycore_tuple.h"         // _PyTuple_ITEMS()
#include "pycore_typeobject.h"    // _PySuper_Lookup()

#include "pycore_dict.h"
#include "dictobject.h"
#include "pycore_frame.h"
#include "opcode.h"
#include "optimizer.h"
#include "pydtrace.h"
#include "setobject.h"


#define USE_COMPUTED_GOTOS 0
#include "ceval_macros.h"

/* Flow control macros */

#define inst(name, ...) case name:
#define op(name, ...) /* NAME is ignored */
#define macro(name) static int MACRO_##name
#define super(name) static int SUPER_##name
#define family(name, ...) static int family_##name
#define pseudo(name) static int pseudo_##name
#define label(name) name:

/* Annotations */
#define guard
#define override
#define specializing
#define replicate(TIMES)
#define tier1
#define no_save_ip

// Dummy variables for stack effects.
static PyObject *value, *value1, *value2, *left, *right, *res, *sum, *prod, *sub;
static PyObject *container, *start, *stop, *v, *lhs, *rhs, *res2;
static PyObject *list, *tuple, *dict, *owner, *set, *str, *tup, *map, *keys;
static PyObject *exit_func, *lasti, *val, *retval, *obj, *iter, *exhausted;
static PyObject *aiter, *awaitable, *iterable, *w, *exc_value, *bc, *locals;
static PyObject *orig, *excs, *update, *b, *fromlist, *level, *from;
static PyObject **pieces, **values;
static size_t jump;
// Dummy variables for cache effects
static uint16_t invert, counter, index, hint;
#define unused 0  // Used in a macro def, can't be static
static uint32_t type_version;
static _PyExecutorObject *current_executor;

static PyObject *
dummy_func(
    PyThreadState *tstate,
    _PyInterpreterFrame *frame,
    unsigned char opcode,
    unsigned int oparg,
    _Py_CODEUNIT *next_instr,
    PyObject **stack_pointer,
    int throwflag,
    PyObject *args[]
)
{
    // Dummy labels.
    pop_1_error:
    // Dummy locals.
    PyObject *dummy;
    _Py_CODEUNIT *this_instr;
    PyObject *attr;
    PyObject *attrs;
    PyObject *bottom;
    PyObject *callable;
    PyObject *callargs;
    PyObject *codeobj;
    PyObject *cond;
    PyObject *descr;
    PyObject *exc;
    PyObject *exit;
    PyObject *fget;
    PyObject *fmt_spec;
    PyObject *func;
    uint32_t func_version;
    PyObject *getattribute;
    PyObject *kwargs;
    PyObject *kwdefaults;
    PyObject *len_o;
    PyObject *match;
    PyObject *match_type;
    PyObject *method;
    PyObject *mgr;
    Py_ssize_t min_args;
    PyObject *names;
    PyObject *new_exc;
    PyObject *next;
    PyObject *none;
    PyObject *null;
    PyObject *prev_exc;
    PyObject *receiver;
    PyObject *rest;
    int result;
    PyObject *self;
    PyObject *seq;
    PyObject *slice;
    PyObject *step;
    PyObject *subject;
    PyObject *top;
    PyObject *type;
    PyObject *typevars;
    PyObject *val0;
    PyObject *val1;
    int values_or_none;

    switch (opcode) {

// BEGIN BYTECODES //
        pure inst(NOP, (--)) {
        }

        family(RESUME, 0) = {
            RESUME_CHECK,
        };

        macro(NOT_TAKEN) = NOP;

        op(_CHECK_PERIODIC, (--)) {
            _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
            QSBR_QUIESCENT_STATE(tstate);
            if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
                int err = _Py_HandlePending(tstate);
                ERROR_IF(err != 0);
            }
        }

        op(_CHECK_PERIODIC_IF_NOT_YIELD_FROM, (--)) {
            if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) {
                _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY();
                QSBR_QUIESCENT_STATE(tstate);
                if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) {
                    int err = _Py_HandlePending(tstate);
                    ERROR_IF(err != 0);
                }
            }
        }

        op(_QUICKEN_RESUME, (--)) {
            #if ENABLE_SPECIALIZATION_FT
            if (tstate->tracing == 0 && this_instr->op.code == RESUME) {
                FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK);
            }
            #endif  /* ENABLE_SPECIALIZATION_FT */
        }

        tier1 op(_MAYBE_INSTRUMENT, (--)) {
            if (tstate->tracing == 0) {
                uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK;
                uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version);
                if (code_version != global_version) {
                    int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp);
                    if (err) {
                        ERROR_NO_POP();
                    }
                    next_instr = this_instr;
                    DISPATCH();
                }
            }
        }

        op(_LOAD_BYTECODE, (--)) {
            #ifdef Py_GIL_DISABLED
            if (frame->tlbc_index !=
                ((_PyThreadStateImpl *)tstate)->tlbc_index) {
                _Py_CODEUNIT *bytecode =
                    _PyEval_GetExecutableCode(tstate, _PyFrame_GetCode(frame));
                ERROR_IF(bytecode == NULL);
                ptrdiff_t off = this_instr - _PyFrame_GetBytecode(frame);
                frame->tlbc_index = ((_PyThreadStateImpl *)tstate)->tlbc_index;
                frame->instr_ptr = bytecode + off;
                // Make sure this_instr gets reset correctly for any uops that
                // follow
                next_instr = frame->instr_ptr;
                DISPATCH();
            }
            #endif
        }

        macro(RESUME) =
            _LOAD_BYTECODE +
            _MAYBE_INSTRUMENT +
            _QUICKEN_RESUME +
            _CHECK_PERIODIC_IF_NOT_YIELD_FROM;

        inst(RESUME_CHECK, (--)) {
#if defined(__EMSCRIPTEN__)
            DEOPT_IF(_Py_emscripten_signal_clock == 0);
            _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING;
#endif
            uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker);
            uintptr_t version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version);
            assert((version & _PY_EVAL_EVENTS_MASK) == 0);
            DEOPT_IF(eval_breaker != version);
            #ifdef Py_GIL_DISABLED
            DEOPT_IF(frame->tlbc_index !=
                     ((_PyThreadStateImpl *)tstate)->tlbc_index);
            #endif
        }

        op(_MONITOR_RESUME, (--)) {
            int err = _Py_call_instrumentation(
                    tstate, oparg > 0, frame, this_instr);
            ERROR_IF(err);
            if (frame->instr_ptr != this_instr) {
                /* Instrumentation has jumped */
                next_instr = frame->instr_ptr;
            }
        }

        macro(INSTRUMENTED_RESUME) =
            _LOAD_BYTECODE +
            _MAYBE_INSTRUMENT +
            _CHECK_PERIODIC_IF_NOT_YIELD_FROM +
            _MONITOR_RESUME;

        pseudo(LOAD_CLOSURE, (-- unused)) = {
            LOAD_FAST,
        };

        inst(LOAD_FAST_CHECK, (-- value)) {
            _PyStackRef value_s = GETLOCAL(oparg);
            if (PyStackRef_IsNull(value_s)) {
                _PyEval_FormatExcCheckArg(tstate, PyExc_UnboundLocalError,
                    UNBOUNDLOCAL_ERROR_MSG,
                    PyTuple_GetItem(_PyFrame_GetCode(frame)->co_localsplusnames, oparg)
                );
                ERROR_IF(true);
            }
            value = PyStackRef_DUP(value_s);
        }

        replicate(8) pure inst(LOAD_FAST, (-- value)) {
            assert(!PyStackRef_IsNull(GETLOCAL(oparg)));
            value = PyStackRef_DUP(GETLOCAL(oparg));
        }

        replicate(8) pure inst (LOAD_FAST_BORROW, (-- value)) {
            assert(!PyStackRef_IsNull(GETLOCAL(oparg)));
            value = PyStackRef_Borrow(GETLOCAL(oparg));
        }

        inst(LOAD_FAST_AND_CLEAR, (-- value)) {
            value = GETLOCAL(oparg);
            GETLOCAL(oparg) = PyStackRef_NULL;
        }

        inst(LOAD_FAST_LOAD_FAST, ( -- value1, value2)) {
            uint32_t oparg1 = oparg >> 4;
            uint32_t oparg2 = oparg & 15;
            value1 = PyStackRef_DUP(GETLOCAL(oparg1));
            value2 = PyStackRef_DUP(GETLOCAL(oparg2));
        }

        inst(LOAD_FAST_BORROW_LOAD_FAST_BORROW, ( -- value1, value2)) {
            uint32_t oparg1 = oparg >> 4;
            uint32_t oparg2 = oparg & 15;
            value1 = PyStackRef_Borrow(GETLOCAL(oparg1));
            value2 = PyStackRef_Borrow(GETLOCAL(oparg2));
        }

        family(LOAD_CONST, 0) = {
            LOAD_CONST_MORTAL,
            LOAD_CONST_IMMORTAL,
        };

        inst(LOAD_CONST, (-- value)) {
            /* We can't do this in the bytecode compiler as
             * marshalling can intern strings and make them immortal. */
            PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg);
            value = PyStackRef_FromPyObjectNew(obj);
#if ENABLE_SPECIALIZATION_FT
#ifdef Py_GIL_DISABLED
            uint8_t expected = LOAD_CONST;
            if (!_Py_atomic_compare_exchange_uint8(
                    &this_instr->op.code, &expected,
                    _Py_IsImmortal(obj) ? LOAD_CONST_IMMORTAL : LOAD_CONST_MORTAL)) {
                // We might lose a race with instrumentation, which we don't care about.
                assert(expected >= MIN_INSTRUMENTED_OPCODE);
            }
#else
            if (this_instr->op.code == LOAD_CONST) {
                this_instr->op.code = _Py_IsImmortal(obj) ? LOAD_CONST_IMMORTAL : LOAD_CONST_MORTAL;
            }
#endif
#endif
        }

        inst(LOAD_CONST_MORTAL, (-- value)) {
            PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg);
            value = PyStackRef_FromPyObjectNewMortal(obj);
        }

        inst(LOAD_CONST_IMMORTAL, (-- value)) {
            PyObject *obj = GETITEM(FRAME_CO_CONSTS, oparg);
            assert(_Py_IsImmortal(obj));
            value = PyStackRef_FromPyObjectImmortal(obj);
        }

        replicate(4) inst(LOAD_SMALL_INT, (-- value)) {
            assert(oparg < _PY_NSMALLPOSINTS);
            PyObject *obj = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + oparg];
            value = PyStackRef_FromPyObjectImmortal(obj);
        }

        replicate(8) inst(STORE_FAST, (value --)) {
            assert(
                ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) ||
                PyStackRef_IsHeapSafe(value)
            );
            _PyStackRef tmp = GETLOCAL(oparg);
            GETLOCAL(oparg) = value;
            DEAD(value);
            PyStackRef_XCLOSE(tmp);
        }

        pseudo(STORE_FAST_MAYBE_NULL, (unused --)) = {
            STORE_FAST,
        };

        inst(STORE_FAST_LOAD_FAST, (value1 -- value2)) {
            assert(
                ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) ||
                PyStackRef_IsHeapSafe(value1)
            );
            uint32_t oparg1 = oparg >> 4;
            uint32_t oparg2 = oparg & 15;
            _PyStackRef tmp = GETLOCAL(oparg1);
            GETLOCAL(oparg1) = value1;
            DEAD(value1);
            value2 = PyStackRef_DUP(GETLOCAL(oparg2));
            PyStackRef_XCLOSE(tmp);
        }

        inst(STORE_FAST_STORE_FAST, (value2, value1 --)) {
            assert(
                ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) ||
                PyStackRef_IsHeapSafe(value1)
            );
            assert(
                ((_PyFrame_GetCode(frame)->co_flags & (CO_COROUTINE | CO_GENERATOR)) == 0) ||
                PyStackRef_IsHeapSafe(value2)
            );
            uint32_t oparg1 = oparg >> 4;
            uint32_t oparg2 = oparg & 15;
            _PyStackRef tmp = GETLOCAL(oparg1);
            GETLOCAL(oparg1) = value1;
            DEAD(value1);
            PyStackRef_XCLOSE(tmp);
            tmp = GETLOCAL(oparg2);
            GETLOCAL(oparg2) = value2;
            DEAD(value2);
            PyStackRef_XCLOSE(tmp);
        }

        pure inst(POP_TOP, (value --)) {
            PyStackRef_CLOSE(value);
        }

        pure inst(PUSH_NULL, (-- res)) {
            res = PyStackRef_NULL;
        }

        no_save_ip inst(END_FOR, (value -- )) {
            /* Don't update instr_ptr, so that POP_ITER sees
             * the FOR_ITER as the previous instruction.
             * This has the benign side effect that if value is
             * finalized it will see the location as the FOR_ITER's.
             */
            PyStackRef_CLOSE(value);
        }

        macro(POP_ITER) = POP_TOP;

        no_save_ip tier1 inst(INSTRUMENTED_END_FOR, (receiver, value -- receiver)) {
            /* Need to create a fake StopIteration error here,
             * to conform to PEP 380 */
            if (PyStackRef_GenCheck(receiver)) {
                int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value));
                if (err) {
                    ERROR_NO_POP();
                }
            }
            PyStackRef_CLOSE(value);
        }

        tier1 inst(INSTRUMENTED_POP_ITER, (iter -- )) {
            INSTRUMENTED_JUMP(prev_instr, this_instr+1, PY_MONITORING_EVENT_BRANCH_RIGHT);
            PyStackRef_CLOSE(iter);
        }

        pure inst(END_SEND, (receiver, value -- val)) {
            val = value;
            DEAD(value);
            PyStackRef_CLOSE(receiver);
        }

        tier1 inst(INSTRUMENTED_END_SEND, (receiver, value -- val)) {
            PyObject *receiver_o = PyStackRef_AsPyObjectBorrow(receiver);
            if (PyGen_Check(receiver_o) || PyCoro_CheckExact(receiver_o)) {
                int err = monitor_stop_iteration(tstate, frame, this_instr, PyStackRef_AsPyObjectBorrow(value));
                if (err) {
                    ERROR_NO_POP();
                }
            }
            val = value;
            DEAD(value);
            PyStackRef_CLOSE(receiver);
        }

        inst(UNARY_NEGATIVE, (value -- res)) {
            PyObject *res_o = PyNumber_Negative(PyStackRef_AsPyObjectBorrow(value));
            PyStackRef_CLOSE(value);
            ERROR_IF(res_o == NULL);
            res = PyStackRef_FromPyObjectSteal(res_o);
        }

        pure inst(UNARY_NOT, (value -- res)) {
            assert(PyStackRef_BoolCheck(value));
            res = PyStackRef_IsFalse(value)
                ? PyStackRef_True : PyStackRef_False;
            DEAD(value);
        }

        family(TO_BOOL, INLINE_CACHE_ENTRIES_TO_BOOL) = {
            TO_BOOL_ALWAYS_TRUE,
            TO_BOOL_BOOL,
            TO_BOOL_INT,
            TO_BOOL_LIST,
            TO_BOOL_NONE,
            TO_BOOL_STR,
        };

        specializing op(_SPECIALIZE_TO_BOOL, (counter/1, value -- value)) {
            #if ENABLE_SPECIALIZATION_FT
            if (ADAPTIVE_COUNTER_TRIGGERS(counter)) {
                next_instr = this_instr;
                _Py_Specialize_ToBool(value, next_instr);
                DISPATCH_SAME_OPARG();
            }
            OPCODE_DEFERRED_INC(TO_BOOL);
            ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
            #endif  /* ENABLE_SPECIALIZATION_FT */
        }

        op(_TO_BOOL, (value -- res)) {
            int err = PyObject_IsTrue(PyStackRef_AsPyObjectBorrow(value));
            PyStackRef_CLOSE(value);
            ERROR_IF(err < 0);
            res = err ? PyStackRef_True : PyStackRef_False;
        }

        macro(TO_BOOL) = _SPECIALIZE_TO_BOOL + unused/2 + _TO_BOOL;

        inst(TO_BOOL_BOOL, (unused/1, unused/2, value -- value)) {
            EXIT_IF(!PyStackRef_BoolCheck(value));
            STAT_INC(TO_BOOL, hit);
        }

        inst(TO_BOOL_INT, (unused/1, unused/2, value -- res)) {
            PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
            EXIT_IF(!PyLong_CheckExact(value_o));
            STAT_INC(TO_BOOL, hit);
            if (_PyLong_IsZero((PyLongObject *)value_o)) {
                assert(_Py_IsImmortal(value_o));
                DEAD(value);
                res = PyStackRef_False;
            }
            else {
                PyStackRef_CLOSE(value);
                res = PyStackRef_True;
            }
        }

        op(_GUARD_NOS_LIST, (nos, unused -- nos, unused)) {
            PyObject *o = PyStackRef_AsPyObjectBorrow(nos);
            EXIT_IF(!PyList_CheckExact(o));
        }

        op(_GUARD_TOS_LIST, (tos -- tos)) {
            PyObject *o = PyStackRef_AsPyObjectBorrow(tos);
            EXIT_IF(!PyList_CheckExact(o));
        }

        macro(TO_BOOL_LIST) = _GUARD_TOS_LIST + unused/1 + unused/2 + _TO_BOOL_LIST;

        op(_TO_BOOL_LIST, (value -- res)) {
            PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
            assert(PyList_CheckExact(value_o));
            STAT_INC(TO_BOOL, hit);
            res = PyList_GET_SIZE(value_o) ? PyStackRef_True : PyStackRef_False;
            DECREF_INPUTS();
        }

        inst(TO_BOOL_NONE, (unused/1, unused/2, value -- res)) {
            // This one is a bit weird, because we expect *some* failures:
            EXIT_IF(!PyStackRef_IsNone(value));
            DEAD(value);
            STAT_INC(TO_BOOL, hit);
            res = PyStackRef_False;
        }

        op(_GUARD_NOS_UNICODE, (nos, unused -- nos, unused)) {
            PyObject *o = PyStackRef_AsPyObjectBorrow(nos);
            EXIT_IF(!PyUnicode_CheckExact(o));
        }

        op(_GUARD_TOS_UNICODE, (value -- value)) {
            PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
            EXIT_IF(!PyUnicode_CheckExact(value_o));
        }

        op(_TO_BOOL_STR, (value -- res)) {
            STAT_INC(TO_BOOL, hit);
            PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
            if (value_o == &_Py_STR(empty)) {
                assert(_Py_IsImmortal(value_o));
                DEAD(value);
                res = PyStackRef_False;
            }
            else {
                assert(Py_SIZE(value_o));
                PyStackRef_CLOSE(value);
                res = PyStackRef_True;
            }
        }

        macro(TO_BOOL_STR) =
            _GUARD_TOS_UNICODE + unused/1 + unused/2 + _TO_BOOL_STR;

        op(_REPLACE_WITH_TRUE, (value -- res)) {
            PyStackRef_CLOSE(value);
            res = PyStackRef_True;
        }

        macro(TO_BOOL_ALWAYS_TRUE) =
            unused/1 +
            _GUARD_TYPE_VERSION +
            _REPLACE_WITH_TRUE;

        inst(UNARY_INVERT, (value -- res)) {
            PyObject *res_o = PyNumber_Invert(PyStackRef_AsPyObjectBorrow(value));
            PyStackRef_CLOSE(value);
            ERROR_IF(res_o == NULL);
            res = PyStackRef_FromPyObjectSteal(res_o);
        }

        family(BINARY_OP, INLINE_CACHE_ENTRIES_BINARY_OP) = {
            BINARY_OP_MULTIPLY_INT,
            BINARY_OP_ADD_INT,
            BINARY_OP_SUBTRACT_INT,
            BINARY_OP_MULTIPLY_FLOAT,
            BINARY_OP_ADD_FLOAT,
            BINARY_OP_SUBTRACT_FLOAT,
            BINARY_OP_ADD_UNICODE,
            BINARY_OP_SUBSCR_LIST_INT,
            BINARY_OP_SUBSCR_TUPLE_INT,
            BINARY_OP_SUBSCR_STR_INT,
            BINARY_OP_SUBSCR_DICT,
            BINARY_OP_SUBSCR_GETITEM,
            // BINARY_OP_INPLACE_ADD_UNICODE,  // See comments at that opcode.
            BINARY_OP_EXTEND,
        };

        op(_GUARD_NOS_INT, (left, unused -- left, unused)) {
            PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
            EXIT_IF(!PyLong_CheckExact(left_o));
        }

        op(_GUARD_TOS_INT, (value -- value)) {
            PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
            EXIT_IF(!PyLong_CheckExact(value_o));
        }

        pure op(_BINARY_OP_MULTIPLY_INT, (left, right -- res)) {
            PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
            PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
            assert(PyLong_CheckExact(left_o));
            assert(PyLong_CheckExact(right_o));

            STAT_INC(BINARY_OP, hit);
            PyObject *res_o = _PyLong_Multiply((PyLongObject *)left_o, (PyLongObject *)right_o);
            PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
            PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
            INPUTS_DEAD();
            ERROR_IF(res_o == NULL);
            res = PyStackRef_FromPyObjectSteal(res_o);
        }

        pure op(_BINARY_OP_ADD_INT, (left, right -- res)) {
            PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
            PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
            assert(PyLong_CheckExact(left_o));
            assert(PyLong_CheckExact(right_o));

            STAT_INC(BINARY_OP, hit);
            PyObject *res_o = _PyLong_Add((PyLongObject *)left_o, (PyLongObject *)right_o);
            PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
            PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
            INPUTS_DEAD();
            ERROR_IF(res_o == NULL);
            res = PyStackRef_FromPyObjectSteal(res_o);
        }

        pure op(_BINARY_OP_SUBTRACT_INT, (left, right -- res)) {
            PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
            PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
            assert(PyLong_CheckExact(left_o));
            assert(PyLong_CheckExact(right_o));

            STAT_INC(BINARY_OP, hit);
            PyObject *res_o = _PyLong_Subtract((PyLongObject *)left_o, (PyLongObject *)right_o);
            PyStackRef_CLOSE_SPECIALIZED(right, _PyLong_ExactDealloc);
            PyStackRef_CLOSE_SPECIALIZED(left, _PyLong_ExactDealloc);
            INPUTS_DEAD();
            ERROR_IF(res_o == NULL);
            res = PyStackRef_FromPyObjectSteal(res_o);
        }

        macro(BINARY_OP_MULTIPLY_INT) =
            _GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_MULTIPLY_INT;
        macro(BINARY_OP_ADD_INT) =
            _GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_ADD_INT;
        macro(BINARY_OP_SUBTRACT_INT) =
            _GUARD_TOS_INT + _GUARD_NOS_INT + unused/5 + _BINARY_OP_SUBTRACT_INT;

        op(_GUARD_NOS_FLOAT, (left, unused -- left, unused)) {
            PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
            EXIT_IF(!PyFloat_CheckExact(left_o));
        }

        op(_GUARD_TOS_FLOAT, (value -- value)) {
            PyObject *value_o = PyStackRef_AsPyObjectBorrow(value);
            EXIT_IF(!PyFloat_CheckExact(value_o));
        }

        pure op(_BINARY_OP_MULTIPLY_FLOAT, (left, right -- res)) {
            PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
            PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
            assert(PyFloat_CheckExact(left_o));
            assert(PyFloat_CheckExact(right_o));

            STAT_INC(BINARY_OP, hit);
            double dres =
                ((PyFloatObject *)left_o)->ob_fval *
                ((PyFloatObject *)right_o)->ob_fval;
            res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres);
            INPUTS_DEAD();
            ERROR_IF(PyStackRef_IsNull(res));
        }

        pure op(_BINARY_OP_ADD_FLOAT, (left, right -- res)) {
            PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
            PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
            assert(PyFloat_CheckExact(left_o));
            assert(PyFloat_CheckExact(right_o));

            STAT_INC(BINARY_OP, hit);
            double dres =
                ((PyFloatObject *)left_o)->ob_fval +
                ((PyFloatObject *)right_o)->ob_fval;
            res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres);
            INPUTS_DEAD();
            ERROR_IF(PyStackRef_IsNull(res));
        }

        pure op(_BINARY_OP_SUBTRACT_FLOAT, (left, right -- res)) {
            PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
            PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
            assert(PyFloat_CheckExact(left_o));
            assert(PyFloat_CheckExact(right_o));

            STAT_INC(BINARY_OP, hit);
            double dres =
                ((PyFloatObject *)left_o)->ob_fval -
                ((PyFloatObject *)right_o)->ob_fval;
            res = _PyFloat_FromDouble_ConsumeInputs(left, right, dres);
            INPUTS_DEAD();
            ERROR_IF(PyStackRef_IsNull(res));
        }

        macro(BINARY_OP_MULTIPLY_FLOAT) =
            _GUARD_TOS_FLOAT + _GUARD_NOS_FLOAT + unused/5 + _BINARY_OP_MULTIPLY_FLOAT;
        macro(BINARY_OP_ADD_FLOAT) =
            _GUARD_TOS_FLOAT + _GUARD_NOS_FLOAT + unused/5 + _BINARY_OP_ADD_FLOAT;
        macro(BINARY_OP_SUBTRACT_FLOAT) =
            _GUARD_TOS_FLOAT + _GUARD_NOS_FLOAT + unused/5 + _BINARY_OP_SUBTRACT_FLOAT;

        pure op(_BINARY_OP_ADD_UNICODE, (left, right -- res)) {
            PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
            PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
            assert(PyUnicode_CheckExact(left_o));
            assert(PyUnicode_CheckExact(right_o));

            STAT_INC(BINARY_OP, hit);
            PyObject *res_o = PyUnicode_Concat(left_o, right_o);
            PyStackRef_CLOSE_SPECIALIZED(right, _PyUnicode_ExactDealloc);
            PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc);
            INPUTS_DEAD();
            ERROR_IF(res_o == NULL);
            res = PyStackRef_FromPyObjectSteal(res_o);
        }

        macro(BINARY_OP_ADD_UNICODE) =
            _GUARD_TOS_UNICODE + _GUARD_NOS_UNICODE + unused/5 + _BINARY_OP_ADD_UNICODE;

        // This is a subtle one. It's a super-instruction for
        // BINARY_OP_ADD_UNICODE followed by STORE_FAST
        // where the store goes into the left argument.
        // So the inputs are the same as for all BINARY_OP
        // specializations, but there is no output.
        // At the end we just skip over the STORE_FAST.
        op(_BINARY_OP_INPLACE_ADD_UNICODE, (left, right --)) {
            PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
            assert(PyUnicode_CheckExact(left_o));
            assert(PyUnicode_CheckExact(PyStackRef_AsPyObjectBorrow(right)));

            int next_oparg;
        #if TIER_ONE
            assert(next_instr->op.code == STORE_FAST);
            next_oparg = next_instr->op.arg;
        #else
            next_oparg = CURRENT_OPERAND0();
        #endif
            _PyStackRef *target_local = &GETLOCAL(next_oparg);
            assert(PyUnicode_CheckExact(left_o));
            DEOPT_IF(PyStackRef_AsPyObjectBorrow(*target_local) != left_o);
            STAT_INC(BINARY_OP, hit);
            /* Handle `left = left + right` or `left += right` for str.
             *
             * When possible, extend `left` in place rather than
             * allocating a new PyUnicodeObject. This attempts to avoid
             * quadratic behavior when one neglects to use str.join().
             *
             * If `left` has only two references remaining (one from
             * the stack, one in the locals), DECREFing `left` leaves
             * only the locals reference, so PyUnicode_Append knows
             * that the string is safe to mutate.
             */
            assert(Py_REFCNT(left_o) >= 2 || !PyStackRef_IsHeapSafe(left));
            PyStackRef_CLOSE_SPECIALIZED(left, _PyUnicode_ExactDealloc);
            DEAD(left);
            PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local);
            PyObject *right_o = PyStackRef_AsPyObjectSteal(right);
            PyUnicode_Append(&temp, right_o);
            *target_local = PyStackRef_FromPyObjectSteal(temp);
            Py_DECREF(right_o);
            ERROR_IF(PyStackRef_IsNull(*target_local));
        #if TIER_ONE
            // The STORE_FAST is already done. This is done here in tier one,
            // and during trace projection in tier two:
            assert(next_instr->op.code == STORE_FAST);
            SKIP_OVER(1);
        #endif
        }

       op(_GUARD_BINARY_OP_EXTEND, (descr/4, left, right -- left, right)) {
            PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
            PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
            _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr;
            assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5);
            assert(d && d->guard);
            int res = d->guard(left_o, right_o);
            DEOPT_IF(!res);
        }

        pure op(_BINARY_OP_EXTEND, (descr/4, left, right -- res)) {
            PyObject *left_o = PyStackRef_AsPyObjectBorrow(left);
            PyObject *right_o = PyStackRef_AsPyObjectBorrow(right);
            assert(INLINE_CACHE_ENTRIES_BINARY_OP == 5);
            _PyBinaryOpSpecializationDescr *d = (_PyBinaryOpSpecializationDescr*)descr;

            STAT_INC(BINARY_OP, hit);

            PyObject *res_o = d->action(left_o, right_o);
            DECREF_INPUTS();
            res = PyStackRef_FromPyObjectSteal(res_o);
        }

        macro(BINARY_OP_EXTEND) =
            unused/1 + _GUARD_BINARY_OP_EXTEND + rewind/-4 + _BINARY_OP_EXTEND;

        macro(BINARY_OP_INPLACE_ADD_UNICODE) =
            _GUARD_TOS_UNICODE + _GUARD_NOS_UNICODE + unused/5 + _BINARY_OP_INPLACE_ADD_UNICODE;

        specializing op(_SPECIALIZE_BINARY_SLICE, (container, start, stop -- container, start, stop)) {
            // Placeholder until we implement BINARY_SLICE specialization
            #if ENABLE_SPECIALIZATION
            OPCODE_DEFERRED_INC(BINARY_SLICE);
            #endif  /* ENABLE_SPECIALIZATION */
        }

        op(_BINARY_SLICE, (container, start, stop -- res)) {
            PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start),
                                                        PyStackRef_AsPyObjectSteal(stop));
            PyObject *res_o;
            // Can't use ERROR_IF() here, because we haven't
            // DECREF'ed container yet, and we still own slice.
            if (slice == NULL) {
                res_o = NULL;
            }
            else {
                res_o = PyObject_GetItem(PyStackRef_AsPyObjectBorrow(container), slice);
                Py_DECREF(slice);
            }
            PyStackRef_CLOSE(container);
            ERROR_IF(res_o == NULL);
            res = PyStackRef_FromPyObjectSteal(res_o);
        }

        macro(BINARY_SLICE) = _SPECIALIZE_BINARY_SLICE + _BINARY_SLICE;

        specializing op(_SPECIALIZE_STORE_SLICE, (v, container, start, stop -- v, container, start, stop)) {
            // Placeholder until we implement STORE_SLICE specialization
            #if ENABLE_SPECIALIZATION
            OPCODE_DEFERRED_INC(STORE_SLICE);
            #endif  /* ENABLE_SPECIALIZATION */
        }

        op(_STORE_SLICE, (v, container, start, stop -- )) {
            PyObject *slice = _PyBuildSlice_ConsumeRefs(PyStackRef_AsPyObjectSteal(start),
                                                        PyStackRef_AsPyObjectSteal(stop));
            int err;
            if (slice == NULL) {
                err = 1;
            }
            else {
                err = PyObject_SetItem(PyStackRef_AsPyObjectBorrow(container), slice, PyStackRef_AsPyObjectBorrow(v));
                Py_DECREF(slice);
            }
            DECREF_INPUTS();
            ERROR_IF(err);
        }

        macro(STORE_SLICE) = _SPECIALIZE_STORE_SLICE + _STORE_SLICE;

        macro(BINARY_OP_SUBSCR_LIST_INT) =
            _GUARD_TOS_INT + _GUARD_NOS_LIST + unused/5 + _BINARY_OP_SUBSCR_LIST_INT;

        op(_BINARY_OP_SUBSCR_LIST_INT, (list_st, sub_st -- res)) {
            PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
            PyObject *list = PyStackRef_AsPyObjectBorrow(list_st);

            assert(PyLong_CheckExact(sub));
            assert(PyList_CheckExact(list));

            // Deopt unless 0 <= sub < PyList_Size(list)
            DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub));
            Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
#ifdef Py_GIL_DISABLED
            PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index);
            DEOPT_IF(res_o == NULL);
            STAT_INC(BINARY_OP, hit);
            res = PyStackRef_FromPyObjectSteal(res_o);
#else
            DEOPT_IF(index >= PyList_GET_SIZE(list));
            STAT_INC(BINARY_OP, hit);
            PyObject *res_o = PyList_GET_ITEM(list, index);
            assert(res_o != NULL);
            res = PyStackRef_FromPyObjectNew(res_o);
#endif
            STAT_INC(BINARY_OP, hit);
            DECREF_INPUTS();
        }

        macro(BINARY_OP_SUBSCR_STR_INT) =
            _GUARD_TOS_INT + _GUARD_NOS_UNICODE + unused/5 + _BINARY_OP_SUBSCR_STR_INT;

        op(_BINARY_OP_SUBSCR_STR_INT, (str_st, sub_st -- res)) {
            PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
            PyObject *str = PyStackRef_AsPyObjectBorrow(str_st);

            assert(PyLong_CheckExact(sub));
            assert(PyUnicode_CheckExact(str));
            DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub));
            Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
            DEOPT_IF(PyUnicode_GET_LENGTH(str) <= index);
            // Specialize for reading an ASCII character from any string:
            Py_UCS4 c = PyUnicode_READ_CHAR(str, index);
            DEOPT_IF(Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c);
            STAT_INC(BINARY_OP, hit);
            PyObject *res_o = (PyObject*)&_Py_SINGLETON(strings).ascii[c];
            PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc);
            DEAD(sub_st);
            PyStackRef_CLOSE(str_st);
            res = PyStackRef_FromPyObjectImmortal(res_o);
        }

        op(_GUARD_NOS_TUPLE, (nos, unused -- nos, unused)) {
            PyObject *o = PyStackRef_AsPyObjectBorrow(nos);
            EXIT_IF(!PyTuple_CheckExact(o));
        }

        op(_GUARD_TOS_TUPLE, (tos -- tos)) {
            PyObject *o = PyStackRef_AsPyObjectBorrow(tos);
            EXIT_IF(!PyTuple_CheckExact(o));
        }

        macro(BINARY_OP_SUBSCR_TUPLE_INT) =
            _GUARD_TOS_INT + _GUARD_NOS_TUPLE + unused/5 + _BINARY_OP_SUBSCR_TUPLE_INT;

        op(_BINARY_OP_SUBSCR_TUPLE_INT, (tuple_st, sub_st -- res)) {
            PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
            PyObject *tuple = PyStackRef_AsPyObjectBorrow(tuple_st);

            assert(PyLong_CheckExact(sub));
            assert(PyTuple_CheckExact(tuple));

            // Deopt unless 0 <= sub < PyTuple_Size(list)
            DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub));
            Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0];
            DEOPT_IF(index >= PyTuple_GET_SIZE(tuple));
            STAT_INC(BINARY_OP, hit);
            PyObject *res_o = PyTuple_GET_ITEM(tuple, index);
            assert(res_o != NULL);
            PyStackRef_CLOSE_SPECIALIZED(sub_st, _PyLong_ExactDealloc);
            res = PyStackRef_FromPyObjectNew(res_o);
            DECREF_INPUTS();
        }

        op(_GUARD_NOS_DICT, (nos, unused -- nos, unused)) {
            PyObject *o = PyStackRef_AsPyObjectBorrow(nos);
            EXIT_IF(!PyDict_CheckExact(o));
        }

        op(_GUARD_TOS_DICT, (tos -- tos)) {
            PyObject *o = PyStackRef_AsPyObjectBorrow(tos);
            EXIT_IF(!PyDict_CheckExact(o));
        }

        macro(BINARY_OP_SUBSCR_DICT) =
            _GUARD_NOS_DICT + unused/5 + _BINARY_OP_SUBSCR_DICT;

        op(_BINARY_OP_SUBSCR_DICT, (dict_st, sub_st -- res)) {
            PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st);
            PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st);

            assert(PyDict_CheckExact(dict));
            STAT_INC(BINARY_OP, hit);
            PyObject *res_o;
            int rc = PyDict_GetItemRef(dict, sub, &res_o);
            if (rc == 0) {
                _PyErr_SetKeyError(sub);
            }
            DECREF_INPUTS();
            ERROR_IF(rc <= 0); // not found or error
            res = PyStackRef_FromPyObjectSteal(res_o);
        }

        op(_BINARY_OP_SUBSCR_CHECK_FUNC, (container, unused -- container, unused, getitem)) {
            PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container));
            DEOPT_IF(!PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE));
            PyHeapTypeObject *ht = (PyHeapTypeObject *)tp;
            PyObject *getitem_o = FT_ATOMIC_LOAD_PTR_ACQUIRE(ht->_spec_cache.getitem);
            DEOPT_IF(getitem_o == NULL);
            assert(PyFunction_Check(getitem_o));
            uint32_t cached_version = FT_ATOMIC_LOAD_UINT32_RELAXED(ht->_spec_cache.getitem_version);
            DEOPT_IF(((PyFunctionObject *)getitem_o)->func_version != cached_version);
            PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(getitem_o);
            assert(code->co_argcount == 2);
            DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize));
            getitem = PyStackRef_FromPyObjectNew(getitem_o);
            STAT_INC(BINARY_OP, hit);