codegen.c

/*
 * This file implements the compiler's code generation stage, which
 * produces a sequence of pseudo-instructions from an AST.
 *
 * The primary entry point is _PyCodegen_Module() for modules, and
 * _PyCodegen_Expression() for expressions.
 *
 * CAUTION: The VISIT_* macros abort the current function when they
 * encounter a problem. So don't invoke them when there is memory
 * which needs to be released. Code blocks are OK, as the compiler
 * structure takes care of releasing those.  Use the arena to manage
 * objects.
 */

#include "Python.h"
#include "opcode.h"
#include "pycore_ast.h"           // _PyAST_GetDocString()
#define NEED_OPCODE_TABLES
#include "pycore_opcode_utils.h"
#undef NEED_OPCODE_TABLES
#include "pycore_c_array.h"       // _Py_c_array_t
#include "pycore_code.h"          // COMPARISON_LESS_THAN
#include "pycore_compile.h"
#include "pycore_instruction_sequence.h" // _PyInstructionSequence_NewLabel()
#include "pycore_intrinsics.h"
#include "pycore_long.h"          // _PyLong_GetZero()
#include "pycore_object.h"        // _Py_ANNOTATE_FORMAT_VALUE_WITH_FAKE_GLOBALS
#include "pycore_pystate.h"       // _Py_GetConfig()
#include "pycore_symtable.h"      // PySTEntryObject
#include "pycore_unicodeobject.h" // _PyUnicode_EqualToASCIIString

#define NEED_OPCODE_METADATA
#include "pycore_opcode_metadata.h" // _PyOpcode_opcode_metadata, _PyOpcode_num_popped/pushed
#undef NEED_OPCODE_METADATA

#include <stdbool.h>

#define COMP_GENEXP   0
#define COMP_LISTCOMP 1
#define COMP_SETCOMP  2
#define COMP_DICTCOMP 3

#undef SUCCESS
#undef ERROR
#define SUCCESS 0
#define ERROR -1

#define RETURN_IF_ERROR(X)  \
    do {                    \
        if ((X) == -1) {    \
            return ERROR;   \
        }                   \
    } while (0)

#define RETURN_IF_ERROR_IN_SCOPE(C, CALL)   \
    do {                                    \
        if ((CALL) < 0) {                   \
            _PyCompile_ExitScope((C));      \
            return ERROR;                   \
        }                                   \
    } while (0)

struct _PyCompiler;
typedef struct _PyCompiler compiler;

#define INSTR_SEQUENCE(C) _PyCompile_InstrSequence(C)
#define FUTURE_FEATURES(C) _PyCompile_FutureFeatures(C)
#define SYMTABLE(C) _PyCompile_Symtable(C)
#define SYMTABLE_ENTRY(C) _PyCompile_SymtableEntry(C)
#define OPTIMIZATION_LEVEL(C) _PyCompile_OptimizationLevel(C)
#define IS_INTERACTIVE_TOP_LEVEL(C) _PyCompile_IsInteractiveTopLevel(C)
#define SCOPE_TYPE(C) _PyCompile_ScopeType(C)
#define QUALNAME(C) _PyCompile_Qualname(C)
#define METADATA(C) _PyCompile_Metadata(C)

typedef _PyInstruction instruction;
typedef _PyInstructionSequence instr_sequence;
typedef _Py_SourceLocation location;
typedef _PyJumpTargetLabel jump_target_label;

typedef _PyCompile_FBlockInfo fblockinfo;

#define LOCATION(LNO, END_LNO, COL, END_COL) \
    ((const _Py_SourceLocation){(LNO), (END_LNO), (COL), (END_COL)})

#define LOC(x) SRC_LOCATION_FROM_AST(x)

#define NEW_JUMP_TARGET_LABEL(C, NAME) \
    jump_target_label NAME = _PyInstructionSequence_NewLabel(INSTR_SEQUENCE(C)); \
    if (!IS_JUMP_TARGET_LABEL(NAME)) { \
        return ERROR; \
    }

#define USE_LABEL(C, LBL) \
    RETURN_IF_ERROR(_PyInstructionSequence_UseLabel(INSTR_SEQUENCE(C), (LBL).id))

static const int compare_masks[] = {
    [Py_LT] = COMPARISON_LESS_THAN,
    [Py_LE] = COMPARISON_LESS_THAN | COMPARISON_EQUALS,
    [Py_EQ] = COMPARISON_EQUALS,
    [Py_NE] = COMPARISON_NOT_EQUALS,
    [Py_GT] = COMPARISON_GREATER_THAN,
    [Py_GE] = COMPARISON_GREATER_THAN | COMPARISON_EQUALS,
};


int
_Py_CArray_Init(_Py_c_array_t* array, int item_size, int initial_num_entries) {
    memset(array, 0, sizeof(_Py_c_array_t));
    array->item_size = item_size;
    array->initial_num_entries = initial_num_entries;
    return 0;
}

void
_Py_CArray_Fini(_Py_c_array_t* array)
{
    if (array->array) {
        PyMem_Free(array->array);
        array->allocated_entries = 0;
    }
}

int
_Py_CArray_EnsureCapacity(_Py_c_array_t *c_array, int idx)
{
    void *arr = c_array->array;
    int alloc = c_array->allocated_entries;
    if (arr == NULL) {
        int new_alloc = c_array->initial_num_entries;
        if (idx >= new_alloc) {
            new_alloc = idx + c_array->initial_num_entries;
        }
        arr = PyMem_Calloc(new_alloc, c_array->item_size);
        if (arr == NULL) {
            PyErr_NoMemory();
            return ERROR;
        }
        alloc = new_alloc;
    }
    else if (idx >= alloc) {
        size_t oldsize = alloc * c_array->item_size;
        int new_alloc = alloc << 1;
        if (idx >= new_alloc) {
            new_alloc = idx + c_array->initial_num_entries;
        }
        size_t newsize = new_alloc * c_array->item_size;

        if (oldsize > (SIZE_MAX >> 1)) {
            PyErr_NoMemory();
            return ERROR;
        }

        assert(newsize > 0);
        void *tmp = PyMem_Realloc(arr, newsize);
        if (tmp == NULL) {
            PyErr_NoMemory();
            return ERROR;
        }
        alloc = new_alloc;
        arr = tmp;
        memset((char *)arr + oldsize, 0, newsize - oldsize);
    }

    c_array->array = arr;
    c_array->allocated_entries = alloc;
    return SUCCESS;
}


typedef struct {
    // A list of strings corresponding to name captures. It is used to track:
    // - Repeated name assignments in the same pattern.
    // - Different name assignments in alternatives.
    // - The order of name assignments in alternatives.
    PyObject *stores;
    // If 0, any name captures against our subject will raise.
    int allow_irrefutable;
    // An array of blocks to jump to on failure. Jumping to fail_pop[i] will pop
    // i items off of the stack. The end result looks like this (with each block
    // falling through to the next):
    // fail_pop[4]: POP_TOP
    // fail_pop[3]: POP_TOP
    // fail_pop[2]: POP_TOP
    // fail_pop[1]: POP_TOP
    // fail_pop[0]: NOP
    jump_target_label *fail_pop;
    // The current length of fail_pop.
    Py_ssize_t fail_pop_size;
    // The number of items on top of the stack that need to *stay* on top of the
    // stack. Variable captures go beneath these. All of them will be popped on
    // failure.
    Py_ssize_t on_top;
} pattern_context;

static int codegen_nameop(compiler *, location, identifier, expr_context_ty);

static int codegen_visit_stmt(compiler *, stmt_ty);
static int codegen_visit_keyword(compiler *, keyword_ty);
static int codegen_visit_expr(compiler *, expr_ty);
static int codegen_augassign(compiler *, stmt_ty);
static int codegen_annassign(compiler *, stmt_ty);
static int codegen_subscript(compiler *, expr_ty);
static int codegen_slice_two_parts(compiler *, expr_ty);
static int codegen_slice(compiler *, expr_ty);

static int codegen_body(compiler *, location, asdl_stmt_seq *, bool);
static int codegen_with(compiler *, stmt_ty);
static int codegen_async_with(compiler *, stmt_ty);
static int codegen_with_inner(compiler *, stmt_ty, int);
static int codegen_async_with_inner(compiler *, stmt_ty, int);
static int codegen_async_for(compiler *, stmt_ty);
static int codegen_call_simple_kw_helper(compiler *c,
                                         location loc,
                                         asdl_keyword_seq *keywords,
                                         Py_ssize_t nkwelts);
static int codegen_call_helper_impl(compiler *c, location loc,
                                    int n, /* Args already pushed */
                                    asdl_expr_seq *args,
                                    PyObject *injected_arg,
                                    asdl_keyword_seq *keywords);
static int codegen_call_helper(compiler *c, location loc,
                               int n, asdl_expr_seq *args,
                               asdl_keyword_seq *keywords);
static int codegen_try_except(compiler *, stmt_ty);
static int codegen_try_star_except(compiler *, stmt_ty);

static int codegen_sync_comprehension_generator(
                                      compiler *c, location loc,
                                      asdl_comprehension_seq *generators, int gen_index,
                                      int depth,
                                      expr_ty elt, expr_ty val, int type,
                                      int iter_on_stack);

static int codegen_async_comprehension_generator(
                                      compiler *c, location loc,
                                      asdl_comprehension_seq *generators, int gen_index,
                                      int depth,
                                      expr_ty elt, expr_ty val, int type,
                                      int iter_on_stack);

static int codegen_pattern(compiler *, pattern_ty, pattern_context *);
static int codegen_match(compiler *, stmt_ty);
static int codegen_pattern_subpattern(compiler *,
                                      pattern_ty, pattern_context *);
static int codegen_make_closure(compiler *c, location loc,
                                PyCodeObject *co, Py_ssize_t flags);


/* Add an opcode with an integer argument */
static int
codegen_addop_i(instr_sequence *seq, int opcode, Py_ssize_t oparg, location loc)
{
    /* oparg value is unsigned, but a signed C int is usually used to store
       it in the C code (like Python/ceval.c).

       Limit to 32-bit signed C int (rather than INT_MAX) for portability.

       The argument of a concrete bytecode instruction is limited to 8-bit.
       EXTENDED_ARG is used for 16, 24, and 32-bit arguments. */

    int oparg_ = Py_SAFE_DOWNCAST(oparg, Py_ssize_t, int);
    assert(!IS_ASSEMBLER_OPCODE(opcode));
    return _PyInstructionSequence_Addop(seq, opcode, oparg_, loc);
}

#define ADDOP_I(C, LOC, OP, O) \
    RETURN_IF_ERROR(codegen_addop_i(INSTR_SEQUENCE(C), (OP), (O), (LOC)))

#define ADDOP_I_IN_SCOPE(C, LOC, OP, O) \
    RETURN_IF_ERROR_IN_SCOPE(C, codegen_addop_i(INSTR_SEQUENCE(C), (OP), (O), (LOC)))

static int
codegen_addop_noarg(instr_sequence *seq, int opcode, location loc)
{
    assert(!OPCODE_HAS_ARG(opcode));
    assert(!IS_ASSEMBLER_OPCODE(opcode));
    return _PyInstructionSequence_Addop(seq, opcode, 0, loc);
}

#define ADDOP(C, LOC, OP) \
    RETURN_IF_ERROR(codegen_addop_noarg(INSTR_SEQUENCE(C), (OP), (LOC)))

#define ADDOP_IN_SCOPE(C, LOC, OP) \
    RETURN_IF_ERROR_IN_SCOPE((C), codegen_addop_noarg(INSTR_SEQUENCE(C), (OP), (LOC)))

static int
codegen_addop_load_const(compiler *c, location loc, PyObject *o)
{
    Py_ssize_t arg = _PyCompile_AddConst(c, o);
    if (arg < 0) {
        return ERROR;
    }
    ADDOP_I(c, loc, LOAD_CONST, arg);
    return SUCCESS;
}

#define ADDOP_LOAD_CONST(C, LOC, O) \
    RETURN_IF_ERROR(codegen_addop_load_const((C), (LOC), (O)))

#define ADDOP_LOAD_CONST_IN_SCOPE(C, LOC, O) \
    RETURN_IF_ERROR_IN_SCOPE((C), codegen_addop_load_const((C), (LOC), (O)))

/* Same as ADDOP_LOAD_CONST, but steals a reference. */
#define ADDOP_LOAD_CONST_NEW(C, LOC, O)                                 \
    do {                                                                \
        PyObject *__new_const = (O);                                    \
        if (__new_const == NULL) {                                      \
            return ERROR;                                               \
        }                                                               \
        if (codegen_addop_load_const((C), (LOC), __new_const) < 0) {    \
            Py_DECREF(__new_const);                                     \
            return ERROR;                                               \
        }                                                               \
        Py_DECREF(__new_const);                                         \
    } while (0)

static int
codegen_addop_o(compiler *c, location loc,
                int opcode, PyObject *dict, PyObject *o)
{
    Py_ssize_t arg = _PyCompile_DictAddObj(dict, o);
    RETURN_IF_ERROR(arg);
    ADDOP_I(c, loc, opcode, arg);
    return SUCCESS;
}

#define ADDOP_N(C, LOC, OP, O, TYPE)                                    \
    do {                                                                \
        assert(!OPCODE_HAS_CONST(OP)); /* use ADDOP_LOAD_CONST_NEW */   \
        int ret = codegen_addop_o((C), (LOC), (OP),                     \
                                  METADATA(C)->u_ ## TYPE, (O));        \
        Py_DECREF((O));                                                 \
        RETURN_IF_ERROR(ret);                                           \
    } while (0)

#define ADDOP_N_IN_SCOPE(C, LOC, OP, O, TYPE)                           \
    do {                                                                \
        assert(!OPCODE_HAS_CONST(OP)); /* use ADDOP_LOAD_CONST_NEW */   \
        int ret = codegen_addop_o((C), (LOC), (OP),                     \
                                  METADATA(C)->u_ ## TYPE, (O));        \
        Py_DECREF((O));                                                 \
        RETURN_IF_ERROR_IN_SCOPE((C), ret);                             \
    } while (0)

#define LOAD_METHOD -1
#define LOAD_SUPER_METHOD -2
#define LOAD_ZERO_SUPER_ATTR -3
#define LOAD_ZERO_SUPER_METHOD -4

static int
codegen_addop_name(compiler *c, location loc,
                   int opcode, PyObject *dict, PyObject *o)
{
    PyObject *mangled = _PyCompile_MaybeMangle(c, o);
    if (!mangled) {
        return ERROR;
    }
    Py_ssize_t arg = _PyCompile_DictAddObj(dict, mangled);
    Py_DECREF(mangled);
    if (arg < 0) {
        return ERROR;
    }
    if (opcode == LOAD_ATTR) {
        arg <<= 1;
    }
    if (opcode == LOAD_METHOD) {
        opcode = LOAD_ATTR;
        arg <<= 1;
        arg |= 1;
    }
    if (opcode == LOAD_SUPER_ATTR) {
        arg <<= 2;
        arg |= 2;
    }
    if (opcode == LOAD_SUPER_METHOD) {
        opcode = LOAD_SUPER_ATTR;
        arg <<= 2;
        arg |= 3;
    }
    if (opcode == LOAD_ZERO_SUPER_ATTR) {
        opcode = LOAD_SUPER_ATTR;
        arg <<= 2;
    }
    if (opcode == LOAD_ZERO_SUPER_METHOD) {
        opcode = LOAD_SUPER_ATTR;
        arg <<= 2;
        arg |= 1;
    }
    ADDOP_I(c, loc, opcode, arg);
    return SUCCESS;
}

#define ADDOP_NAME(C, LOC, OP, O, TYPE) \
    RETURN_IF_ERROR(codegen_addop_name((C), (LOC), (OP), METADATA(C)->u_ ## TYPE, (O)))

static int
codegen_addop_j(instr_sequence *seq, location loc,
                int opcode, jump_target_label target)
{
    assert(IS_JUMP_TARGET_LABEL(target));
    assert(OPCODE_HAS_JUMP(opcode) || IS_BLOCK_PUSH_OPCODE(opcode));
    assert(!IS_ASSEMBLER_OPCODE(opcode));
    return _PyInstructionSequence_Addop(seq, opcode, target.id, loc);
}

#define ADDOP_JUMP(C, LOC, OP, O) \
    RETURN_IF_ERROR(codegen_addop_j(INSTR_SEQUENCE(C), (LOC), (OP), (O)))

#define ADDOP_COMPARE(C, LOC, CMP) \
    RETURN_IF_ERROR(codegen_addcompare((C), (LOC), (cmpop_ty)(CMP)))

#define ADDOP_BINARY(C, LOC, BINOP) \
    RETURN_IF_ERROR(addop_binary((C), (LOC), (BINOP), false))

#define ADDOP_INPLACE(C, LOC, BINOP) \
    RETURN_IF_ERROR(addop_binary((C), (LOC), (BINOP), true))

#define ADD_YIELD_FROM(C, LOC, await) \
    RETURN_IF_ERROR(codegen_add_yield_from((C), (LOC), (await)))

#define POP_EXCEPT_AND_RERAISE(C, LOC) \
    RETURN_IF_ERROR(codegen_pop_except_and_reraise((C), (LOC)))

#define ADDOP_YIELD(C, LOC) \
    RETURN_IF_ERROR(codegen_addop_yield((C), (LOC)))

/* VISIT and VISIT_SEQ takes an ASDL type as their second argument.  They use
   the ASDL name to synthesize the name of the C type and the visit function.
*/

#define VISIT(C, TYPE, V) \
    RETURN_IF_ERROR(codegen_visit_ ## TYPE((C), (V)))

#define VISIT_IN_SCOPE(C, TYPE, V) \
    RETURN_IF_ERROR_IN_SCOPE((C), codegen_visit_ ## TYPE((C), (V)))

#define VISIT_SEQ(C, TYPE, SEQ)                                             \
    do {                                                                    \
        asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */    \
        for (int _i = 0; _i < asdl_seq_LEN(seq); _i++) {                    \
            TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, _i);           \
            RETURN_IF_ERROR(codegen_visit_ ## TYPE((C), elt));              \
        }                                                                   \
    } while (0)

#define VISIT_SEQ_IN_SCOPE(C, TYPE, SEQ)                                    \
    do {                                                                    \
        asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */    \
        for (int _i = 0; _i < asdl_seq_LEN(seq); _i++) {                    \
            TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, _i);           \
            if (codegen_visit_ ## TYPE((C), elt) < 0) {                     \
                _PyCompile_ExitScope(C);                                    \
                return ERROR;                                               \
            }                                                               \
        }                                                                   \
    } while (0)

static int
codegen_call_exit_with_nones(compiler *c, location loc)
{
    ADDOP_LOAD_CONST(c, loc, Py_None);
    ADDOP_LOAD_CONST(c, loc, Py_None);
    ADDOP_LOAD_CONST(c, loc, Py_None);
    ADDOP_I(c, loc, CALL, 3);
    return SUCCESS;
}

static int
codegen_add_yield_from(compiler *c, location loc, int await)
{
    NEW_JUMP_TARGET_LABEL(c, send);
    NEW_JUMP_TARGET_LABEL(c, fail);
    NEW_JUMP_TARGET_LABEL(c, exit);

    USE_LABEL(c, send);
    ADDOP_JUMP(c, loc, SEND, exit);
    // Set up a virtual try/except to handle when StopIteration is raised during
    // a close or throw call. The only way YIELD_VALUE raises if they do!
    ADDOP_JUMP(c, loc, SETUP_FINALLY, fail);
    ADDOP_I(c, loc, YIELD_VALUE, 1);
    ADDOP(c, NO_LOCATION, POP_BLOCK);
    ADDOP_I(c, loc, RESUME, await ? RESUME_AFTER_AWAIT : RESUME_AFTER_YIELD_FROM);
    ADDOP_JUMP(c, loc, JUMP_NO_INTERRUPT, send);

    USE_LABEL(c, fail);
    ADDOP(c, loc, CLEANUP_THROW);

    USE_LABEL(c, exit);
    ADDOP(c, loc, END_SEND);
    return SUCCESS;
}

static int
codegen_pop_except_and_reraise(compiler *c, location loc)
{
    /* Stack contents
     * [exc_info, lasti, exc]            COPY        3
     * [exc_info, lasti, exc, exc_info]  POP_EXCEPT
     * [exc_info, lasti, exc]            RERAISE      1
     * (exception_unwind clears the stack)
     */

    ADDOP_I(c, loc, COPY, 3);
    ADDOP(c, loc, POP_EXCEPT);
    ADDOP_I(c, loc, RERAISE, 1);
    return SUCCESS;
}

/* Unwind a frame block.  If preserve_tos is true, the TOS before
 * popping the blocks will be restored afterwards, unless another
 * return, break or continue is found. In which case, the TOS will
 * be popped.
 */
static int
codegen_unwind_fblock(compiler *c, location *ploc,
                      fblockinfo *info, int preserve_tos)
{
    switch (info->fb_type) {
        case COMPILE_FBLOCK_WHILE_LOOP:
        case COMPILE_FBLOCK_EXCEPTION_HANDLER:
        case COMPILE_FBLOCK_EXCEPTION_GROUP_HANDLER:
        case COMPILE_FBLOCK_ASYNC_COMPREHENSION_GENERATOR:
        case COMPILE_FBLOCK_STOP_ITERATION:
            return SUCCESS;

        case COMPILE_FBLOCK_FOR_LOOP:
            /* Pop the iterator */
            if (preserve_tos) {
                ADDOP_I(c, *ploc, SWAP, 2);
            }
            ADDOP(c, *ploc, POP_TOP);
            return SUCCESS;

        case COMPILE_FBLOCK_TRY_EXCEPT:
            ADDOP(c, *ploc, POP_BLOCK);
            return SUCCESS;

        case COMPILE_FBLOCK_FINALLY_TRY:
            /* This POP_BLOCK gets the line number of the unwinding statement */
            ADDOP(c, *ploc, POP_BLOCK);
            if (preserve_tos) {
                RETURN_IF_ERROR(
                    _PyCompile_PushFBlock(c, *ploc, COMPILE_FBLOCK_POP_VALUE,
                                          NO_LABEL, NO_LABEL, NULL));
            }
            /* Emit the finally block */
            VISIT_SEQ(c, stmt, info->fb_datum);
            if (preserve_tos) {
                _PyCompile_PopFBlock(c, COMPILE_FBLOCK_POP_VALUE, NO_LABEL);
            }
            /* The finally block should appear to execute after the
             * statement causing the unwinding, so make the unwinding
             * instruction artificial */
            *ploc = NO_LOCATION;
            return SUCCESS;

        case COMPILE_FBLOCK_FINALLY_END:
            if (preserve_tos) {
                ADDOP_I(c, *ploc, SWAP, 2);
            }
            ADDOP(c, *ploc, POP_TOP); /* exc_value */
            if (preserve_tos) {
                ADDOP_I(c, *ploc, SWAP, 2);
            }
            ADDOP(c, *ploc, POP_BLOCK);
            ADDOP(c, *ploc, POP_EXCEPT);
            return SUCCESS;

        case COMPILE_FBLOCK_WITH:
        case COMPILE_FBLOCK_ASYNC_WITH:
            *ploc = info->fb_loc;
            ADDOP(c, *ploc, POP_BLOCK);
            if (preserve_tos) {
                ADDOP_I(c, *ploc, SWAP, 3);
                ADDOP_I(c, *ploc, SWAP, 2);
            }
            RETURN_IF_ERROR(codegen_call_exit_with_nones(c, *ploc));
            if (info->fb_type == COMPILE_FBLOCK_ASYNC_WITH) {
                ADDOP_I(c, *ploc, GET_AWAITABLE, 2);
                ADDOP_LOAD_CONST(c, *ploc, Py_None);
                ADD_YIELD_FROM(c, *ploc, 1);
            }
            ADDOP(c, *ploc, POP_TOP);
            /* The exit block should appear to execute after the
             * statement causing the unwinding, so make the unwinding
             * instruction artificial */
            *ploc = NO_LOCATION;
            return SUCCESS;

        case COMPILE_FBLOCK_HANDLER_CLEANUP: {
            if (info->fb_datum) {
                ADDOP(c, *ploc, POP_BLOCK);
            }
            if (preserve_tos) {
                ADDOP_I(c, *ploc, SWAP, 2);
            }
            ADDOP(c, *ploc, POP_BLOCK);
            ADDOP(c, *ploc, POP_EXCEPT);
            if (info->fb_datum) {
                ADDOP_LOAD_CONST(c, *ploc, Py_None);
                RETURN_IF_ERROR(codegen_nameop(c, *ploc, info->fb_datum, Store));
                RETURN_IF_ERROR(codegen_nameop(c, *ploc, info->fb_datum, Del));
            }
            return SUCCESS;
        }
        case COMPILE_FBLOCK_POP_VALUE: {
            if (preserve_tos) {
                ADDOP_I(c, *ploc, SWAP, 2);
            }
            ADDOP(c, *ploc, POP_TOP);
            return SUCCESS;
        }
    }
    Py_UNREACHABLE();
}

/** Unwind block stack. If loop is not NULL, then stop when the first loop is encountered. */
static int
codegen_unwind_fblock_stack(compiler *c, location *ploc,
                            int preserve_tos, fblockinfo **loop)
{
    fblockinfo *top = _PyCompile_TopFBlock(c);
    if (top == NULL) {
        return SUCCESS;
    }
    if (top->fb_type == COMPILE_FBLOCK_EXCEPTION_GROUP_HANDLER) {
        return _PyCompile_Error(
            c, *ploc, "'break', 'continue' and 'return' cannot appear in an except* block");
    }
    if (loop != NULL && (top->fb_type == COMPILE_FBLOCK_WHILE_LOOP ||
                         top->fb_type == COMPILE_FBLOCK_FOR_LOOP)) {
        *loop = top;
        return SUCCESS;
    }
    fblockinfo copy = *top;
    _PyCompile_PopFBlock(c, top->fb_type, top->fb_block);
    RETURN_IF_ERROR(codegen_unwind_fblock(c, ploc, &copy, preserve_tos));
    RETURN_IF_ERROR(codegen_unwind_fblock_stack(c, ploc, preserve_tos, loop));
    _PyCompile_PushFBlock(c, copy.fb_loc, copy.fb_type, copy.fb_block,
                          copy.fb_exit, copy.fb_datum);
    return SUCCESS;
}

static int
codegen_enter_scope(compiler *c, identifier name, int scope_type,
                    void *key, int lineno, PyObject *private,
                    _PyCompile_CodeUnitMetadata *umd)
{
    RETURN_IF_ERROR(
        _PyCompile_EnterScope(c, name, scope_type, key, lineno, private, umd));
    location loc = LOCATION(lineno, lineno, 0, 0);
    if (scope_type == COMPILE_SCOPE_MODULE) {
        loc.lineno = 0;
    }
    ADDOP_I(c, loc, RESUME, RESUME_AT_FUNC_START);
    if (scope_type == COMPILE_SCOPE_MODULE) {
        ADDOP(c, loc, ANNOTATIONS_PLACEHOLDER);
    }
    return SUCCESS;
}

static int
codegen_setup_annotations_scope(compiler *c, location loc,
                                void *key, PyObject *name)
{
    _PyCompile_CodeUnitMetadata umd = {
        .u_posonlyargcount = 1,
    };
    RETURN_IF_ERROR(
        codegen_enter_scope(c, name, COMPILE_SCOPE_ANNOTATIONS,
                            key, loc.lineno, NULL, &umd));

    // if .format > VALUE_WITH_FAKE_GLOBALS: raise NotImplementedError
    PyObject *value_with_fake_globals = PyLong_FromLong(_Py_ANNOTATE_FORMAT_VALUE_WITH_FAKE_GLOBALS);
    assert(!SYMTABLE_ENTRY(c)->ste_has_docstring);
    _Py_DECLARE_STR(format, ".format");
    ADDOP_I(c, loc, LOAD_FAST, 0);
    ADDOP_LOAD_CONST(c, loc, value_with_fake_globals);
    ADDOP_I(c, loc, COMPARE_OP, (Py_GT << 5) | compare_masks[Py_GT]);
    NEW_JUMP_TARGET_LABEL(c, body);
    ADDOP_JUMP(c, loc, POP_JUMP_IF_FALSE, body);
    ADDOP_I(c, loc, LOAD_COMMON_CONSTANT, CONSTANT_NOTIMPLEMENTEDERROR);
    ADDOP_I(c, loc, RAISE_VARARGS, 1);
    USE_LABEL(c, body);
    return SUCCESS;
}

static int
codegen_leave_annotations_scope(compiler *c, location loc)
{
    ADDOP_IN_SCOPE(c, loc, RETURN_VALUE);
    PyCodeObject *co = _PyCompile_OptimizeAndAssemble(c, 1);
    if (co == NULL) {
        return ERROR;
    }

    // We want the parameter to __annotate__ to be named "format" in the
    // signature  shown by inspect.signature(), but we need to use a
    // different name (.format) in the symtable; if the name
    // "format" appears in the annotations, it doesn't get clobbered
    // by this name.  This code is essentially:
    // co->co_localsplusnames = ("format", *co->co_localsplusnames[1:])
    const Py_ssize_t size = PyObject_Size(co->co_localsplusnames);
    if (size == -1) {
        Py_DECREF(co);
        return ERROR;
    }
    PyObject *new_names = PyTuple_New(size);
    if (new_names == NULL) {
        Py_DECREF(co);
        return ERROR;
    }
    PyTuple_SET_ITEM(new_names, 0, Py_NewRef(&_Py_ID(format)));
    for (int i = 1; i < size; i++) {
        PyObject *item = PyTuple_GetItem(co->co_localsplusnames, i);
        if (item == NULL) {
            Py_DECREF(co);
            Py_DECREF(new_names);
            return ERROR;
        }
        Py_INCREF(item);
        PyTuple_SET_ITEM(new_names, i, item);
    }
    Py_SETREF(co->co_localsplusnames, new_names);

    _PyCompile_ExitScope(c);
    int ret = codegen_make_closure(c, loc, co, 0);
    Py_DECREF(co);
    RETURN_IF_ERROR(ret);
    return SUCCESS;
}

static int
codegen_deferred_annotations_body(compiler *c, location loc,
    PyObject *deferred_anno, PyObject *conditional_annotation_indices, int scope_type)
{
    Py_ssize_t annotations_len = PyList_GET_SIZE(deferred_anno);

    assert(PyList_CheckExact(conditional_annotation_indices));
    assert(annotations_len == PyList_Size(conditional_annotation_indices));

    ADDOP_I(c, loc, BUILD_MAP, 0); // stack now contains <annos>

    for (Py_ssize_t i = 0; i < annotations_len; i++) {
        PyObject *ptr = PyList_GET_ITEM(deferred_anno, i);
        stmt_ty st = (stmt_ty)PyLong_AsVoidPtr(ptr);
        if (st == NULL) {
            return ERROR;
        }
        PyObject *mangled = _PyCompile_Mangle(c, st->v.AnnAssign.target->v.Name.id);
        if (!mangled) {
            return ERROR;
        }
        PyObject *cond_index = PyList_GET_ITEM(conditional_annotation_indices, i);
        assert(PyLong_CheckExact(cond_index));
        long idx = PyLong_AS_LONG(cond_index);
        NEW_JUMP_TARGET_LABEL(c, not_set);

        if (idx != -1) {
            ADDOP_LOAD_CONST(c, LOC(st), cond_index);
            if (scope_type == COMPILE_SCOPE_CLASS) {
                ADDOP_NAME(
                    c, LOC(st), LOAD_DEREF, &_Py_ID(__conditional_annotations__), freevars);
            }
            else {
                ADDOP_NAME(
                    c, LOC(st), LOAD_GLOBAL, &_Py_ID(__conditional_annotations__), names);
            }

            ADDOP_I(c, LOC(st), CONTAINS_OP, 0);
            ADDOP_JUMP(c, LOC(st), POP_JUMP_IF_FALSE, not_set);
        }

        VISIT(c, expr, st->v.AnnAssign.annotation);
        ADDOP_I(c, LOC(st), COPY, 2);
        ADDOP_LOAD_CONST_NEW(c, LOC(st), mangled);
        // stack now contains <annos> <name> <annos> <value>
        ADDOP(c, loc, STORE_SUBSCR);
        // stack now contains <annos>

        USE_LABEL(c, not_set);
    }
    return SUCCESS;
}

static int
codegen_process_deferred_annotations(compiler *c, location loc)
{
    PyObject *deferred_anno = NULL;
    PyObject *conditional_annotation_indices = NULL;
    _PyCompile_DeferredAnnotations(c, &deferred_anno, &conditional_annotation_indices);
    if (deferred_anno == NULL) {
        assert(conditional_annotation_indices == NULL);
        return SUCCESS;
    }

    int scope_type = SCOPE_TYPE(c);
    bool need_separate_block = scope_type == COMPILE_SCOPE_MODULE;
    if (need_separate_block) {
        if (_PyCompile_StartAnnotationSetup(c) == ERROR) {
            goto error;
        }
    }

    // It's possible that ste_annotations_block is set but
    // u_deferred_annotations is not, because the former is still
    // set if there are only non-simple annotations (i.e., annotations
    // for attributes, subscripts, or parenthesized names). However, the
    // reverse should not be possible.
    PySTEntryObject *ste = SYMTABLE_ENTRY(c);
    assert(ste->ste_annotation_block != NULL);
    void *key = (void *)((uintptr_t)ste->ste_id + 1);
    if (codegen_setup_annotations_scope(c, loc, key,
                                        ste->ste_annotation_block->ste_name) < 0) {
        goto error;
    }
    if (codegen_deferred_annotations_body(c, loc, deferred_anno,
                                          conditional_annotation_indices, scope_type) < 0) {
        _PyCompile_ExitScope(c);
        goto error;
    }

    Py_DECREF(deferred_anno);
    Py_DECREF(conditional_annotation_indices);

    RETURN_IF_ERROR(codegen_leave_annotations_scope(c, loc));
    RETURN_IF_ERROR(codegen_nameop(
        c, loc,
        ste->ste_type == ClassBlock ? &_Py_ID(__annotate_func__) : &_Py_ID(__annotate__),
        Store));

    if (need_separate_block) {
        RETURN_IF_ERROR(_PyCompile_EndAnnotationSetup(c));
    }

    return SUCCESS;
error:
    Py_XDECREF(deferred_anno);
    Py_XDECREF(conditional_annotation_indices);
    return ERROR;
}

/* Compile an expression */
int
_PyCodegen_Expression(compiler *c, expr_ty e)
{
    VISIT(c, expr, e);
    return SUCCESS;
}

/* Compile a sequence of statements, checking for a docstring
   and for annotations. */

int
_PyCodegen_Module(compiler *c, location loc, asdl_stmt_seq *stmts, bool is_interactive)
{
    if (SYMTABLE_ENTRY(c)->ste_has_conditional_annotations) {
        ADDOP_I(c, loc, BUILD_SET, 0);
        ADDOP_N(c, loc, STORE_NAME, &_Py_ID(__conditional_annotations__), names);
    }
    return codegen_body(c, loc, stmts, is_interactive);
}

int
codegen_body(compiler *c, location loc, asdl_stmt_seq *stmts, bool is_interactive)
{
    /* If from __future__ import annotations is active,
     * every annotated class and module should have __annotations__.
     * Else __annotate__ is created when necessary. */
    PySTEntryObject *ste = SYMTABLE_ENTRY(c);
    if ((FUTURE_FEATURES(c) & CO_FUTURE_ANNOTATIONS) && ste->ste_annotations_used) {
        ADDOP(c, loc, SETUP_ANNOTATIONS);
    }
    if (!asdl_seq_LEN(stmts)) {
        return SUCCESS;
    }
    Py_ssize_t first_instr = 0;
    if (!is_interactive) { /* A string literal on REPL prompt is not a docstring */
        if (ste->ste_has_docstring) {
            PyObject *docstring = _PyAST_GetDocString(stmts);
            assert(docstring);
            first_instr = 1;
            /* set docstring */
            assert(OPTIMIZATION_LEVEL(c) < 2);
            PyObject *cleandoc = _PyCompile_CleanDoc(docstring);
            if (cleandoc == NULL) {
                return ERROR;
            }
            stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0);
            assert(st->kind == Expr_kind);
            location loc = LOC(st->v.Expr.value);
            ADDOP_LOAD_CONST(c, loc, cleandoc);
            Py_DECREF(cleandoc);
            RETURN_IF_ERROR(codegen_nameop(c, NO_LOCATION, &_Py_ID(__doc__), Store));
        }
    }
    for (Py_ssize_t i = first_instr; i < asdl_seq_LEN(stmts); i++) {
        VISIT(c, stmt, (stmt_ty)asdl_seq_GET(stmts, i));
    }
    // If there are annotations and the future import is not on, we
    // collect the annotations in a separate pass and generate an
    // __annotate__ function. See PEP 649.
    if (!(FUTURE_FEATURES(c) & CO_FUTURE_ANNOTATIONS)) {
        RETURN_IF_ERROR(codegen_process_deferred_annotations(c, loc));
    }
    return SUCCESS;
}

int
_PyCodegen_EnterAnonymousScope(compiler* c, mod_ty mod)
{
    _Py_DECLARE_STR(anon_module, "<module>");
    RETURN_IF_ERROR(
        codegen_enter_scope(c, &_Py_STR(anon_module), COMPILE_SCOPE_MODULE,
                            mod, 1, NULL, NULL));
    return SUCCESS;
}

static int
codegen_make_closure(compiler *c, location loc,
                     PyCodeObject *co, Py_ssize_t flags)
{
    if (co->co_nfreevars) {
        int i = PyUnstable_Code_GetFirstFree(co);
        for (; i < co->co_nlocalsplus; ++i) {
            /* Bypass com_addop_varname because it will generate
               LOAD_DEREF but LOAD_CLOSURE is needed.
            */
            PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i);
            int arg = _PyCompile_LookupArg(c, co, name);
            RETURN_IF_ERROR(arg);
            ADDOP_I(c, loc, LOAD_CLOSURE, arg);
        }
        flags |= MAKE_FUNCTION_CLOSURE;
        ADDOP_I(c, loc, BUILD_TUPLE, co->co_nfreevars);
    }
    ADDOP_LOAD_CONST(c, loc, (PyObject*)co);

    ADDOP(c, loc, MAKE_FUNCTION);

    if (flags & MAKE_FUNCTION_CLOSURE) {
        ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_CLOSURE);
    }
    if (flags & MAKE_FUNCTION_ANNOTATIONS) {
        ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_ANNOTATIONS);
    }
    if (flags & MAKE_FUNCTION_ANNOTATE) {
        ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_ANNOTATE);
    }
    if (flags & MAKE_FUNCTION_KWDEFAULTS) {
        ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_KWDEFAULTS);
    }
    if (flags & MAKE_FUNCTION_DEFAULTS) {
        ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_DEFAULTS);
    }
    return SUCCESS;
}

static int
codegen_decorators(compiler *c, asdl_expr_seq* decos)
{
    if (!decos) {
        return SUCCESS;
    }

    for (Py_ssize_t i = 0; i < asdl_seq_LEN(decos); i++) {
        VISIT(c, expr, (expr_ty)asdl_seq_GET(decos, i));
    }
    return SUCCESS;
}

static int
codegen_apply_decorators(compiler *c, asdl_expr_seq* decos)
{
    if (!decos) {
        return SUCCESS;
    }

    for (Py_ssize_t i = asdl_seq_LEN(decos) - 1; i > -1; i--) {
        location loc = LOC((expr_ty)asdl_seq_GET(decos, i));
        ADDOP_I(c, loc, CALL, 0);
    }
    return SUCCESS;
}

static int
codegen_kwonlydefaults(compiler *c, location loc,
                       asdl_arg_seq *kwonlyargs, asdl_expr_seq *kw_defaults)
{
    /* Push a dict of keyword-only default values.

       Return -1 on error, 0 if no dict pushed, 1 if a dict is pushed.
       */
    int default_count = 0;
    for (int i = 0; i < asdl_seq_LEN(kwonlyargs); i++) {
        arg_ty arg = asdl_seq_GET(kwonlyargs, i);
        expr_ty default_ = asdl_seq_GET(kw_defaults, i);
        if (default_) {
            default_count++;