py/emitnative: Implement yield and yield-from in native emitter.

This commit adds first class support for yield and yield-from in the native
emitter, including send and throw support, and yields enclosed in exception
handlers (which requires pulling down the NLR stack before yielding, then
rebuilding it when resuming).

This has been fully tested and is working on unix x86 and x86-64, and
stm32.  Also basic tests have been done with the esp8266 port.  Performance
of existing native code is unchanged.

py/compile.c

@@ -1703,6 +1703,7 @@ STATIC void compile_yield_from(compiler_t *comp) {
     EMIT_ARG(get_iter, false);
     EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE);
     EMIT_ARG(yield, MP_EMIT_YIELD_FROM);
+    reserve_labels_for_native(comp, 3);
 }
 
 #if MICROPY_PY_ASYNC_AWAIT
@@ -2634,6 +2635,7 @@ STATIC void compile_yield_expr(compiler_t *comp, mp_parse_node_struct_t *pns) {
     if (MP_PARSE_NODE_IS_NULL(pns->nodes[0])) {
         EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE);
         EMIT_ARG(yield, MP_EMIT_YIELD_VALUE);
+        reserve_labels_for_native(comp, 1);
     } else if (MP_PARSE_NODE_IS_STRUCT_KIND(pns->nodes[0], PN_yield_arg_from)) {
         pns = (mp_parse_node_struct_t*)pns->nodes[0];
         compile_node(comp, pns->nodes[0]);
@@ -2641,6 +2643,7 @@ STATIC void compile_yield_expr(compiler_t *comp, mp_parse_node_struct_t *pns) {
     } else {
         compile_node(comp, pns->nodes[0]);
         EMIT_ARG(yield, MP_EMIT_YIELD_VALUE);
+        reserve_labels_for_native(comp, 1);
     }
 }
 
@@ -2873,6 +2876,7 @@ STATIC void compile_scope_comp_iter(compiler_t *comp, mp_parse_node_struct_t *pn
         compile_node(comp, pn_inner_expr);
         if (comp->scope_cur->kind == SCOPE_GEN_EXPR) {
             EMIT_ARG(yield, MP_EMIT_YIELD_VALUE);
+            reserve_labels_for_native(comp, 1);
             EMIT(pop_top);
         } else {
             EMIT_ARG(store_comp, comp->scope_cur->kind, 4 * for_depth + 5);

py/emitglue.c

@@ -136,6 +136,10 @@ mp_obj_t mp_make_function_from_raw_code(const mp_raw_code_t *rc, mp_obj_t def_ar
         case MP_CODE_NATIVE_PY:
         case MP_CODE_NATIVE_VIPER:
             fun = mp_obj_new_fun_native(def_args, def_kw_args, rc->data.u_native.fun_data, rc->data.u_native.const_table);
+            // Check for a generator function, and if so change the type of the object
+            if ((rc->scope_flags & MP_SCOPE_FLAG_GENERATOR) != 0) {
+                ((mp_obj_base_t*)MP_OBJ_TO_PTR(fun))->type = &mp_type_native_gen_wrap;
+            }
             break;
         #endif
         #if MICROPY_EMIT_INLINE_ASM

py/emitnx86.c

@@ -66,6 +66,7 @@ STATIC byte mp_f_n_args[MP_F_NUMBER_OF] = {
     [MP_F_SETUP_CODE_STATE] = 4,
     [MP_F_SMALL_INT_FLOOR_DIVIDE] = 2,
     [MP_F_SMALL_INT_MODULO] = 2,
+    [MP_F_NATIVE_YIELD_FROM] = 3,
 };
 
 #define N_X86 (1)

py/nativeglue.c

@@ -106,7 +106,7 @@ mp_obj_t mp_native_call_function_n_kw(mp_obj_t fun_in, size_t n_args_kw, const m
 // wrapper that makes raise obj and raises it
 // END_FINALLY opcode requires that we don't raise if o==None
 void mp_native_raise(mp_obj_t o) {
-    if (o != mp_const_none) {
+    if (o != MP_OBJ_NULL && o != mp_const_none) {
         nlr_raise(mp_make_raise_obj(o));
     }
 }
@@ -137,6 +137,42 @@ STATIC mp_obj_t mp_native_iternext(mp_obj_iter_buf_t *iter) {
     return mp_iternext(obj);
 }
 
+STATIC bool mp_native_yield_from(mp_obj_t gen, mp_obj_t send_value, mp_obj_t *ret_value) {
+    mp_vm_return_kind_t ret_kind;
+    nlr_buf_t nlr_buf;
+    mp_obj_t throw_value = *ret_value;
+    if (nlr_push(&nlr_buf) == 0) {
+        if (throw_value != MP_OBJ_NULL) {
+            send_value = MP_OBJ_NULL;
+        }
+        ret_kind = mp_resume(gen, send_value, throw_value, ret_value);
+        nlr_pop();
+    } else {
+        ret_kind = MP_VM_RETURN_EXCEPTION;
+        *ret_value = nlr_buf.ret_val;
+    }
+
+    if (ret_kind == MP_VM_RETURN_YIELD) {
+        return true;
+    } else if (ret_kind == MP_VM_RETURN_NORMAL) {
+        if (*ret_value == MP_OBJ_STOP_ITERATION) {
+            *ret_value = mp_const_none;
+        }
+    } else {
+        assert(ret_kind == MP_VM_RETURN_EXCEPTION);
+        if (!mp_obj_exception_match(*ret_value, MP_OBJ_FROM_PTR(&mp_type_StopIteration))) {
+            nlr_raise(*ret_value);
+        }
+        *ret_value = mp_obj_exception_get_value(*ret_value);
+    }
+
+    if (throw_value != MP_OBJ_NULL && mp_obj_exception_match(throw_value, MP_OBJ_FROM_PTR(&mp_type_GeneratorExit))) {
+        nlr_raise(mp_make_raise_obj(throw_value));
+    }
+
+    return false;
+}
+
 // these must correspond to the respective enum in runtime0.h
 void *const mp_fun_table[MP_F_NUMBER_OF] = {
     mp_convert_obj_to_native,
@@ -189,6 +225,7 @@ void *const mp_fun_table[MP_F_NUMBER_OF] = {
     mp_setup_code_state,
     mp_small_int_floor_divide,
     mp_small_int_modulo,
+    mp_native_yield_from,
 };
 
 /*

py/obj.h

@@ -558,6 +558,7 @@ extern const mp_obj_type_t mp_type_zip;
 extern const mp_obj_type_t mp_type_array;
 extern const mp_obj_type_t mp_type_super;
 extern const mp_obj_type_t mp_type_gen_wrap;
+extern const mp_obj_type_t mp_type_native_gen_wrap;
 extern const mp_obj_type_t mp_type_gen_instance;
 extern const mp_obj_type_t mp_type_fun_builtin_0;
 extern const mp_obj_type_t mp_type_fun_builtin_1;

py/objfun.c

@@ -154,7 +154,7 @@ STATIC const mp_obj_type_t mp_type_fun_native;
 qstr mp_obj_fun_get_name(mp_const_obj_t fun_in) {
     const mp_obj_fun_bc_t *fun = MP_OBJ_TO_PTR(fun_in);
     #if MICROPY_EMIT_NATIVE
-    if (fun->base.type == &mp_type_fun_native) {
+    if (fun->base.type == &mp_type_fun_native || fun->base.type == &mp_type_native_gen_wrap) {
         // TODO native functions don't have name stored
         return MP_QSTR_;
     }

py/objgenerator.c

@@ -73,6 +73,53 @@ const mp_obj_type_t mp_type_gen_wrap = {
     #endif
 };
 
+/******************************************************************************/
+// native generator wrapper
+
+#if MICROPY_EMIT_NATIVE
+
+STATIC mp_obj_t native_gen_wrap_call(mp_obj_t self_in, size_t n_args, size_t n_kw, const mp_obj_t *args) {
+    // The state for a native generating function is held in the same struct as a bytecode function
+    mp_obj_fun_bc_t *self_fun = MP_OBJ_TO_PTR(self_in);
+
+    // Determine start of prelude, and extract n_state from it
+    uintptr_t prelude_offset = ((uintptr_t*)self_fun->bytecode)[0];
+    size_t n_state = mp_decode_uint_value(self_fun->bytecode + prelude_offset);
+    size_t n_exc_stack = 0;
+
+    // Allocate the generator object, with room for local stack and exception stack
+    mp_obj_gen_instance_t *o = m_new_obj_var(mp_obj_gen_instance_t, byte,
+        n_state * sizeof(mp_obj_t) + n_exc_stack * sizeof(mp_exc_stack_t));
+    o->base.type = &mp_type_gen_instance;
+
+    // Parse the input arguments and set up the code state
+    o->globals = self_fun->globals;
+    o->code_state.fun_bc = self_fun;
+    o->code_state.ip = (const byte*)prelude_offset;
+    mp_setup_code_state(&o->code_state, n_args, n_kw, args);
+
+    // Indicate we are a native function, which doesn't use this variable
+    o->code_state.exc_sp = NULL;
+
+    // Prepare the generator instance for execution
+    uintptr_t start_offset = ((uintptr_t*)self_fun->bytecode)[1];
+    o->code_state.ip = MICROPY_MAKE_POINTER_CALLABLE((void*)(self_fun->bytecode + start_offset));
+
+    return MP_OBJ_FROM_PTR(o);
+}
+
+const mp_obj_type_t mp_type_native_gen_wrap = {
+    { &mp_type_type },
+    .name = MP_QSTR_generator,
+    .call = native_gen_wrap_call,
+    .unary_op = mp_generic_unary_op,
+    #if MICROPY_PY_FUNCTION_ATTRS
+    .attr = mp_obj_fun_bc_attr,
+    #endif
+};
+
+#endif // MICROPY_EMIT_NATIVE
+
 /******************************************************************************/
 /* generator instance                                                         */
 
@@ -118,7 +165,22 @@ mp_vm_return_kind_t mp_obj_gen_resume(mp_obj_t self_in, mp_obj_t send_value, mp_
     self->code_state.old_globals = mp_globals_get();
     mp_globals_set(self->globals);
     self->globals = NULL;
-    mp_vm_return_kind_t ret_kind = mp_execute_bytecode(&self->code_state, throw_value);
+
+    mp_vm_return_kind_t ret_kind;
+
+    #if MICROPY_EMIT_NATIVE
+    if (self->code_state.exc_sp == NULL) {
+        // A native generator, with entry point 2 words into the "bytecode" pointer
+        typedef uintptr_t (*mp_fun_native_gen_t)(void*, mp_obj_t);
+        mp_fun_native_gen_t fun = MICROPY_MAKE_POINTER_CALLABLE((const void*)(self->code_state.fun_bc->bytecode + 2 * sizeof(uintptr_t)));
+        ret_kind = fun((void*)&self->code_state, throw_value);
+    } else
+    #endif
+    {
+        // A bytecode generator
+        ret_kind = mp_execute_bytecode(&self->code_state, throw_value);
+    }
+
     self->globals = mp_globals_get();
     mp_globals_set(self->code_state.old_globals);
 

py/runtime0.h

@@ -197,6 +197,7 @@ typedef enum {
     MP_F_SETUP_CODE_STATE,
     MP_F_SMALL_INT_FLOOR_DIVIDE,
     MP_F_SMALL_INT_MODULO,
+    MP_F_NATIVE_YIELD_FROM,
     MP_F_NUMBER_OF,
 } mp_fun_kind_t;