Merge remote-tracking branch 'upstream/master'

py/builtin.c

@@ -16,30 +16,20 @@
 
 mp_obj_t mp_builtin___build_class__(mp_obj_t o_class_fun, mp_obj_t o_class_name) {
     // we differ from CPython: we set the new __locals__ object here
-    mp_map_t *old_locals = rt_get_map_locals();
+    mp_map_t *old_locals = rt_locals_get();
     mp_map_t *class_locals = mp_map_new(MP_MAP_QSTR, 0);
-    rt_set_map_locals(class_locals);
+    rt_locals_set(class_locals);
 
     // call the class code
     rt_call_function_1(o_class_fun, (mp_obj_t)0xdeadbeef);
 
     // restore old __locals__ object
-    rt_set_map_locals(old_locals);
+    rt_locals_set(old_locals);
 
     // create and return the new class
     return mp_obj_new_class(class_locals);
 }
 
-mp_obj_t mp_builtin___import__(int n, mp_obj_t *args) {
-    printf("import:\n");
-    for (int i = 0; i < n; i++) {
-    printf("  ");
-    mp_obj_print(args[i]);
-    printf("\n");
-    }
-    return mp_const_none;
-}
-
 mp_obj_t mp_builtin___repl_print__(mp_obj_t o) {
     if (o != mp_const_none) {
         mp_obj_print(o);

py/builtinimport.c

@@ -0,0 +1,84 @@
+#include <stdint.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <stdarg.h>
+#include <string.h>
+#include <assert.h>
+
+#include "nlr.h"
+#include "misc.h"
+#include "mpconfig.h"
+#include "lexer.h"
+#include "lexerunix.h"
+#include "parse.h"
+#include "obj.h"
+#include "compile.h"
+#include "runtime0.h"
+#include "runtime.h"
+#include "map.h"
+#include "builtin.h"
+
+mp_obj_t mp_builtin___import__(int n, mp_obj_t *args) {
+    /*
+    printf("import:\n");
+    for (int i = 0; i < n; i++) {
+        printf("  ");
+        mp_obj_print(args[i]);
+        printf("\n");
+    }
+    */
+
+    // find the file to import
+    qstr mod_name = mp_obj_get_qstr(args[0]);
+    mp_lexer_t *lex = mp_import_open_file(mod_name);
+    if (lex == NULL) {
+        // TODO handle lexer error correctly
+        return mp_const_none;
+    }
+
+    // create a new module object
+    mp_obj_t module_obj = mp_obj_new_module(mp_obj_get_qstr(args[0]));
+
+    // save the old context
+    mp_map_t *old_locals = rt_locals_get();
+    mp_map_t *old_globals = rt_globals_get();
+
+    // set the new context
+    rt_locals_set(mp_obj_module_get_globals(module_obj));
+    rt_globals_set(mp_obj_module_get_globals(module_obj));
+
+    // parse the imported script
+    mp_parse_node_t pn = mp_parse(lex, MP_PARSE_FILE_INPUT);
+    mp_lexer_free(lex);
+
+    if (pn == MP_PARSE_NODE_NULL) {
+        // TODO handle parse error correctly
+        rt_locals_set(old_locals);
+        rt_globals_set(old_globals);
+        return mp_const_none;
+    }
+
+    if (!mp_compile(pn, false)) {
+        // TODO handle compile error correctly
+        rt_locals_set(old_locals);
+        rt_globals_set(old_globals);
+        return mp_const_none;
+    }
+
+    // complied successfully, execute it
+    mp_obj_t module_fun = rt_make_function_from_id(1); // TODO we should return from mp_compile the unique_code_id for the module
+    nlr_buf_t nlr;
+    if (nlr_push(&nlr) == 0) {
+        rt_call_function_0(module_fun);
+        nlr_pop();
+    } else {
+        // exception; restore context and re-raise same exception
+        rt_locals_set(old_locals);
+        rt_globals_set(old_globals);
+        nlr_jump(nlr.ret_val);
+    }
+    rt_locals_set(old_locals);
+    rt_globals_set(old_globals);
+
+    return module_obj;
+}

py/compile.c

@@ -10,9 +10,11 @@
 #include "lexer.h"
 #include "parse.h"
 #include "scope.h"
-#include "compile.h"
 #include "runtime0.h"
 #include "emit.h"
+#include "obj.h"
+#include "compile.h"
+#include "runtime.h"
 
 // TODO need to mangle __attr names
 
@@ -3016,7 +3018,7 @@ void compile_scope_compute_things(compiler_t *comp, scope_t *scope) {
     }
 }
 
-bool mp_compile(mp_parse_node_t pn, bool is_repl) {
+mp_obj_t mp_compile(mp_parse_node_t pn, bool is_repl) {
     compiler_t *comp = m_new(compiler_t, 1);
 
     comp->qstr___class__ = qstr_from_str_static("__class__");
@@ -3146,7 +3148,19 @@ bool mp_compile(mp_parse_node_t pn, bool is_repl) {
         }
     }
 
+    bool had_error = comp->had_error;
     m_del_obj(compiler_t, comp);
 
-    return !comp->had_error;
+    if (had_error) {
+        // TODO return a proper error message
+        return mp_const_none;
+    } else {
+#if MICROPY_EMIT_CPYTHON
+        // can't create code, so just return true
+        return mp_const_true;
+#else
+        // return function that executes the outer module
+        return rt_make_function_from_id(1);
+#endif
+    }
 }

py/compile.h

@@ -1 +1 @@
-bool mp_compile(mp_parse_node_t pn, bool is_repl);
+mp_obj_t mp_compile(mp_parse_node_t pn, bool is_repl);

py/emitbc.c

@@ -9,7 +9,6 @@
 #include "mpconfig.h"
 #include "lexer.h"
 #include "parse.h"
-#include "compile.h"
 #include "scope.h"
 #include "runtime0.h"
 #include "emit.h"

py/emitcpy.c

@@ -9,7 +9,6 @@
 #include "mpconfig.h"
 #include "lexer.h"
 #include "parse.h"
-#include "compile.h"
 #include "scope.h"
 #include "runtime0.h"
 #include "emit.h"

py/emitpass1.c

@@ -9,7 +9,6 @@
 #include "mpconfig.h"
 #include "lexer.h"
 #include "parse.h"
-#include "compile.h"
 #include "scope.h"
 #include "runtime0.h"
 #include "emit.h"

py/lexer.h

@@ -138,3 +138,6 @@ bool mp_lexer_opt_str(mp_lexer_t *lex, const char *str);
 */
 bool mp_lexer_show_error(mp_lexer_t *lex, const char *msg);
 bool mp_lexer_show_error_pythonic(mp_lexer_t *lex, const char *msg);
+
+// used to import a module; must be implemented for a specific port
+mp_lexer_t *mp_import_open_file(qstr mod_name);

py/lexerunix.c

@@ -58,3 +58,23 @@ mp_lexer_t *mp_lexer_new_from_file(const char *filename) {
 
     return mp_lexer_new_from_str_len(filename, data, size, true);
 }
+
+/******************************************************************************/
+/* unix implementation of import                                              */
+
+// TODO properly!
+
+static const char *import_base_dir = NULL;
+
+void mp_import_set_directory(const char *dir) {
+    import_base_dir = dir;
+}
+
+mp_lexer_t *mp_import_open_file(qstr mod_name) {
+    vstr_t *vstr = vstr_new();
+    if (import_base_dir != NULL) {
+        vstr_printf(vstr, "%s/", import_base_dir);
+    }
+    vstr_printf(vstr, "%s.py", qstr_str(mod_name));
+    return mp_lexer_new_from_file(vstr_str(vstr)); // TODO does lexer need to copy the string? can we free it here?
+}

py/lexerunix.h

@@ -1,2 +1,4 @@
 mp_lexer_t *mp_lexer_new_from_str_len(const char *src_name, const char *str, uint len, bool free_str);
 mp_lexer_t *mp_lexer_new_from_file(const char *filename);
+
+void mp_import_set_directory(const char *dir);

py/malloc.c

@@ -2,8 +2,15 @@
 #include <stdlib.h>
 
 #include "misc.h"
+#include "mpconfig.h"
 
+#if MICROPY_MEM_STATS
 static int total_bytes_allocated = 0;
+static int current_bytes_allocated = 0;
+static int peak_bytes_allocated = 0;
+
+#define UPDATE_PEAK() { if (current_bytes_allocated > peak_bytes_allocated) peak_bytes_allocated = current_bytes_allocated; }
+#endif
 
 void *m_malloc(int num_bytes) {
     if (num_bytes == 0) {
@@ -14,7 +21,11 @@ void *m_malloc(int num_bytes) {
         printf("could not allocate memory, allocating %d bytes\n", num_bytes);
         return NULL;
     }
+#if MICROPY_MEM_STATS
     total_bytes_allocated += num_bytes;
+    current_bytes_allocated += num_bytes;
+    UPDATE_PEAK();
+#endif
     return ptr;
 }
 
@@ -27,7 +38,11 @@ void *m_malloc0(int num_bytes) {
         printf("could not allocate memory, allocating %d bytes\n", num_bytes);
         return NULL;
     }
+#if MICROPY_MEM_STATS
     total_bytes_allocated += num_bytes;
+    current_bytes_allocated += num_bytes;
+    UPDATE_PEAK();
+#endif
     return ptr;
 }
 
@@ -41,7 +56,17 @@ void *m_realloc(void *ptr, int old_num_bytes, int new_num_bytes) {
         printf("could not allocate memory, reallocating %d bytes\n", new_num_bytes);
         return NULL;
     }
-    total_bytes_allocated += new_num_bytes;
+#if MICROPY_MEM_STATS
+    // At first thought, "Total bytes allocated" should only grow,
+    // after all, it's *total*. But consider for example 2K block
+    // shrunk to 1K and then grown to 2K again. It's still 2K
+    // allocated total. If we process only positive increments,
+    // we'll count 3K.
+    int diff = new_num_bytes - old_num_bytes;
+    total_bytes_allocated += diff;
+    current_bytes_allocated += diff;
+    UPDATE_PEAK();
+#endif
     return ptr;
 }
 
@@ -49,8 +74,31 @@ void m_free(void *ptr, int num_bytes) {
     if (ptr != NULL) {
         free(ptr);
     }
+#if MICROPY_MEM_STATS
+    current_bytes_allocated -= num_bytes;
+#endif
 }
 
 int m_get_total_bytes_allocated(void) {
+#if MICROPY_MEM_STATS
     return total_bytes_allocated;
+#else
+    return -1;
+#endif
+}
+
+int m_get_current_bytes_allocated(void) {
+#if MICROPY_MEM_STATS
+    return current_bytes_allocated;
+#else
+    return -1;
+#endif
+}
+
+int m_get_peak_bytes_allocated(void) {
+#if MICROPY_MEM_STATS
+    return peak_bytes_allocated;
+#else
+    return -1;
+#endif
 }

py/map.h

@@ -23,10 +23,6 @@ typedef struct _mp_set_t {
     mp_obj_t *table;
 } mp_set_t;
 
-// these are defined in runtime.c
-mp_map_t *rt_get_map_locals(void);
-void rt_set_map_locals(mp_map_t *m);
-
 int get_doubling_prime_greater_or_equal_to(int x);
 void mp_map_init(mp_map_t *map, mp_map_kind_t kind, int n);
 mp_map_t *mp_map_new(mp_map_kind_t kind, int n);

py/misc.h

@@ -32,6 +32,8 @@ void *m_realloc(void *ptr, int old_num_bytes, int new_num_bytes);
 void m_free(void *ptr, int num_bytes);
 
 int m_get_total_bytes_allocated(void);
+int m_get_current_bytes_allocated(void);
+int m_get_peak_bytes_allocated(void);
 
 /** unichar / UTF-8 *********************************************/
 

py/mpconfig.h

@@ -0,0 +1,13 @@
+// This file contains default configuration settings for MicroPython.
+// You can override any of these options using mpconfigport.h file located
+// in a directory of your port.
+
+#include <mpconfigport.h>
+
+// Any options not explicitly set in mpconfigport.h will get default
+// values below.
+
+// Whether to collect memory allocation stats
+#ifndef MICROPY_MEM_STATS
+#define MICROPY_MEM_STATS (1)
+#endif

py/obj.h

@@ -215,11 +215,14 @@ mp_obj_t mp_obj_dict_store(mp_obj_t self_in, mp_obj_t key, mp_obj_t value);
 void mp_obj_set_store(mp_obj_t self_in, mp_obj_t item);
 
 // functions
-typedef struct _mp_obj_fun_native_t { // need this so we can define static objects
+typedef struct _mp_obj_fun_native_t { // need this so we can define const objects (to go in ROM)
     mp_obj_base_t base;
     machine_uint_t n_args_min; // inclusive
     machine_uint_t n_args_max; // inclusive
     void *fun;
+    // TODO add mp_map_t *globals
+    // for const function objects, make an empty, const map
+    // such functions won't be able to access the global scope, but that's probably okay
 } mp_obj_fun_native_t;
 extern const mp_obj_type_t fun_native_type;
 extern const mp_obj_type_t fun_bc_type;

py/objfun.c

@@ -7,6 +7,7 @@
 #include "misc.h"
 #include "mpconfig.h"
 #include "obj.h"
+#include "map.h"
 #include "runtime.h"
 #include "bc.h"
 
@@ -129,9 +130,10 @@ mp_obj_t rt_make_function_var_between(int n_args_min, int n_args_max, mp_fun_var
 
 typedef struct _mp_obj_fun_bc_t {
     mp_obj_base_t base;
-    int n_args;
-    uint n_state;
-    const byte *code;
+    mp_map_t *globals;      // the context within which this function was defined
+    int n_args;             // number of arguments this function takes
+    uint n_state;           // total state size for the executing function (incl args, locals, stack)
+    const byte *bytecode;   // bytecode for the function
 } mp_obj_fun_bc_t;
 
 // args are in reverse order in the array
@@ -142,15 +144,17 @@ mp_obj_t fun_bc_call_n(mp_obj_t self_in, int n_args, const mp_obj_t *args) {
         nlr_jump(mp_obj_new_exception_msg_2_args(rt_q_TypeError, "function takes %d positional arguments but %d were given", (const char*)(machine_int_t)self->n_args, (const char*)(machine_int_t)n_args));
     }
 
-    return mp_execute_byte_code(self->code, args, n_args, self->n_state);
-}
-
-void mp_obj_fun_bc_get(mp_obj_t self_in, int *n_args, uint *n_state, const byte **code) {
-    assert(MP_OBJ_IS_TYPE(self_in, &fun_bc_type));
-    mp_obj_fun_bc_t *self = self_in;
-    *n_args = self->n_args;
-    *n_state = self->n_state;
-    *code = self->code;
+    // optimisation: allow the compiler to optimise this tail call for
+    // the common case when the globals don't need to be changed
+    mp_map_t *old_globals = rt_globals_get();
+    if (self->globals == old_globals) {
+        return mp_execute_byte_code(self->bytecode, args, n_args, self->n_state);
+    } else {
+        rt_globals_set(self->globals);
+        mp_obj_t result = mp_execute_byte_code(self->bytecode, args, n_args, self->n_state);
+        rt_globals_set(old_globals);
+        return result;
+    }
 }
 
 const mp_obj_type_t fun_bc_type = {
@@ -170,12 +174,21 @@ const mp_obj_type_t fun_bc_type = {
 mp_obj_t mp_obj_new_fun_bc(int n_args, uint n_state, const byte *code) {
     mp_obj_fun_bc_t *o = m_new_obj(mp_obj_fun_bc_t);
     o->base.type = &fun_bc_type;
+    o->globals = rt_globals_get();
     o->n_args = n_args;
     o->n_state = n_state;
-    o->code = code;
+    o->bytecode = code;
     return o;
 }
 
+void mp_obj_fun_bc_get(mp_obj_t self_in, int *n_args, uint *n_state, const byte **code) {
+    assert(MP_OBJ_IS_TYPE(self_in, &fun_bc_type));
+    mp_obj_fun_bc_t *self = self_in;
+    *n_args = self->n_args;
+    *n_state = self->n_state;
+    *code = self->bytecode;
+}
+
 /******************************************************************************/
 /* inline assembler functions                                                 */
 

py/runtime.c

@@ -281,14 +281,6 @@ void rt_assign_inline_asm_code(int unique_code_id, void *fun, uint len, int n_ar
 #endif
 }
 
-mp_map_t *rt_get_map_locals(void) {
-    return map_locals;
-}
-
-void rt_set_map_locals(mp_map_t *m) {
-    map_locals = m;
-}
-
 static bool fit_small_int(mp_small_int_t o) {
     return true;
 }
@@ -479,6 +471,16 @@ mp_obj_t rt_unary_op(int op, mp_obj_t arg) {
 
 mp_obj_t rt_binary_op(int op, mp_obj_t lhs, mp_obj_t rhs) {
     DEBUG_OP_printf("binary %d %p %p\n", op, lhs, rhs);
+
+    // TODO correctly distinguish inplace operators for mutable objects
+    // lookup logic that CPython uses for +=:
+    //   check for implemented +=
+    //   then check for implemented +
+    //   then check for implemented seq.inplace_concat
+    //   then check for implemented seq.concat
+    //   then fail
+    // note that list does not implement + or +=, so that inplace_concat is reached first for +=
+
     if (MP_OBJ_IS_SMALL_INT(lhs) && MP_OBJ_IS_SMALL_INT(rhs)) {
         mp_small_int_t lhs_val = MP_OBJ_SMALL_INT_VALUE(lhs);
         mp_small_int_t rhs_val = MP_OBJ_SMALL_INT_VALUE(rhs);
@@ -786,6 +788,7 @@ mp_obj_t rt_load_attr(mp_obj_t base, qstr attr) {
     } else if (MP_OBJ_IS_TYPE(base, &instance_type)) {
         return mp_obj_instance_load_attr(base, attr);
     } else if (MP_OBJ_IS_TYPE(base, &module_type)) {
+        DEBUG_OP_printf("lookup module map %p\n", mp_obj_module_get_globals(base));
         mp_map_elem_t *elem = mp_qstr_map_lookup(mp_obj_module_get_globals(base), attr, false);
         if (elem == NULL) {
             // TODO what about generic method lookup?
@@ -913,6 +916,24 @@ mp_obj_t rt_import_from(mp_obj_t module, qstr name) {
     return x;
 }
 
+mp_map_t *rt_locals_get(void) {
+    return map_locals;
+}
+
+void rt_locals_set(mp_map_t *m) {
+    DEBUG_OP_printf("rt_locals_set(%p)\n", m);
+    map_locals = m;
+}
+
+mp_map_t *rt_globals_get(void) {
+    return map_globals;
+}
+
+void rt_globals_set(mp_map_t *m) {
+    DEBUG_OP_printf("rt_globals_set(%p)\n", m);
+    map_globals = m;
+}
+
 // these must correspond to the respective enum
 void *const rt_fun_table[RT_F_NUMBER_OF] = {
     rt_load_const_dec,

py/runtime.h

@@ -57,3 +57,9 @@ mp_obj_t rt_getiter(mp_obj_t o);
 mp_obj_t rt_iternext(mp_obj_t o);
 mp_obj_t rt_import_name(qstr name, mp_obj_t fromlist, mp_obj_t level);
 mp_obj_t rt_import_from(mp_obj_t module, qstr name);
+
+struct _mp_map_t;
+struct _mp_map_t *rt_locals_get(void);
+void rt_locals_set(struct _mp_map_t *m);
+struct _mp_map_t *rt_globals_get(void);
+void rt_globals_set(struct _mp_map_t *m);