py: Add basic support for dynamically loadable native modules.

Using the import system you can now import native modules dynamically,
as you would a normal .py module.

Includes a simple example in extmod/modx.  To test:

>>> import modx
>>> modx.data
>>> modx.add1(1)

where modx.mpy (compiled from extmod/modx) is in your path.

Author: dpgeorge

extmod/modx/Makefile

@@ -0,0 +1,24 @@
+include ../../py/mkenv.mk
+
+CROSS = 0
+
+ifeq ($(CROSS), 1)
+CROSS_COMPILE = arm-none-eabi-
+endif
+
+INC =  -I. -I../..
+
+ifeq ($(CROSS), 1)
+CFLAGS_CORTEX_M4 = -mthumb -mtune=cortex-m4 -mabi=aapcs-linux -mcpu=cortex-m4 -mfpu=fpv4-sp-d16 -mfloat-abi=hard -fsingle-precision-constant -Wdouble-promotion
+CFLAGS = $(INC) -Wall -Werror -ansi -std=gnu99 -nostdlib -fPIC -Os $(CFLAGS_CORTEX_M4)
+else
+CFLAGS = $(INC) -Wall -Werror -ansi -std=gnu99 -fPIC -Os
+endif
+
+all: modx.mpy
+
+modx.mpy: modx.elf
+	$(OBJCOPY) -O binary -j .all $< $@
+
+modx.elf: modx.o
+	$(LD) -T mpextern.ld -o $@ $<

extmod/modx/genhdr/qstrdefs.generated.h

@@ -0,0 +1,15 @@
+#include "py/mpextern.h"
+
+STATIC mp_obj_t modx_add1(const mp_ext_table_t *et, mp_obj_t x) {
+    return et->mp_binary_op(MP_BINARY_OP_ADD, x, MP_OBJ_NEW_SMALL_INT(1));
+}
+
+MP_EXT_HEADER
+
+MP_EXT_INIT
+void init(const mp_ext_table_t *et) {
+    mp_obj_t f_add1 = et->mp_obj_new_fun_extern(false, 1, 1, modx_add1);
+    mp_obj_t list[6] = {MP_OBJ_NEW_SMALL_INT(1), MP_OBJ_NEW_SMALL_INT(2), MP_OBJ_NEW_SMALL_INT(3), et->mp_const_true_, MP_OBJ_NEW_QSTR(et->qstr_from_str("modx")), f_add1};
+    et->mp_store_global(et->qstr_from_str("data"), et->mp_obj_new_list(6, list));
+    et->mp_store_global(et->qstr_from_str("add1"), f_add1);
+}

extmod/modx/modx.c

@@ -0,0 +1,81 @@
+/*
+ * This file is part of the Micro Python project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2013, 2014 Damien P. George
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+// options to control how Micro Python is built
+
+#define MICROPY_ALLOC_PATH_MAX      (PATH_MAX)
+#define MICROPY_COMP_MODULE_CONST   (1)
+#define MICROPY_ENABLE_GC           (1)
+#define MICROPY_ENABLE_FINALISER    (1)
+#define MICROPY_STACK_CHECK         (1)
+#define MICROPY_MEM_STATS           (1)
+#define MICROPY_DEBUG_PRINTERS      (1)
+#define MICROPY_HELPER_REPL         (1)
+#define MICROPY_HELPER_LEXER_UNIX   (1)
+#define MICROPY_ENABLE_SOURCE_LINE  (1)
+#define MICROPY_FLOAT_IMPL          (MICROPY_FLOAT_IMPL_DOUBLE)
+#define MICROPY_LONGINT_IMPL        (MICROPY_LONGINT_IMPL_MPZ)
+#define MICROPY_STREAMS_NON_BLOCK   (1)
+#define MICROPY_OPT_COMPUTED_GOTO   (1)
+#define MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE (1)
+#define MICROPY_CAN_OVERRIDE_BUILTINS (1)
+#define MICROPY_MODULE_EXTERN       (1)
+#define MICROPY_PY_BUILTINS_STR_UNICODE (1)
+#define MICROPY_PY_BUILTINS_MEMORYVIEW (1)
+#define MICROPY_PY_BUILTINS_FROZENSET (1)
+#define MICROPY_PY_BUILTINS_COMPILE (1)
+#define MICROPY_PY_MICROPYTHON_MEM_INFO (1)
+#define MICROPY_PY_SYS_EXIT         (1)
+#define MICROPY_PY_SYS_PLATFORM     "linux"
+#define MICROPY_PY_SYS_MAXSIZE      (1)
+#define MICROPY_PY_SYS_STDFILES     (1)
+#define MICROPY_PY_CMATH            (1)
+#define MICROPY_PY_IO_FILEIO        (1)
+#define MICROPY_PY_GC_COLLECT_RETVAL (1)
+
+// type definitions for the specific machine
+
+#ifdef __LP64__
+typedef long mp_int_t; // must be pointer size
+typedef unsigned long mp_uint_t; // must be pointer size
+#else
+// These are definitions for machines where sizeof(int) == sizeof(void*),
+// regardless for actual size.
+typedef int mp_int_t; // must be pointer size
+typedef unsigned int mp_uint_t; // must be pointer size
+#endif
+
+#define BYTES_PER_WORD sizeof(mp_int_t)
+
+// Cannot include <sys/types.h>, as it may lead to symbol name clashes
+#if _FILE_OFFSET_BITS == 64 && !defined(__LP64__)
+typedef long long mp_off_t;
+#else
+typedef long mp_off_t;
+#endif
+
+typedef void *machine_ptr_t; // must be of pointer size
+typedef const void *machine_const_ptr_t; // must be of pointer size

extmod/modx/mpconfigport.h

@@ -0,0 +1,16 @@
+/*
+    GNU linker script for MPY external modules
+*/
+
+SECTIONS
+{
+    .all :
+    {
+        . = ALIGN(4);
+        KEEP(*(.mpyheader))
+        KEEP(*(.mpytext))
+        *(.text*)
+        *(.rodata*)
+        . = ALIGN(4);
+    }
+}

extmod/modx/mpextern.ld

@@ -35,6 +35,7 @@
 #include "py/runtime.h"
 #include "py/builtin.h"
 #include "py/frozenmod.h"
+#include "py/mpextern.h"
 
 #if 0 // print debugging info
 #define DEBUG_PRINT (1)
@@ -71,6 +72,14 @@ STATIC mp_import_stat_t stat_dir_or_file(vstr_t *path) {
     if (stat == MP_IMPORT_STAT_FILE) {
         return stat;
     }
+    #if MICROPY_MODULE_EXTERN
+    vstr_cut_tail_bytes(path, 2);
+    vstr_add_str(path, "mpy");
+    stat = mp_import_stat(vstr_null_terminated_str(path));
+    if (stat == MP_IMPORT_STAT_FILE) {
+        return stat;
+    }
+    #endif
     return MP_IMPORT_STAT_NO_EXIST;
 }
 
@@ -150,6 +159,18 @@ STATIC void chop_component(const char *start, const char **end) {
     *end = p;
 }
 
+#if MICROPY_MODULE_EXTERN
+STATIC void do_load_extern(mp_obj_t module_obj, vstr_t *file) {
+    #if MICROPY_PY___FILE__
+    mp_store_attr(module_obj, MP_QSTR___file__, MP_OBJ_NEW_QSTR(qstr_from_str(vstr_str(file))));
+    #endif
+
+    // load the extern module in its context
+    mp_obj_dict_t *mod_globals = mp_obj_module_get_globals(module_obj);
+    mp_extern_load(vstr_str(file), mod_globals);
+}
+#endif
+
 mp_obj_t mp_builtin___import__(mp_uint_t n_args, const mp_obj_t *args) {
 #if DEBUG_PRINT
     DEBUG_printf("__import__:\n");
@@ -377,7 +398,14 @@ mp_obj_t mp_builtin___import__(mp_uint_t n_args, const mp_obj_t *args) {
                         vstr_cut_tail_bytes(&path, sizeof("/__init__.py") - 1); // cut off /__init__.py
                     }
                 } else { // MP_IMPORT_STAT_FILE
-                    do_load(module_obj, &path);
+                    #if MICROPY_MODULE_EXTERN
+                    if (path.buf[path.len - 3] == 'm') {
+                        do_load_extern(module_obj, &path);
+                    } else
+                    #endif
+                    {
+                        do_load(module_obj, &path);
+                    }
                     // TODO: We cannot just break here, at the very least, we must execute
                     // trailer code below. But otherwise if there're remaining components,
                     // that would be (??) object path within module, not modules path within FS.

py/builtinimport.c

@@ -439,6 +439,11 @@ typedef double mp_float_t;
 #define MICROPY_MODULE_FROZEN (0)
 #endif
 
+// Whether to support loading external native modules
+#ifndef MICROPY_MODULE_EXTERN
+#define MICROPY_MODULE_EXTERN (0)
+#endif
+
 // Whether you can override builtins in the builtins module
 #ifndef MICROPY_CAN_OVERRIDE_BUILTINS
 #define MICROPY_CAN_OVERRIDE_BUILTINS (0)

py/mpconfig.h

@@ -0,0 +1,128 @@
+#include "mpextern.h"
+
+#if MICROPY_MODULE_EXTERN
+
+typedef mp_obj_t (*mp_fun_ext_0_t)(const mp_ext_table_t *et);
+typedef mp_obj_t (*mp_fun_ext_1_t)(const mp_ext_table_t *et, mp_obj_t);
+typedef mp_obj_t (*mp_fun_ext_2_t)(const mp_ext_table_t *et, mp_obj_t, mp_obj_t);
+typedef mp_obj_t (*mp_fun_ext_3_t)(const mp_ext_table_t *et, mp_obj_t, mp_obj_t, mp_obj_t);
+typedef mp_obj_t (*mp_fun_ext_var_t)(const mp_ext_table_t *et, mp_uint_t n, const mp_obj_t *);
+typedef mp_obj_t (*mp_fun_ext_kw_t)(const mp_ext_table_t *et, mp_uint_t n, const mp_obj_t *, mp_map_t *);
+
+STATIC const mp_obj_type_t mp_type_fun_extern;
+STATIC const mp_ext_table_t mp_ext_table;
+
+typedef struct _mp_obj_fun_extern_t { // use this to make const objects that go in ROM
+    mp_obj_base_t base;
+    bool is_kw : 1;
+    mp_uint_t n_args_min : 15; // inclusive
+    mp_uint_t n_args_max : 16; // inclusive
+    void *fun; // must be a pointer to a callable function in ROM
+    mp_obj_dict_t *dict_globals;
+} mp_obj_fun_extern_t;
+
+STATIC mp_obj_t fun_extern_call(mp_obj_t self_in, mp_uint_t n_args, mp_uint_t n_kw, const mp_obj_t *args) {
+    assert(MP_OBJ_IS_TYPE(self_in, &mp_type_fun_extern));
+    mp_obj_fun_extern_t *self = self_in;
+
+    // check number of arguments
+    mp_arg_check_num(n_args, n_kw, self->n_args_min, self->n_args_max, self->is_kw);
+
+    if (self->is_kw) {
+        // function allows keywords
+
+        // we create a map directly from the given args array
+        mp_map_t kw_args;
+        mp_map_init_fixed_table(&kw_args, n_kw, args + n_args);
+
+        return ((mp_fun_ext_kw_t)self->fun)(&mp_ext_table, n_args, args, &kw_args);
+
+    } else if (self->n_args_min <= 3 && self->n_args_min == self->n_args_max) {
+        // function requires a fixed number of arguments
+
+        // dispatch function call
+        switch (self->n_args_min) {
+            case 0:
+                return ((mp_fun_ext_0_t)self->fun)(&mp_ext_table);
+
+            case 1:
+                return ((mp_fun_ext_1_t)self->fun)(&mp_ext_table, args[0]);
+
+            case 2:
+                return ((mp_fun_ext_2_t)self->fun)(&mp_ext_table, args[0], args[1]);
+
+            case 3:
+            default:
+                return ((mp_fun_ext_3_t)self->fun)(&mp_ext_table, args[0], args[1], args[2]);
+        }
+
+    } else {
+        // function takes a variable number of arguments, but no keywords
+
+        return ((mp_fun_ext_var_t)self->fun)(&mp_ext_table, n_args, args);
+    }
+}
+
+STATIC const mp_obj_type_t mp_type_fun_extern = {
+    { &mp_type_type },
+    .name = MP_QSTR_function,
+    .call = fun_extern_call,
+};
+
+STATIC mp_obj_t mp_obj_new_fun_extern(bool is_kw, mp_uint_t n_args_min, mp_uint_t n_args_max, void *f) {
+    mp_obj_fun_extern_t *self = m_new_obj(mp_obj_fun_extern_t);
+    self->base.type = &mp_type_fun_extern;
+    self->is_kw = is_kw;
+    self->n_args_min = n_args_min;
+    self->n_args_max = n_args_max;
+    self->fun = f;
+    self->dict_globals = mp_globals_get();
+    return self;
+}
+
+STATIC const mp_ext_table_t mp_ext_table = {
+    .mp_const_none_ = mp_const_none,
+    .mp_const_false_ = mp_const_false,
+    .mp_const_true_ = mp_const_true,
+    .mp_obj_new_fun_extern = mp_obj_new_fun_extern,
+    .qstr_from_str = qstr_from_str,
+    .mp_store_global = mp_store_global,
+    .mp_obj_new_list = mp_obj_new_list,
+    .mp_binary_op = mp_binary_op,
+};
+
+void mp_extern_load(const char *ext_name, mp_obj_dict_t *globals) {
+    const byte *buf = mp_extern_load_binary(ext_name);
+
+    if (buf == NULL) {
+        nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_OSError, "could not load MPY binary"));
+    }
+
+    if (!(buf[0] == 'M' && buf[1] == 'P' && buf[2] == 'Y')) {
+        nlr_raise(mp_obj_new_exception_msg(&mp_type_OSError, "not a valid MPY binary"));
+    }
+
+    if (buf[3] != MP_EXT_VERSION_MAJOR || buf[4] > MP_EXT_VERSION_MINOR) {
+        nlr_raise(mp_obj_new_exception_msg(&mp_type_OSError, "MPY binary has wrong version"));
+    }
+
+    if (buf[6] != MP_EXT_ARCH_CURRENT) {
+        nlr_raise(mp_obj_new_exception_msg(&mp_type_OSError, "MPY binary has wrong arch"));
+    }
+
+    // push context
+    mp_obj_dict_t *old_locals = mp_locals_get();
+    mp_obj_dict_t *old_globals = mp_globals_get();
+    mp_locals_set(globals);
+    mp_globals_set(globals);
+
+    // call extern init
+    mp_obj_t (*f)(const mp_ext_table_t*) = (mp_obj_t(*)(const mp_ext_table_t*))MICROPY_MAKE_POINTER_CALLABLE(buf + 8);
+    f(&mp_ext_table);
+
+    // pop context
+    mp_globals_set(old_globals);
+    mp_locals_set(old_locals);
+}
+
+#endif // MICROPY_MODULE_EXTERN