moductypes: Foreign data interface module, roughly based on ctype ideas.

But much smaller and memory-efficient. Uses Python builtin data structures
(dict, tuple, int) to describe structure layout.

py/builtin.h

@@ -80,3 +80,6 @@ extern const mp_obj_module_t mp_module_micropython;
 extern const mp_obj_module_t mp_module_struct;
 extern const mp_obj_module_t mp_module_sys;
 extern const mp_obj_module_t mp_module_gc;
+
+// extmod modules
+extern const mp_obj_module_t mp_module_uctypes;

py/builtintables.c

@@ -190,6 +190,12 @@ STATIC const mp_map_elem_t mp_builtin_module_table[] = {
     { MP_OBJ_NEW_QSTR(MP_QSTR_gc), (mp_obj_t)&mp_module_gc },
 #endif
 
+    // extmod modules
+
+#if MICROPY_PY_UCTYPES
+    { MP_OBJ_NEW_QSTR(MP_QSTR_uctypes), (mp_obj_t)&mp_module_uctypes },
+#endif
+
     // extra builtin modules as defined by a port
     MICROPY_PORT_BUILTIN_MODULES
 };

py/compile.c

@@ -104,6 +104,9 @@ STATIC void compile_syntax_error(compiler_t *comp, mp_parse_node_t pn, const cha
 }
 
 STATIC const mp_map_elem_t mp_constants_table[] = {
+    #if MICROPY_PY_UCTYPES
+    { MP_OBJ_NEW_QSTR(MP_QSTR_uctypes), (mp_obj_t)&mp_module_uctypes },
+    #endif
     // Extra constants as defined by a port
     MICROPY_PORT_CONSTANTS
 };

py/mpconfig.h

@@ -351,6 +351,12 @@ typedef double mp_float_t;
 #define MICROPY_PY_SYS_STDFILES (0)
 #endif
 
+
+// Extended modules
+#ifndef MICROPY_PY_UCTYPES
+#define MICROPY_PY_UCTYPES (0)
+#endif
+
 /*****************************************************************************/
 /* Hooks for a port to add builtins                                          */
 

py/py.mk

@@ -102,6 +102,7 @@ PY_O_BASENAME = \
 	repl.o \
 	smallint.o \
 	pfenv.o \
+	../extmod/moductypes.o
 
 # prepend the build destination prefix to the py object files
 PY_O = $(addprefix $(PY_BUILD)/, $(PY_O_BASENAME))

py/qstrdefs.h

@@ -363,6 +363,43 @@ Q(pack)
 Q(unpack)
 #endif
 
+#if MICROPY_PY_UCTYPES
+Q(uctypes)
+Q(sizeof)
+Q(addressof)
+Q(bytes_at)
+Q(bytearray_at)
+
+Q(NATIVE)
+Q(LITTLE_ENDIAN)
+Q(BIG_ENDIAN)
+
+Q(VOID)
+
+Q(UINT8)
+Q(INT8)
+Q(UINT16)
+Q(INT16)
+Q(UINT32)
+Q(INT32)
+Q(UINT64)
+Q(INT64)
+
+Q(BFUINT8)
+Q(BFINT8)
+Q(BFUINT16)
+Q(BFINT16)
+Q(BFUINT32)
+Q(BFINT32)
+
+Q(FLOAT32)
+Q(FLOAT64)
+
+Q(ARRAY)
+Q(PTR)
+//Q(BITFIELD)
+#endif
+
 #if MICROPY_PY_IO
 Q(_io)
 Q(readall)

tests/extmod/uctypes_le.py

@@ -0,0 +1,68 @@
+import uctypes
+
+desc = {
+    "s0": uctypes.UINT16 | 0,
+    "sub": (0, {
+        "b0": uctypes.UINT8 | 0,
+        "b1": uctypes.UINT8 | 1,
+    }),
+    "arr": (uctypes.ARRAY | 0, uctypes.UINT8 | 2),
+    "arr2": (uctypes.ARRAY | 0, 2, {"b": uctypes.UINT8 | 0}),
+    "bitf0": uctypes.BFUINT16 | 0 | 0 << 17 | 8 << 22,
+    "bitf1": uctypes.BFUINT16 | 0 | 8 << 17 | 8 << 22,
+
+    "bf0": uctypes.BFUINT16 | 0 | 0 << 17 | 4 << 22,
+    "bf1": uctypes.BFUINT16 | 0 | 4 << 17 | 4 << 22,
+    "bf2": uctypes.BFUINT16 | 0 | 8 << 17 | 4 << 22,
+    "bf3": uctypes.BFUINT16 | 0 | 12 << 17 | 4 << 22,
+
+    "ptr": (uctypes.PTR | 0, uctypes.UINT8),
+    "ptr2": (uctypes.PTR | 0, {"b": uctypes.UINT8 | 0}),
+}
+
+data = bytearray(b"01")
+
+S = uctypes.struct(desc, uctypes.addressof(data), uctypes.LITTLE_ENDIAN)
+
+#print(S)
+print(hex(S.s0))
+assert hex(S.s0) == "0x3130"
+#print(S.sub.b0)
+print(S.sub.b0, S.sub.b1)
+assert S.sub.b0, S.sub.b1 == (0x30, 0x31)
+
+try:
+    S[0]
+    assert False, "Can't index struct"
+except TypeError:
+    print("TypeError")
+
+print("arr:", S.arr[0], S.arr[1])
+assert (S.arr[0], S.arr[1]) == (0x30, 0x31)
+
+print("arr of struct:", S.arr2[0].b, S.arr2[1].b)
+assert (S.arr2[0].b, S.arr2[1].b) == (0x30, 0x31)
+
+
+try:
+    S.arr[2]
+    assert False, "Out of bounds index"
+except IndexError:
+    print("IndexError")
+
+print("bf:", S.bitf0, S.bitf1)
+assert (S.bitf0, S.bitf1) == (0x30, 0x31)
+
+print("bf 4bit:", S.bf3, S.bf2, S.bf1, S.bf0)
+assert (S.bf3, S.bf2, S.bf1, S.bf0) == (3, 1, 3, 0)
+
+
+# Write access
+
+S.sub.b0 = ord("2")
+print(data)
+assert bytes(data) == b"21"
+
+S.bf3 = 5
+print(data)
+assert bytes(data) == b"2Q"

tests/extmod/uctypes_le.py.exp

@@ -0,0 +1,10 @@
+0x3130
+48 49
+TypeError
+arr: 48 49
+arr of struct: 48 49
+IndexError
+bf: 48 49
+bf 4bit: 3 1 3 0
+bytearray(b'21')
+bytearray(b'2Q')

tests/extmod/uctypes_native_le.py

@@ -0,0 +1,76 @@
+# This test is exactly like uctypes_le.py, but uses native structure layout.
+# Codepaths for packed vs native structures are different. This test only works
+# on little-endian machine (no matter if 32 or 64 bit).
+import sys
+import uctypes
+
+if sys.byteorder != "little":
+    print("SKIP")
+    sys.exit()
+
+
+desc = {
+    "s0": uctypes.UINT16 | 0,
+    "sub": (0, {
+        "b0": uctypes.UINT8 | 0,
+        "b1": uctypes.UINT8 | 1,
+    }),
+    "arr": (uctypes.ARRAY | 0, uctypes.UINT8 | 2),
+    "arr2": (uctypes.ARRAY | 0, 2, {"b": uctypes.UINT8 | 0}),
+    "bitf0": uctypes.BFUINT16 | 0 | 0 << 17 | 8 << 22,
+    "bitf1": uctypes.BFUINT16 | 0 | 8 << 17 | 8 << 22,
+
+    "bf0": uctypes.BFUINT16 | 0 | 0 << 17 | 4 << 22,
+    "bf1": uctypes.BFUINT16 | 0 | 4 << 17 | 4 << 22,
+    "bf2": uctypes.BFUINT16 | 0 | 8 << 17 | 4 << 22,
+    "bf3": uctypes.BFUINT16 | 0 | 12 << 17 | 4 << 22,
+
+    "ptr": (uctypes.PTR | 0, uctypes.UINT8),
+    "ptr2": (uctypes.PTR | 0, {"b": uctypes.UINT8 | 0}),
+}
+
+data = bytearray(b"01")
+
+S = uctypes.struct(desc, uctypes.addressof(data), uctypes.NATIVE)
+
+#print(S)
+print(hex(S.s0))
+assert hex(S.s0) == "0x3130"
+#print(S.sub.b0)
+print(S.sub.b0, S.sub.b1)
+assert S.sub.b0, S.sub.b1 == (0x30, 0x31)
+
+try:
+    S[0]
+    assert False, "Can't index struct"
+except TypeError:
+    print("TypeError")
+
+print("arr:", S.arr[0], S.arr[1])
+assert (S.arr[0], S.arr[1]) == (0x30, 0x31)
+
+print("arr of struct:", S.arr2[0].b, S.arr2[1].b)
+assert (S.arr2[0].b, S.arr2[1].b) == (0x30, 0x31)
+
+
+try:
+    S.arr[2]
+    assert False, "Out of bounds index"
+except IndexError:
+    print("IndexError")
+
+print("bf:", S.bitf0, S.bitf1)
+assert (S.bitf0, S.bitf1) == (0x30, 0x31)
+
+print("bf 4bit:", S.bf3, S.bf2, S.bf1, S.bf0)
+assert (S.bf3, S.bf2, S.bf1, S.bf0) == (3, 1, 3, 0)
+
+# Write access
+
+S.sub.b0 = ord("2")
+print(data)
+assert bytes(data) == b"21"
+
+S.bf3 = 5
+print(data)
+assert bytes(data) == b"2Q"

tests/extmod/uctypes_native_le.py.exp

@@ -0,0 +1,10 @@
+0x3130
+48 49
+TypeError
+arr: 48 49
+arr of struct: 48 49
+IndexError
+bf: 48 49
+bf 4bit: 3 1 3 0
+bytearray(b'21')
+bytearray(b'2Q')

tests/extmod/uctypes_ptr_le.py

@@ -0,0 +1,25 @@
+import uctypes
+
+desc = {
+    "ptr": (uctypes.PTR | 0, uctypes.UINT8),
+    "ptr16": (uctypes.PTR | 0, uctypes.UINT16),
+    "ptr2": (uctypes.PTR | 0, {"b": uctypes.UINT8 | 0}),
+}
+
+bytes = b"01"
+
+addr = uctypes.addressof(bytes)
+buf = addr.to_bytes(4)
+
+S = uctypes.struct(desc, uctypes.addressof(buf), uctypes.LITTLE_ENDIAN)
+
+print(S.ptr[0])
+assert S.ptr[0] == ord("0")
+print(S.ptr[1])
+assert S.ptr[1] == ord("1")
+print(hex(S.ptr16[0]))
+assert hex(S.ptr16[0]) == "0x3130"
+print(S.ptr2[0].b, S.ptr2[1].b)
+print (S.ptr2[0].b, S.ptr2[1].b)
+print(hex(S.ptr16[0]))
+assert (S.ptr2[0].b, S.ptr2[1].b) == (48, 49)

tests/extmod/uctypes_ptr_le.py.exp

@@ -0,0 +1,6 @@
+48
+49
+0x3130
+48 49
+48 49
+0x3130

tests/extmod/uctypes_ptr_native_le.py

@@ -0,0 +1,31 @@
+import sys
+import uctypes
+
+if sys.byteorder != "little":
+    print("SKIP")
+    sys.exit()
+
+
+desc = {
+    "ptr": (uctypes.PTR | 0, uctypes.UINT8),
+    "ptr16": (uctypes.PTR | 0, uctypes.UINT16),
+    "ptr2": (uctypes.PTR | 0, {"b": uctypes.UINT8 | 0}),
+}
+
+bytes = b"01"
+
+addr = uctypes.addressof(bytes)
+buf = addr.to_bytes(4)
+
+S = uctypes.struct(desc, uctypes.addressof(buf), uctypes.NATIVE)
+
+print(S.ptr[0])
+assert S.ptr[0] == ord("0")
+print(S.ptr[1])
+assert S.ptr[1] == ord("1")
+print(hex(S.ptr16[0]))
+assert hex(S.ptr16[0]) == "0x3130"
+print(S.ptr2[0].b, S.ptr2[1].b)
+print (S.ptr2[0].b, S.ptr2[1].b)
+print(hex(S.ptr16[0]))
+assert (S.ptr2[0].b, S.ptr2[1].b) == (48, 49)

tests/extmod/uctypes_ptr_native_le.py.exp

@@ -0,0 +1,6 @@
+48
+49
+0x3130
+48 49
+48 49
+0x3130

tests/extmod/uctypes_sizeof_native.py

@@ -0,0 +1,53 @@
+import uctypes
+
+S1 = {}
+assert uctypes.sizeof(S1) == 0
+
+S2 = {"a": uctypes.UINT8 | 0}
+assert uctypes.sizeof(S2) == 1
+
+S3 = {
+    "a": uctypes.UINT8 | 0,
+    "b": uctypes.UINT8 | 1,
+}
+assert uctypes.sizeof(S3) == 2
+
+S4 = {
+    "a": uctypes.UINT8 | 0,
+    "b": uctypes.UINT32 | 4,
+    "c": uctypes.UINT8 | 8,
+}
+assert uctypes.sizeof(S4) == 12
+
+S5 = {
+    "a": uctypes.UINT8 | 0,
+    "b": uctypes.UINT32 | 4,
+    "c": uctypes.UINT8 | 8,
+    "d": uctypes.UINT32 | 0,
+    "sub": (4, {
+        "b0": uctypes.UINT8 | 0,
+        "b1": uctypes.UINT8 | 1,
+    }),
+}
+
+assert uctypes.sizeof(S5) == 12
+
+s5 = uctypes.struct(S5, 0)
+assert uctypes.sizeof(s5) == 12
+assert uctypes.sizeof(s5.sub) == 2
+
+S6 = {
+    "ptr": (uctypes.PTR | 0, uctypes.UINT8),
+}
+# As if there're no other arch bitnesses
+assert uctypes.sizeof(S6) in (4, 8)
+
+S7 = {
+    "arr": (uctypes.ARRAY | 0, uctypes.UINT8 | 5),
+}
+assert uctypes.sizeof(S7) == 5
+
+S8 = {
+    "arr": (uctypes.ARRAY | 0, 3, {"a": uctypes.UINT32 | 0, "b": uctypes.UINT8 | 4}),
+}
+assert uctypes.sizeof(S8) == 24

unix/mpconfigport.h

@@ -50,6 +50,9 @@
 #define MICROPY_PY_CMATH            (1)
 #define MICROPY_PY_IO_FILEIO        (1)
 #define MICROPY_PY_GC_COLLECT_RETVAL (1)
+
+#define MICROPY_PY_UCTYPES          (1)
+
 // Define to MICROPY_ERROR_REPORTING_DETAILED to get function, etc.
 // names in exception messages (may require more RAM).
 #define MICROPY_ERROR_REPORTING     (MICROPY_ERROR_REPORTING_DETAILED)