bpo-11063, bpo-20519: avoid ctypes and improve import time for uuid (#3796)

bpo-11063, bpo-20519: avoid ctypes and improve import time for uuid.

Lib/uuid.py

@@ -45,6 +45,7 @@ Typical usage:
 """
 
 import os
+import sys
 
 from enum import Enum
 
@@ -475,73 +476,112 @@ def _netbios_getnode():
             continue
         return int.from_bytes(bytes, 'big')
 
-# Thanks to Thomas Heller for ctypes and for his help with its use here.
 
-# If ctypes is available, use it to find system routines for UUID generation.
-# XXX This makes the module non-thread-safe!
-_uuid_generate_time = _UuidCreate = None
+_generate_time_safe = _UuidCreate = None
+_has_uuid_generate_time_safe = None
+
+# Import optional C extension at toplevel, to help disabling it when testing
 try:
-    import ctypes, ctypes.util
-    import sys
+    import _uuid
+except ImportError:
+    _uuid = None
 
-    # The uuid_generate_* routines are provided by libuuid on at least
-    # Linux and FreeBSD, and provided by libc on Mac OS X.
-    _libnames = ['uuid']
-    if not sys.platform.startswith('win'):
-        _libnames.append('c')
-    for libname in _libnames:
-        try:
-            lib = ctypes.CDLL(ctypes.util.find_library(libname))
-        except Exception:                           # pragma: nocover
-            continue
-        # Try to find the safe variety first.
-        if hasattr(lib, 'uuid_generate_time_safe'):
-            _uuid_generate_time = lib.uuid_generate_time_safe
-            # int uuid_generate_time_safe(uuid_t out);
-            break
-        elif hasattr(lib, 'uuid_generate_time'):    # pragma: nocover
-            _uuid_generate_time = lib.uuid_generate_time
-            # void uuid_generate_time(uuid_t out);
-            _uuid_generate_time.restype = None
-            break
-    del _libnames
 
-    # The uuid_generate_* functions are broken on MacOS X 10.5, as noted
-    # in issue #8621 the function generates the same sequence of values
-    # in the parent process and all children created using fork (unless
-    # those children use exec as well).
-    #
-    # Assume that the uuid_generate functions are broken from 10.5 onward,
-    # the test can be adjusted when a later version is fixed.
-    if sys.platform == 'darwin':
-        if int(os.uname().release.split('.')[0]) >= 9:
-            _uuid_generate_time = None
+def _load_system_functions():
+    """
+    Try to load platform-specific functions for generating uuids.
+    """
+    global _generate_time_safe, _UuidCreate, _has_uuid_generate_time_safe
+
+    if _has_uuid_generate_time_safe is not None:
+        return
+
+    _has_uuid_generate_time_safe = False
+
+    if sys.platform == "darwin" and int(os.uname().release.split('.')[0]) < 9:
+        # The uuid_generate_* functions are broken on MacOS X 10.5, as noted
+        # in issue #8621 the function generates the same sequence of values
+        # in the parent process and all children created using fork (unless
+        # those children use exec as well).
+        #
+        # Assume that the uuid_generate functions are broken from 10.5 onward,
+        # the test can be adjusted when a later version is fixed.
+        pass
+    elif _uuid is not None:
+        _generate_time_safe = _uuid.generate_time_safe
+        _has_uuid_generate_time_safe = True
+        return
 
-    # On Windows prior to 2000, UuidCreate gives a UUID containing the
-    # hardware address.  On Windows 2000 and later, UuidCreate makes a
-    # random UUID and UuidCreateSequential gives a UUID containing the
-    # hardware address.  These routines are provided by the RPC runtime.
-    # NOTE:  at least on Tim's WinXP Pro SP2 desktop box, while the last
-    # 6 bytes returned by UuidCreateSequential are fixed, they don't appear
-    # to bear any relationship to the MAC address of any network device
-    # on the box.
     try:
-        lib = ctypes.windll.rpcrt4
-    except:
-        lib = None
-    _UuidCreate = getattr(lib, 'UuidCreateSequential',
-                          getattr(lib, 'UuidCreate', None))
-except:
-    pass
+        # If we couldn't find an extension module, try ctypes to find
+        # system routines for UUID generation.
+        # Thanks to Thomas Heller for ctypes and for his help with its use here.
+        import ctypes
+        import ctypes.util
 
-def _unixdll_getnode():
-    """Get the hardware address on Unix using ctypes."""
-    _buffer = ctypes.create_string_buffer(16)
-    _uuid_generate_time(_buffer)
-    return UUID(bytes=bytes_(_buffer.raw)).node
+        # The uuid_generate_* routines are provided by libuuid on at least
+        # Linux and FreeBSD, and provided by libc on Mac OS X.
+        _libnames = ['uuid']
+        if not sys.platform.startswith('win'):
+            _libnames.append('c')
+        for libname in _libnames:
+            try:
+                lib = ctypes.CDLL(ctypes.util.find_library(libname))
+            except Exception:                           # pragma: nocover
+                continue
+            # Try to find the safe variety first.
+            if hasattr(lib, 'uuid_generate_time_safe'):
+                _uuid_generate_time_safe = lib.uuid_generate_time_safe
+                # int uuid_generate_time_safe(uuid_t out);
+                def _generate_time_safe():
+                    _buffer = ctypes.create_string_buffer(16)
+                    res = _uuid_generate_time_safe(_buffer)
+                    return bytes(_buffer.raw), res
+                _has_uuid_generate_time_safe = True
+                break
+
+            elif hasattr(lib, 'uuid_generate_time'):    # pragma: nocover
+                _uuid_generate_time = lib.uuid_generate_time
+                # void uuid_generate_time(uuid_t out);
+                _uuid_generate_time.restype = None
+                def _generate_time_safe():
+                    _buffer = ctypes.create_string_buffer(16)
+                    _uuid_generate_time(_buffer)
+                    return bytes(_buffer.raw), None
+                break
+
+        # On Windows prior to 2000, UuidCreate gives a UUID containing the
+        # hardware address.  On Windows 2000 and later, UuidCreate makes a
+        # random UUID and UuidCreateSequential gives a UUID containing the
+        # hardware address.  These routines are provided by the RPC runtime.
+        # NOTE:  at least on Tim's WinXP Pro SP2 desktop box, while the last
+        # 6 bytes returned by UuidCreateSequential are fixed, they don't appear
+        # to bear any relationship to the MAC address of any network device
+        # on the box.
+        try:
+            lib = ctypes.windll.rpcrt4
+        except:
+            lib = None
+        _UuidCreate = getattr(lib, 'UuidCreateSequential',
+                              getattr(lib, 'UuidCreate', None))
+
+    except Exception as exc:
+        import warnings
+        warnings.warn(f"Could not find fallback ctypes uuid functions: {exc}",
+                      ImportWarning)
+
+
+def _unix_getnode():
+    """Get the hardware address on Unix using the _uuid extension module
+    or ctypes."""
+    _load_system_functions()
+    uuid_time, _ = _generate_time_safe()
+    return UUID(bytes=uuid_time).node
 
 def _windll_getnode():
     """Get the hardware address on Windows using ctypes."""
+    import ctypes
+    _load_system_functions()
     _buffer = ctypes.create_string_buffer(16)
     if _UuidCreate(_buffer) == 0:
         return UUID(bytes=bytes_(_buffer.raw)).node
@@ -551,6 +591,7 @@ def _random_getnode():
     import random
     return random.getrandbits(48) | 0x010000000000
 
+
 _node = None
 
 def getnode():
@@ -561,16 +602,14 @@ def getnode():
     choose a random 48-bit number with its eighth bit set to 1 as recommended
     in RFC 4122.
     """
-
     global _node
     if _node is not None:
         return _node
 
-    import sys
     if sys.platform == 'win32':
         getters = [_windll_getnode, _netbios_getnode, _ipconfig_getnode]
     else:
-        getters = [_unixdll_getnode, _ifconfig_getnode, _ip_getnode,
+        getters = [_unix_getnode, _ifconfig_getnode, _ip_getnode,
                    _arp_getnode, _lanscan_getnode, _netstat_getnode]
 
     for getter in getters + [_random_getnode]:
@@ -581,6 +620,7 @@ def getnode():
         if _node is not None:
             return _node
 
+
 _last_timestamp = None
 
 def uuid1(node=None, clock_seq=None):
@@ -591,14 +631,14 @@ def uuid1(node=None, clock_seq=None):
 
     # When the system provides a version-1 UUID generator, use it (but don't
     # use UuidCreate here because its UUIDs don't conform to RFC 4122).
-    if _uuid_generate_time and node is clock_seq is None:
-        _buffer = ctypes.create_string_buffer(16)
-        safely_generated = _uuid_generate_time(_buffer)
+    _load_system_functions()
+    if _generate_time_safe is not None and node is clock_seq is None:
+        uuid_time, safely_generated = _generate_time_safe()
         try:
             is_safe = SafeUUID(safely_generated)
         except ValueError:
             is_safe = SafeUUID.unknown
-        return UUID(bytes=bytes_(_buffer.raw), is_safe=is_safe)
+        return UUID(bytes=uuid_time, is_safe=is_safe)
 
     global _last_timestamp
     import time

Misc/NEWS.d/next/Library/2017-09-28-13-17-33.bpo-20519.FteeQQ.rst

@@ -0,0 +1 @@
+Avoid ctypes use (if possible) and improve import time for uuid.

Modules/_uuidmodule.c

@@ -0,0 +1,35 @@
+#define PY_SSIZE_T_CLEAN
+
+#include "Python.h"
+#include <uuid/uuid.h>
+
+
+static PyObject *
+py_uuid_generate_time_safe(void)
+{
+    uuid_t out;
+    int res;
+
+    res = uuid_generate_time_safe(out);
+    return Py_BuildValue("y#i", (const char *) out, sizeof(out), res);
+}
+
+
+static PyMethodDef uuid_methods[] = {
+    {"generate_time_safe", (PyCFunction) py_uuid_generate_time_safe, METH_NOARGS, NULL},
+    {NULL, NULL, 0, NULL}           /* sentinel */
+};
+
+static struct PyModuleDef uuidmodule = {
+    PyModuleDef_HEAD_INIT,
+    .m_name = "_uuid",
+    .m_size = -1,
+    .m_methods = uuid_methods,
+};
+
+PyMODINIT_FUNC
+PyInit__uuid(void)
+{
+    assert(sizeof(uuid_t) == 16);
+    return PyModule_Create(&uuidmodule);
+}

setup.py

@@ -1668,6 +1668,20 @@ class PyBuildExt(build_ext):
         if '_tkinter' not in [e.name for e in self.extensions]:
             missing.append('_tkinter')
 
+        # Build the _uuid module if possible
+        uuid_incs = find_file("uuid.h", inc_dirs, ["/usr/include/uuid"])
+        if uuid_incs:
+            if self.compiler.find_library_file(lib_dirs, 'uuid'):
+                uuid_libs = ['uuid']
+            else:
+                uuid_libs = []
+        if uuid_incs:
+            self.extensions.append(Extension('_uuid', ['_uuidmodule.c'],
+                                   libraries=uuid_libs,
+                                   include_dirs=uuid_incs))
+        else:
+            missing.append('_uuid')
+
 ##         # Uncomment these lines if you want to play with xxmodule.c
 ##         ext = Extension('xx', ['xxmodule.c'])
 ##         self.extensions.append(ext)