From f385063603ac76f4765a53ce51cc1404e135a020 Mon Sep 17 00:00:00 2001 From: Thomas Petazzoni Date: Wed, 22 Feb 2017 16:21:31 -0800 Subject: [PATCH 01/33] Make the build of pyc files conditional This commit adds a new configure option --disable-pyc-build to disable the compilation of pyc. Signed-off-by: Thomas Petazzoni [ Andrey Smrinov: ported to Python 3.6 ] Signed-off-by: Andrey Smirnov --- Makefile.pre.in | 2 ++ configure.ac | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/Makefile.pre.in b/Makefile.pre.in index beaccf5..bc205d1 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1404,6 +1404,7 @@ libinstall: build_all $(srcdir)/Modules/xxmodule.c $(INSTALL_DATA) $(srcdir)/Modules/xxmodule.c \ $(DESTDIR)$(LIBDEST)/distutils/tests ; \ fi +ifeq (@PYC_BUILD@,yes) -PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \ $(PYTHON_FOR_BUILD) -Wi $(DESTDIR)$(LIBDEST)/compileall.py \ -d $(LIBDEST) -f \ @@ -1431,6 +1432,7 @@ libinstall: build_all $(srcdir)/Modules/xxmodule.c $(PYTHON_FOR_BUILD) -Wi -OO $(DESTDIR)$(LIBDEST)/compileall.py \ -d $(LIBDEST)/site-packages -f \ -x badsyntax $(DESTDIR)$(LIBDEST)/site-packages +endif -PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \ $(PYTHON_FOR_BUILD) -m lib2to3.pgen2.driver $(DESTDIR)$(LIBDEST)/lib2to3/Grammar.txt -PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \ diff --git a/configure.ac b/configure.ac index 1ef0df7..74b6526 100644 --- a/configure.ac +++ b/configure.ac @@ -1107,6 +1107,12 @@ fi AC_MSG_CHECKING(LDLIBRARY) +AC_SUBST(PYC_BUILD) + +AC_ARG_ENABLE(pyc-build, + AS_HELP_STRING([--disable-pyc-build], [disable build of pyc files]), + [ PYC_BUILD="${enableval}" ], [ PYC_BUILD=yes ]) + # MacOSX framework builds need more magic. LDLIBRARY is the dynamic # library that we build, but we do not want to link against it (we # will find it with a -framework option). For this reason there is an -- 2.7.4 From 841e315367262f7567e76f80820485ba7a6fc529 Mon Sep 17 00:00:00 2001 From: Vanya Sergeev Date: Wed, 23 Dec 2015 11:30:33 +0100 Subject: [PATCH 02/33] Disable buggy_getaddrinfo configure test when cross-compiling with IPv6 support Signed-off-by: Vanya Sergeev --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 74b6526..e043d35 100644 --- a/configure.ac +++ b/configure.ac @@ -4059,7 +4059,7 @@ fi AC_MSG_RESULT($ac_cv_buggy_getaddrinfo) -if test $have_getaddrinfo = no || test "$ac_cv_buggy_getaddrinfo" = yes +if test $have_getaddrinfo = no || test "$cross_compiling" != "yes" -a "$ac_cv_buggy_getaddrinfo" = yes then if test $ipv6 = yes then -- 2.7.4 From c5287f9cd9b29a5bd6c75da6a2c541d33392cdbc Mon Sep 17 00:00:00 2001 From: Thomas Petazzoni Date: Wed, 22 Feb 2017 16:33:22 -0800 Subject: [PATCH 03/33] Add infrastructure to disable the build of certain extensions Some of the extensions part of the Python core have dependencies on external libraries (sqlite, tk, etc.) or are relatively big and not necessarly always useful (CJK codecs for example). By extensions, we mean part of Python modules that are written in C and therefore compiled to binary code. Therefore, we introduce a small infrastructure that allows to disable some of those extensions. This can be done inside the configure.ac by adding values to the DISABLED_EXTENSIONS variable (which is a word-separated list of extensions). The implementation works as follow : * configure.ac defines a DISABLED_EXTENSIONS variable, which is substituted (so that when Makefile.pre is generated from Makefile.pre.in, the value of the variable is substituted). For now, this DISABLED_EXTENSIONS variable is empty, later patches will use it. * Makefile.pre.in passes the DISABLED_EXTENSIONS value down to the variables passed in the environment when calling the setup.py script that actually builds and installs those extensions. * setup.py is modified so that the existing "disabled_module_list" is filled with those pre-disabled extensions listed in DISABLED_EXTENSIONS. Patch ported to python2.7 by Maxime Ripard , and then extended by Thomas Petazzoni . Signed-off-by: Thomas Petazzoni [ Andrey Smirnov: ported to Python 3.6 ] Signed-off-by: Andrey Smirnov --- Makefile.pre.in | 6 +++++- configure.ac | 2 ++ setup.py | 6 +++++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index bc205d1..0fd1db7 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -206,6 +206,8 @@ FILEMODE= 644 # configure script arguments CONFIG_ARGS= @CONFIG_ARGS@ +# disabled extensions +DISABLED_EXTENSIONS= @DISABLED_EXTENSIONS@ # Subdirectories with code SRCDIRS= @SRCDIRS@ @@ -620,6 +622,7 @@ sharedmods: $(BUILDPYTHON) pybuilddir.txt Modules/_math.o esac; \ echo "$(RUNSHARED) CC='$(CC)' LDSHARED='$(BLDSHARED)' OPT='$(OPT)' \ _TCLTK_INCLUDES='$(TCLTK_INCLUDES)' _TCLTK_LIBS='$(TCLTK_LIBS)' \ + DISABLED_EXTENSIONS="$(DISABLED_EXTENSIONS)" \ $(PYTHON_FOR_BUILD) $(srcdir)/setup.py $$quiet build"; \ $(RUNSHARED) CC='$(CC)' LDSHARED='$(BLDSHARED)' OPT='$(OPT)' \ _TCLTK_INCLUDES='$(TCLTK_INCLUDES)' _TCLTK_LIBS='$(TCLTK_LIBS)' \ @@ -1537,7 +1540,8 @@ libainstall: @DEF_MAKE_RULE@ python-config # Install the dynamically loadable modules # This goes into $(exec_prefix) sharedinstall: sharedmods - $(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/setup.py install \ + $(RUNSHARED) DISABLED_EXTENSIONS="$(DISABLED_EXTENSIONS)" \ + $(PYTHON_FOR_BUILD) $(srcdir)/setup.py install \ --prefix=$(prefix) \ --install-scripts=$(BINDIR) \ --install-platlib=$(DESTSHARED) \ diff --git a/configure.ac b/configure.ac index e043d35..788516d 100644 --- a/configure.ac +++ b/configure.ac @@ -2966,6 +2966,8 @@ LIBS="$withval $LIBS" PKG_PROG_PKG_CONFIG +AC_SUBST(DISABLED_EXTENSIONS) + # Check for use of the system expat library AC_MSG_CHECKING(for --with-system-expat) AC_ARG_WITH(system_expat, diff --git a/setup.py b/setup.py index 88cff61..6c56426 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,11 @@ host_platform = get_platform() COMPILED_WITH_PYDEBUG = ('--with-pydebug' in sysconfig.get_config_var("CONFIG_ARGS")) # This global variable is used to hold the list of modules to be disabled. -disabled_module_list = [] +try: + disabled_module_list = sysconfig.get_config_var("DISABLED_EXTENSIONS").split(" ") +except KeyError: + disabled_module_list = list() + def add_dir_to_list(dirlist, dir): """Add the directory 'dir' to the list 'dirlist' (after any relative -- 2.7.4 From 0b8e9819206e749f8400ad6c535023db24370e79 Mon Sep 17 00:00:00 2001 From: Thomas Petazzoni Date: Wed, 23 Dec 2015 11:33:14 +0100 Subject: [PATCH 04/33] Adjust library/header paths for cross-compilation When cross-compiling third-party extensions, the get_python_inc() or get_python_lib() can be called, to return the path to headers or libraries. However, they use the sys.prefix of the host Python, which returns incorrect paths when cross-compiling (paths pointing to host headers and libraries). In order to fix this, we introduce the _python_sysroot, _python_prefix and _python_exec_prefix variables, that allow to override these values, and get correct header/library paths when cross-compiling third-party Python modules. Signed-off-by: Thomas Petazzoni --- Lib/distutils/command/build_ext.py | 5 ++++- Lib/distutils/sysconfig.py | 15 +++++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/Lib/distutils/command/build_ext.py b/Lib/distutils/command/build_ext.py index 0428466..dbf879a 100644 --- a/Lib/distutils/command/build_ext.py +++ b/Lib/distutils/command/build_ext.py @@ -234,7 +234,10 @@ class build_ext(Command): if (sysconfig.get_config_var('Py_ENABLE_SHARED')): if not sysconfig.python_build: # building third party extensions - self.library_dirs.append(sysconfig.get_config_var('LIBDIR')) + libdir = sysconfig.get_config_var('LIBDIR') + if "_python_sysroot" in os.environ: + libdir = os.environ.get("_python_sysroot") + libdir + self.library_dirs.append(libdir) else: # building python standard extensions self.library_dirs.append('.') diff --git a/Lib/distutils/sysconfig.py b/Lib/distutils/sysconfig.py index 0a034ee..e5dec2d 100644 --- a/Lib/distutils/sysconfig.py +++ b/Lib/distutils/sysconfig.py @@ -17,10 +17,17 @@ import sys from .errors import DistutilsPlatformError # These are needed in a couple of spots, so just compute them once. -PREFIX = os.path.normpath(sys.prefix) -EXEC_PREFIX = os.path.normpath(sys.exec_prefix) -BASE_PREFIX = os.path.normpath(sys.base_prefix) -BASE_EXEC_PREFIX = os.path.normpath(sys.base_exec_prefix) +if "_python_sysroot" in os.environ: + _sysroot=os.environ.get('_python_sysroot') + PREFIX = os.path.normpath(_sysroot + os.environ.get('_python_prefix')) + EXEC_PREFIX = os.path.normpath(_sysroot + os.environ.get('_python_exec_prefix')) + BASE_PREFIX = PREFIX + BASE_EXEC_PREFIX = EXEC_PREFIX +else: + PREFIX = os.path.normpath(sys.prefix) + EXEC_PREFIX = os.path.normpath(sys.exec_prefix) + BASE_PREFIX = os.path.normpath(sys.base_prefix) + BASE_EXEC_PREFIX = os.path.normpath(sys.base_exec_prefix) # Path to the base directory of the project. On Windows the binary may # live in project/PCbuild/win32 or project/PCbuild/amd64. -- 2.7.4 From ac7a2166ab7c991f3201841f20d9301fd879a9e2 Mon Sep 17 00:00:00 2001 From: Thomas Petazzoni Date: Wed, 23 Dec 2015 11:36:00 +0100 Subject: [PATCH 05/33] Don't look in /usr/lib/termcap for libraries Signed-off-by: Thomas Petazzoni --- setup.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 6c56426..1b0838a 100644 --- a/setup.py +++ b/setup.py @@ -894,12 +894,9 @@ class PyBuildExt(build_ext): pass # Issue 7384: Already linked against curses or tinfo. elif curses_library: readline_libs.append(curses_library) - elif self.compiler.find_library_file(lib_dirs + - ['/usr/lib/termcap'], - 'termcap'): + elif self.compiler.find_library_file(lib_dirs, 'termcap'): readline_libs.append('termcap') exts.append( Extension('readline', ['readline.c'], - library_dirs=['/usr/lib/termcap'], extra_link_args=readline_extra_link_args, libraries=readline_libs) ) else: -- 2.7.4 From b140aca6c2eef959b07d5863c565fa5bdfee0ab1 Mon Sep 17 00:00:00 2001 From: Thomas Petazzoni Date: Wed, 23 Dec 2015 11:36:27 +0100 Subject: [PATCH 06/33] Don't add multiarch paths The add_multiarch_paths() function leads, in certain build environments, to the addition of host header paths to the CFLAGS, which is not appropriate for cross-compilation. This patch fixes that by simply removing the call to add_multiarch_paths() when we're cross-compiling. Investigation done by David . Signed-off-by: Thomas Petazzoni --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1b0838a..ce7472d 100644 --- a/setup.py +++ b/setup.py @@ -591,10 +591,10 @@ class PyBuildExt(build_ext): if not cross_compiling: add_dir_to_list(self.compiler.library_dirs, '/usr/local/lib') add_dir_to_list(self.compiler.include_dirs, '/usr/local/include') + self.add_multiarch_paths() # only change this for cross builds for 3.3, issues on Mageia if cross_compiling: self.add_gcc_paths() - self.add_multiarch_paths() # Add paths specified in the environment variables LDFLAGS and # CPPFLAGS for header and library files. -- 2.7.4 From 3601f296ad1e343a522ba19cd3a4795c809d4519 Mon Sep 17 00:00:00 2001 From: Thomas Petazzoni Date: Wed, 23 Dec 2015 11:43:24 +0100 Subject: [PATCH 07/33] Abort on failed module build When building a Python module fails, the setup.py script currently doesn't exit with an error, and simply continues. This is not a really nice behavior, so this patch changes setup.py to abort with an error, so that the build issue is clearly noticeable. Signed-off-by: Thomas Petazzoni --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index ce7472d..da7984c 100644 --- a/setup.py +++ b/setup.py @@ -402,6 +402,7 @@ class PyBuildExt(build_ext): print("Failed to build these modules:") print_three_column(failed) print() + sys.exit(1) if self.failed_on_import: failed = self.failed_on_import[:] -- 2.7.4 From 381f68374b6f58497bc4e745f2dc884cbcc10a41 Mon Sep 17 00:00:00 2001 From: Baruch Siach Date: Wed, 23 Dec 2015 11:44:02 +0100 Subject: [PATCH 08/33] Serial ioctl() workaround The ioctls.h of some architectures (notably xtensa) references structs from linux/serial.h. Make sure to include this header as well. Also, undef TIOCTTYGSTRUCT that require reference to internal kernel tty_struct, but isn't actually referenced in modern kernels. Signed-off-by: Baruch Siach --- Modules/termios.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Modules/termios.c b/Modules/termios.c index 7601b68..36e4828 100644 --- a/Modules/termios.c +++ b/Modules/termios.c @@ -15,7 +15,9 @@ #endif #include +#include #include +#undef TIOCTTYGSTRUCT /* HP-UX requires that this be included to pick up MDCD, MCTS, MDSR, * MDTR, MRI, and MRTS (appearantly used internally by some things -- 2.7.4 From e8bae9f4b05b4f3053975af402dd47f4e1b5cb33 Mon Sep 17 00:00:00 2001 From: Christophe Vu-Brugier Date: Wed, 23 Dec 2015 11:44:30 +0100 Subject: [PATCH 09/33] Do not adjust the shebang of Python scripts for cross-compilation The copy_scripts() method in distutils copies the scripts listed in the setup file and adjusts the first line to refer to the current Python interpreter. When cross-compiling, this means that the adjusted shebang refers to the host Python interpreter. This patch modifies copy_scripts() to preserve the shebang when cross-compilation is detected. Signed-off-by: Christophe Vu-Brugier --- Lib/distutils/command/build_scripts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/distutils/command/build_scripts.py b/Lib/distutils/command/build_scripts.py index ccc70e6..d6d5419 100644 --- a/Lib/distutils/command/build_scripts.py +++ b/Lib/distutils/command/build_scripts.py @@ -91,7 +91,7 @@ class build_scripts(Command): adjust = True post_interp = match.group(1) or b'' - if adjust: + if adjust and not '_python_sysroot' in os.environ: log.info("copying and adjusting %s -> %s", script, self.build_dir) updated_files.append(outfile) -- 2.7.4 From 7a99661a038e7989add860d92304bcd59df644b5 Mon Sep 17 00:00:00 2001 From: Peter Korsgaard Date: Thu, 20 Nov 2014 13:24:59 +0100 Subject: [PATCH 10/33] Misc/python-config.sh.in: ensure sed invocations only match beginning of strings The build/real prefix handling using sed breaks if build != real and the standard include / lib directories are used ($prefix/include and $prefix/lib). E.G. prefix_build="/usr", libdir="$prefix/lib", includedir="$prefix/include". If this gets installed with make DESTDIR="/foo" install, then we end up with prefix_real = prefix = "/foo/usr" as expected, but includedir="/foo/foo/usr/include" and libdir="/foo/foo/usr/lib" because of the double sed invocation (prefix is already expanded). Work around it by ensuring we only match the beginning of the string. Submitted upstream: http://bugs.python.org/issue22907 Signed-off-by: Peter Korsgaard --- Misc/python-config.sh.in | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Misc/python-config.sh.in b/Misc/python-config.sh.in index a3c479c..db91bd6 100644 --- a/Misc/python-config.sh.in +++ b/Misc/python-config.sh.in @@ -24,18 +24,19 @@ installed_prefix () echo $RESULT } +prefix_build="@prefix@" prefix_real=$(installed_prefix "$0") # Use sed to fix paths from their built-to locations to their installed-to # locations. Keep prefix & exec_prefix using their original values in case # they are referenced in other configure variables, to prevent double # substitution, issue #22140. -prefix="@prefix@" -exec_prefix="@exec_prefix@" +prefix=$(echo "$prefix_build" | sed "s#^$prefix_build#$prefix_real#") +exec_prefix=$(echo "$exec_prefix_build" | sed "s#^$exec_prefix_build#$prefix_real#") exec_prefix_real=${prefix_real} -includedir=$(echo "@includedir@" | sed "s#$prefix#$prefix_real#") -libdir=$(echo "@libdir@" | sed "s#$prefix#$prefix_real#") -CFLAGS=$(echo "@CFLAGS@" | sed "s#$prefix#$prefix_real#") +includedir=$(echo "@includedir@" | sed "s#^$prefix_build#$prefix_real#") +libdir=$(echo "@libdir@" | sed "s#^$prefix_build#$prefix_real#") +CFLAGS=$(echo "@CFLAGS@" | sed "s#^$prefix_build#$prefix_real#") VERSION="@VERSION@" LIBM="@LIBM@" LIBC="@LIBC@" @@ -48,7 +49,7 @@ OPT="@OPT@" PY_ENABLE_SHARED="@PY_ENABLE_SHARED@" LDVERSION="@LDVERSION@" LIBDEST=${prefix_real}/lib/python${VERSION} -LIBPL=$(echo "@LIBPL@" | sed "s#$prefix#$prefix_real#") +LIBPL=$(echo "@LIBPL@" | sed "s#^$prefix_build#$prefix_real#") SO="@EXT_SUFFIX@" PYTHONFRAMEWORK="@PYTHONFRAMEWORK@" INCDIR="-I$includedir/python${VERSION}${ABIFLAGS}" -- 2.7.4 From c7744bc1ae27dd893167dba3328de426d72ea371 Mon Sep 17 00:00:00 2001 From: Samuel Cabrero Date: Wed, 23 Dec 2015 11:45:48 +0100 Subject: [PATCH 11/33] Override system locale and set to default when adding gcc paths Forces the use of the default locale in the function add_gcc_paths, which is called when cross compiling to add the include and library paths. This is necessary because otherwise the gcc output is localized and the output parsing fails, which results in no paths added and detect_modules not able to find any system library (eg. libz, libssl, etc.) [Thomas: patch taken from https://bugs.python.org/issue23767.] Signed-off-by: Samuel Cabrero Signed-off-by: Thomas Petazzoni --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index da7984c..646440e 100644 --- a/setup.py +++ b/setup.py @@ -558,7 +558,7 @@ class PyBuildExt(build_ext): tmpfile = os.path.join(self.build_temp, 'gccpaths') if not os.path.exists(self.build_temp): os.makedirs(self.build_temp) - ret = os.system('%s -E -v - %s 1>/dev/null' % (gcc, tmpfile)) + ret = os.system('LC_ALL=C %s -E -v - %s 1>/dev/null' % (gcc, tmpfile)) is_gcc = False in_incdirs = False inc_dirs = [] -- 2.7.4 From 07805143eda6e2477525db660922eb690348fada Mon Sep 17 00:00:00 2001 From: Christophe Vu-Brugier Date: Wed, 22 Feb 2017 16:48:49 -0800 Subject: [PATCH 12/33] Add importlib fix for PEP 3147 issue Python 3 has a new standard for installing .pyc file, called PEP 3147. Unfortunately, this standard requires both the .py and .pyc files to be installed for a Python module to be found. This is quite annoying on space-constrained embedded systems, since the .py file is technically not required for execution. This patch changes cache_from_source() and source_from_cache() in importlib to get rid of the "__pycache__" directory. This effectively disables PEP 3147 for: * The python standard library * Packages built with distutils or setuptools * Packages built with automake that use the `py-compile` helper Signed-off-by: Christophe Vu-Brugier [ Andrey Smirnov: ported to Python 3.6 ] Signed-off-by: Andrey Smirnov --- Lib/importlib/_bootstrap_external.py | 38 +++++------------------------------- 1 file changed, 5 insertions(+), 33 deletions(-) diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 66a16a6..1638060 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -283,8 +283,6 @@ def cache_from_source(path, debug_override=None, *, optimization=None): a True value is the same as setting 'optimization' to the empty string while a False value is equivalent to setting 'optimization' to '1'. - If sys.implementation.cache_tag is None then NotImplementedError is raised. - """ if debug_override is not None: _warnings.warn('the debug_override parameter is deprecated; use ' @@ -296,10 +294,7 @@ def cache_from_source(path, debug_override=None, *, optimization=None): path = _os.fspath(path) head, tail = _path_split(path) base, sep, rest = tail.rpartition('.') - tag = sys.implementation.cache_tag - if tag is None: - raise NotImplementedError('sys.implementation.cache_tag is None') - almost_filename = ''.join([(base if base else rest), sep, tag]) + almost_filename = ''.join([(base if base else rest)]) if optimization is None: if sys.flags.optimize == 0: optimization = '' @@ -310,40 +305,17 @@ def cache_from_source(path, debug_override=None, *, optimization=None): if not optimization.isalnum(): raise ValueError('{!r} is not alphanumeric'.format(optimization)) almost_filename = '{}.{}{}'.format(almost_filename, _OPT, optimization) - return _path_join(head, _PYCACHE, almost_filename + BYTECODE_SUFFIXES[0]) + return _path_join(head, almost_filename + BYTECODE_SUFFIXES[0]) def source_from_cache(path): """Given the path to a .pyc. file, return the path to its .py file. The .pyc file does not need to exist; this simply returns the path to - the .py file calculated to correspond to the .pyc file. If path does - not conform to PEP 3147/488 format, ValueError will be raised. If - sys.implementation.cache_tag is None then NotImplementedError is raised. - + the .py file calculated to correspond to the .pyc file. """ - if sys.implementation.cache_tag is None: - raise NotImplementedError('sys.implementation.cache_tag is None') - path = _os.fspath(path) - head, pycache_filename = _path_split(path) - head, pycache = _path_split(head) - if pycache != _PYCACHE: - raise ValueError('{} not bottom-level directory in ' - '{!r}'.format(_PYCACHE, path)) - dot_count = pycache_filename.count('.') - if dot_count not in {2, 3}: - raise ValueError('expected only 2 or 3 dots in ' - '{!r}'.format(pycache_filename)) - elif dot_count == 3: - optimization = pycache_filename.rsplit('.', 2)[-2] - if not optimization.startswith(_OPT): - raise ValueError("optimization portion of filename does not start " - "with {!r}".format(_OPT)) - opt_level = optimization[len(_OPT):] - if not opt_level.isalnum(): - raise ValueError("optimization level {!r} is not an alphanumeric " - "value".format(optimization)) - base_filename = pycache_filename.partition('.')[0] + head, filename = _path_split(path) + base_filename = filename.partition('.')[0] return _path_join(head, base_filename + SOURCE_SUFFIXES[0]) -- 2.7.4 From e95437f70805736f9a23dcfa0e981ecbb0c458cc Mon Sep 17 00:00:00 2001 From: Thomas Petazzoni Date: Wed, 22 Feb 2017 17:01:18 -0800 Subject: [PATCH 13/33] Add an option to disable installation of test modules The Python standard distribution comes with many test modules, that are not necessarly useful on embedded targets. Signed-off-by: Thomas Petazzoni Signed-off-by: Samuel Martin [ Andrey Smirnov: ported to Python 3.6 ] Signed-off-by: Andrey Smirnov --- Makefile.pre.in | 54 ++++++++++++++++++++++++++++++++++++------------------ configure.ac | 5 +++++ 2 files changed, 41 insertions(+), 18 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 0fd1db7..353236b 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1270,8 +1270,28 @@ maninstall: altmaninstall # Install the library XMLLIBSUBDIRS= xml xml/dom xml/etree xml/parsers xml/sax -LIBSUBDIRS= tkinter tkinter/test tkinter/test/test_tkinter \ - tkinter/test/test_ttk site-packages test \ + +LIBSUBDIRS= tkinter site-packages \ + asyncio \ + collections concurrent concurrent/futures encodings \ + email email/mime \ + ensurepip ensurepip/_bundled \ + html json http dbm xmlrpc \ + sqlite3 \ + logging csv wsgiref urllib \ + lib2to3 lib2to3/fixes lib2to3/pgen2 \ + ctypes ctypes/macholib \ + idlelib idlelib/Icons \ + distutils distutils/command $(XMLLIBSUBDIRS) \ + importlib \ + turtledemo \ + multiprocessing multiprocessing/dummy \ + unittest \ + venv venv/scripts venv/scripts/common venv/scripts/posix \ + curses pydoc_data + +TESTSUBDIRS= tkinter/test tkinter/test/test_tkinter \ + tkinter/test/test_ttk test \ test/audiodata \ test/capath test/data \ test/cjkencodings test/decimaltestdata test/xmltestdata \ @@ -1325,26 +1345,24 @@ LIBSUBDIRS= tkinter tkinter/test tkinter/test/test_tkinter \ test/test_importlib/source \ test/test_importlib/zipdata01 \ test/test_importlib/zipdata02 \ - asyncio \ test/test_asyncio \ - collections concurrent concurrent/futures encodings \ - email email/mime test/test_email test/test_email/data \ - ensurepip ensurepip/_bundled \ - html json test/test_json http dbm xmlrpc \ - sqlite3 sqlite3/test \ - logging csv wsgiref urllib \ - lib2to3 lib2to3/fixes lib2to3/pgen2 lib2to3/tests \ + test/test_email test/test_email/data \ + test/test_json \ + sqlite3/test \ + lib2to3/tests \ lib2to3/tests/data lib2to3/tests/data/fixers \ lib2to3/tests/data/fixers/myfixes \ - ctypes ctypes/test ctypes/macholib \ - idlelib idlelib/Icons idlelib/idle_test \ - distutils distutils/command distutils/tests $(XMLLIBSUBDIRS) \ + ctypes/test \ + idlelib/idle_test \ + distutils/tests \ + test/test_importlib test/test_importlib/builtin \ test/test_tools test/test_warnings test/test_warnings/data \ - turtledemo \ - multiprocessing multiprocessing/dummy \ - unittest unittest/test unittest/test/testmock \ - venv venv/scripts venv/scripts/common venv/scripts/posix \ - curses pydoc_data + unittest/test unittest/test/testmock + +ifeq (@TEST_MODULES@,yes) +LIBSUBDIRS += $(TESTSUBDIRS) +endif + libinstall: build_all $(srcdir)/Modules/xxmodule.c @for i in $(SCRIPTDIR) $(LIBDEST); \ do \ diff --git a/configure.ac b/configure.ac index 788516d..097793e 100644 --- a/configure.ac +++ b/configure.ac @@ -3229,6 +3229,11 @@ if test "$posix_threads" = "yes"; then AC_CHECK_FUNCS(pthread_getcpuclockid) fi +AC_SUBST(TEST_MODULES) + +AC_ARG_ENABLE(test-modules, + AS_HELP_STRING([--disable-test-modules], [disable test modules]), + [ TEST_MODULES="${enableval}" ], [ TEST_MODULES=yes ]) # Check for enable-ipv6 AH_TEMPLATE(ENABLE_IPV6, [Define if --enable-ipv6 is specified]) -- 2.7.4 From a7fe54a20781adbb7f63fa0162c1cdc47377999e Mon Sep 17 00:00:00 2001 From: Thomas Petazzoni Date: Wed, 22 Feb 2017 17:07:56 -0800 Subject: [PATCH 14/33] Add an option to disable pydoc It removes 0.5 MB of data from the target plus the pydoc script itself. Signed-off-by: Thomas Petazzoni Signed-off-by: Samuel Martin [ Andrey Smirnov: ported to Python 3.6 ] Signed-off-by: Andrey Smirnov --- Makefile.pre.in | 8 +++++++- configure.ac | 6 ++++++ setup.py | 9 +++++++-- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 353236b..5980c69 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1240,7 +1240,9 @@ bininstall: altbininstall -rm -f $(DESTDIR)$(BINDIR)/idle3 (cd $(DESTDIR)$(BINDIR); $(LN) -s idle$(VERSION) idle3) -rm -f $(DESTDIR)$(BINDIR)/pydoc3 +ifeq (@PYDOC@,yes) (cd $(DESTDIR)$(BINDIR); $(LN) -s pydoc$(VERSION) pydoc3) +endif -rm -f $(DESTDIR)$(BINDIR)/2to3 (cd $(DESTDIR)$(BINDIR); $(LN) -s 2to3-$(VERSION) 2to3) -rm -f $(DESTDIR)$(BINDIR)/pyvenv @@ -1288,7 +1290,7 @@ LIBSUBDIRS= tkinter site-packages \ multiprocessing multiprocessing/dummy \ unittest \ venv venv/scripts venv/scripts/common venv/scripts/posix \ - curses pydoc_data + curses TESTSUBDIRS= tkinter/test tkinter/test/test_tkinter \ tkinter/test/test_ttk test \ @@ -1363,6 +1365,10 @@ ifeq (@TEST_MODULES@,yes) LIBSUBDIRS += $(TESTSUBDIRS) endif +ifeq (@PYDOC@,yes) +LIBSUBDIRS += pydoc_data +endif + libinstall: build_all $(srcdir)/Modules/xxmodule.c @for i in $(SCRIPTDIR) $(LIBDEST); \ do \ diff --git a/configure.ac b/configure.ac index 097793e..2cdcb63 100644 --- a/configure.ac +++ b/configure.ac @@ -3229,6 +3229,12 @@ if test "$posix_threads" = "yes"; then AC_CHECK_FUNCS(pthread_getcpuclockid) fi +AC_SUBST(PYDOC) + +AC_ARG_ENABLE(pydoc, + AS_HELP_STRING([--disable-pydoc], [disable pydoc]), + [ PYDOC="${enableval}" ], [ PYDOC=yes ]) + AC_SUBST(TEST_MODULES) AC_ARG_ENABLE(test-modules, diff --git a/setup.py b/setup.py index 646440e..11ac959 100644 --- a/setup.py +++ b/setup.py @@ -2376,6 +2376,12 @@ def main(): # turn off warnings when deprecated modules are imported import warnings warnings.filterwarnings("ignore",category=DeprecationWarning) + + scripts = ['Tools/scripts/idle3', 'Tools/scripts/2to3', + 'Lib/smtpd.py'] + if not '--disable-pydoc' in sysconfig.get_config_var("CONFIG_ARGS"): + scripts += [ 'Tools/scripts/pydoc3' ] + setup(# PyPI Metadata (PEP 301) name = "Python", version = sys.version.split()[0], @@ -2400,8 +2406,7 @@ def main(): # If you change the scripts installed here, you also need to # check the PyBuildScripts command above, and change the links # created by the bininstall target in Makefile.pre.in - scripts = ["Tools/scripts/pydoc3", "Tools/scripts/idle3", - "Tools/scripts/2to3", "Tools/scripts/pyvenv"] + scripts = scripts ) # --install-platlib -- 2.7.4 From 2b57b759f1a91dd99e80096be6abf1c56ae4b3dc Mon Sep 17 00:00:00 2001 From: Thomas Petazzoni Date: Wed, 22 Feb 2017 17:15:31 -0800 Subject: [PATCH 15/33] Add an option to disable lib2to3 lib2to3 is a library to convert Python 2.x code to Python 3.x. As such, it is probably not very useful on embedded system targets. Signed-off-by: Thomas Petazzoni Signed-off-by: Samuel Martin [ Andrey Smirnov: ported to Python 3.6 ] Signed-off-by: Andrey Smirnov --- Makefile.pre.in | 16 ++++++++++++---- configure.ac | 6 ++++++ setup.py | 5 +++-- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 5980c69..037362c 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1244,7 +1244,9 @@ ifeq (@PYDOC@,yes) (cd $(DESTDIR)$(BINDIR); $(LN) -s pydoc$(VERSION) pydoc3) endif -rm -f $(DESTDIR)$(BINDIR)/2to3 +ifeq (@LIB2TO3@,yes) (cd $(DESTDIR)$(BINDIR); $(LN) -s 2to3-$(VERSION) 2to3) +endif -rm -f $(DESTDIR)$(BINDIR)/pyvenv (cd $(DESTDIR)$(BINDIR); $(LN) -s pyvenv-$(VERSION) pyvenv) if test "x$(LIPO_32BIT_FLAGS)" != "x" ; then \ @@ -1281,7 +1283,6 @@ LIBSUBDIRS= tkinter site-packages \ html json http dbm xmlrpc \ sqlite3 \ logging csv wsgiref urllib \ - lib2to3 lib2to3/fixes lib2to3/pgen2 \ ctypes ctypes/macholib \ idlelib idlelib/Icons \ distutils distutils/command $(XMLLIBSUBDIRS) \ @@ -1351,9 +1352,6 @@ TESTSUBDIRS= tkinter/test tkinter/test/test_tkinter \ test/test_email test/test_email/data \ test/test_json \ sqlite3/test \ - lib2to3/tests \ - lib2to3/tests/data lib2to3/tests/data/fixers \ - lib2to3/tests/data/fixers/myfixes \ ctypes/test \ idlelib/idle_test \ distutils/tests \ @@ -1361,6 +1359,14 @@ TESTSUBDIRS= tkinter/test tkinter/test/test_tkinter \ test/test_tools test/test_warnings test/test_warnings/data \ unittest/test unittest/test/testmock +ifeq (@LIB2TO3@,yes) +LIBSUBDIRS += lib2to3 lib2to3/fixes lib2to3/pgen2 +TESTSUBDIRS += lib2to3/tests \ + lib2to3/tests/data \ + lib2to3/tests/data/fixers \ + lib2to3/tests/data/fixers/myfixes +endif + ifeq (@TEST_MODULES@,yes) LIBSUBDIRS += $(TESTSUBDIRS) endif @@ -1460,10 +1466,12 @@ ifeq (@PYC_BUILD@,yes) -d $(LIBDEST)/site-packages -f \ -x badsyntax $(DESTDIR)$(LIBDEST)/site-packages endif +ifeq (@LIB2TO3@,yes) -PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \ $(PYTHON_FOR_BUILD) -m lib2to3.pgen2.driver $(DESTDIR)$(LIBDEST)/lib2to3/Grammar.txt -PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \ $(PYTHON_FOR_BUILD) -m lib2to3.pgen2.driver $(DESTDIR)$(LIBDEST)/lib2to3/PatternGrammar.txt +endif python-config: $(srcdir)/Misc/python-config.in Misc/python-config.sh @ # Substitution happens here, as the completely-expanded BINDIR diff --git a/configure.ac b/configure.ac index 2cdcb63..7b735bf 100644 --- a/configure.ac +++ b/configure.ac @@ -3241,6 +3241,12 @@ AC_ARG_ENABLE(test-modules, AS_HELP_STRING([--disable-test-modules], [disable test modules]), [ TEST_MODULES="${enableval}" ], [ TEST_MODULES=yes ]) +AC_SUBST(LIB2TO3) + +AC_ARG_ENABLE(lib2to3, + AS_HELP_STRING([--disable-lib2to3], [disable lib2to3]), + [ LIB2TO3="${enableval}" ], [ LIB2TO3=yes ]) + # Check for enable-ipv6 AH_TEMPLATE(ENABLE_IPV6, [Define if --enable-ipv6 is specified]) AC_MSG_CHECKING([if --enable-ipv6 is specified]) diff --git a/setup.py b/setup.py index 11ac959..104ed28 100644 --- a/setup.py +++ b/setup.py @@ -2377,10 +2377,11 @@ def main(): import warnings warnings.filterwarnings("ignore",category=DeprecationWarning) - scripts = ['Tools/scripts/idle3', 'Tools/scripts/2to3', - 'Lib/smtpd.py'] + scripts = ['Tools/scripts/idle3', 'Lib/smtpd.py'] if not '--disable-pydoc' in sysconfig.get_config_var("CONFIG_ARGS"): scripts += [ 'Tools/scripts/pydoc3' ] + if not '--disable-lib2to3' in sysconfig.get_config_var("CONFIG_ARGS"): + scripts += [ 'Tools/scripts/2to3' ] setup(# PyPI Metadata (PEP 301) name = "Python", -- 2.7.4 From 14ff67e98e43b4e0c75c53611edc7208cc7633d7 Mon Sep 17 00:00:00 2001 From: Thomas Petazzoni Date: Wed, 22 Feb 2017 17:20:45 -0800 Subject: [PATCH 16/33] Add option to disable the sqlite3 module Signed-off-by: Thomas Petazzoni Signed-off-by: Samuel Martin [ Andrey Smirnov: ported to Python 3.6 ] Signed-off-by: Andrey Smirnov --- Makefile.pre.in | 7 +++++-- configure.ac | 9 +++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 037362c..40c0255 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1281,7 +1281,6 @@ LIBSUBDIRS= tkinter site-packages \ email email/mime \ ensurepip ensurepip/_bundled \ html json http dbm xmlrpc \ - sqlite3 \ logging csv wsgiref urllib \ ctypes ctypes/macholib \ idlelib idlelib/Icons \ @@ -1351,7 +1350,6 @@ TESTSUBDIRS= tkinter/test tkinter/test/test_tkinter \ test/test_asyncio \ test/test_email test/test_email/data \ test/test_json \ - sqlite3/test \ ctypes/test \ idlelib/idle_test \ distutils/tests \ @@ -1367,6 +1365,11 @@ TESTSUBDIRS += lib2to3/tests \ lib2to3/tests/data/fixers/myfixes endif +ifeq (@SQLITE3@,yes) +LIBSUBDIRS += sqlite3 +TESTSUBDIRS += sqlite3/test +endif + ifeq (@TEST_MODULES@,yes) LIBSUBDIRS += $(TESTSUBDIRS) endif diff --git a/configure.ac b/configure.ac index 7b735bf..efb7488 100644 --- a/configure.ac +++ b/configure.ac @@ -3229,6 +3229,15 @@ if test "$posix_threads" = "yes"; then AC_CHECK_FUNCS(pthread_getcpuclockid) fi +AC_SUBST(SQLITE3) +AC_ARG_ENABLE(sqlite3, + AS_HELP_STRING([--disable-sqlite3], [disable sqlite3]), + [ SQLITE3="${enableval}" ], [ SQLITE3=yes ]) + +if test "$SQLITE3" = "no" ; then + DISABLED_EXTENSIONS="${DISABLED_EXTENSIONS} _sqlite3" +fi + AC_SUBST(PYDOC) AC_ARG_ENABLE(pydoc, -- 2.7.4 From 62c298482f43a07cddf98d66bfef1d17887b88b2 Mon Sep 17 00:00:00 2001 From: Thomas Petazzoni Date: Wed, 22 Feb 2017 17:23:42 -0800 Subject: [PATCH 17/33] Add an option to disable the tk module Signed-off-by: Thomas Petazzoni Signed-off-by: Samuel Martin [ Andrey Smirnov: ported to Python 3.6 ] Signed-off-by: Andrey Smirnov --- Makefile.pre.in | 11 ++++++++--- configure.ac | 9 +++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 40c0255..b0e35f5 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1275,7 +1275,7 @@ maninstall: altmaninstall # Install the library XMLLIBSUBDIRS= xml xml/dom xml/etree xml/parsers xml/sax -LIBSUBDIRS= tkinter site-packages \ +LIBSUBDIRS= site-packages \ asyncio \ collections concurrent concurrent/futures encodings \ email email/mime \ @@ -1292,8 +1292,7 @@ LIBSUBDIRS= tkinter site-packages \ venv venv/scripts venv/scripts/common venv/scripts/posix \ curses -TESTSUBDIRS= tkinter/test tkinter/test/test_tkinter \ - tkinter/test/test_ttk test \ +TESTSUBDIRS= test \ test/audiodata \ test/capath test/data \ test/cjkencodings test/decimaltestdata test/xmltestdata \ @@ -1357,6 +1356,12 @@ TESTSUBDIRS= tkinter/test tkinter/test/test_tkinter \ test/test_tools test/test_warnings test/test_warnings/data \ unittest/test unittest/test/testmock +ifeq (@TK@,yes) +LIBSUBDIRS += tkinter +TESTSUBDIRS += tkinter/test tkinter/test/test_tkinter \ + tkinter/test/test_ttk +endif + ifeq (@LIB2TO3@,yes) LIBSUBDIRS += lib2to3 lib2to3/fixes lib2to3/pgen2 TESTSUBDIRS += lib2to3/tests \ diff --git a/configure.ac b/configure.ac index efb7488..298a08c 100644 --- a/configure.ac +++ b/configure.ac @@ -3238,6 +3238,15 @@ if test "$SQLITE3" = "no" ; then DISABLED_EXTENSIONS="${DISABLED_EXTENSIONS} _sqlite3" fi +AC_SUBST(TK) +AC_ARG_ENABLE(tk, + AS_HELP_STRING([--disable-tk], [disable tk]), + [ TK="${enableval}" ], [ TK=yes ]) + +if test "$TK" = "no"; then + DISABLED_EXTENSIONS="${DISABLED_EXTENSIONS} _tkinter" +fi + AC_SUBST(PYDOC) AC_ARG_ENABLE(pydoc, -- 2.7.4 From 7637783b28e8b4e6b1bb0ee874d967824ed41210 Mon Sep 17 00:00:00 2001 From: Thomas Petazzoni Date: Wed, 22 Feb 2017 17:31:51 -0800 Subject: [PATCH 18/33] Add an option to disable the curses module Signed-off-by: Thomas Petazzoni Signed-off-by: Samuel Martin [ Andrey Smirnov: ported to Python 3.6 ] Signed-off-by: Andrey Smirnov --- Makefile.pre.in | 7 +++++-- configure.ac | 9 +++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index b0e35f5..8f16377 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1289,8 +1289,7 @@ LIBSUBDIRS= site-packages \ turtledemo \ multiprocessing multiprocessing/dummy \ unittest \ - venv venv/scripts venv/scripts/common venv/scripts/posix \ - curses + venv venv/scripts venv/scripts/common venv/scripts/posix TESTSUBDIRS= test \ test/audiodata \ @@ -1362,6 +1361,10 @@ TESTSUBDIRS += tkinter/test tkinter/test/test_tkinter \ tkinter/test/test_ttk endif +ifeq (@CURSES@,yes) +LIBSUBDIRS += curses +endif + ifeq (@LIB2TO3@,yes) LIBSUBDIRS += lib2to3 lib2to3/fixes lib2to3/pgen2 TESTSUBDIRS += lib2to3/tests \ diff --git a/configure.ac b/configure.ac index 298a08c..69dd6ca 100644 --- a/configure.ac +++ b/configure.ac @@ -3247,6 +3247,15 @@ if test "$TK" = "no"; then DISABLED_EXTENSIONS="${DISABLED_EXTENSIONS} _tkinter" fi +AC_SUBST(CURSES) +AC_ARG_ENABLE(curses, + AS_HELP_STRING([--disable-curses], [disable curses]), + [ CURSES="${enableval}" ], [ CURSES=yes ]) + +if test "$CURSES" = "no"; then + DISABLED_EXTENSIONS="${DISABLED_EXTENSIONS} _curses _curses_panel" +fi + AC_SUBST(PYDOC) AC_ARG_ENABLE(pydoc, -- 2.7.4 From 065a94499216924c8558e3ba581368777d2c677c Mon Sep 17 00:00:00 2001 From: Thomas Petazzoni Date: Wed, 22 Feb 2017 17:40:45 -0800 Subject: [PATCH 19/33] Add an option to disable expat This patch replaces the existing --with-system-expat option with a --with-expat={system,builtin,none} option, which allows to tell Python whether we want to use the system expat (already installed), the expat builtin the Python sources, or no expat at all (which disables the installation of XML modules). Signed-off-by: Thomas Petazzoni Signed-off-by: Samuel Martin [ Andrey Smirnov: ported to Python 3.6 ] Signed-off-by: Andrey Smirnov --- Makefile.pre.in | 6 +++++- configure.ac | 18 +++++++++++++----- setup.py | 2 +- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 8f16377..6b262f4 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1284,7 +1284,7 @@ LIBSUBDIRS= site-packages \ logging csv wsgiref urllib \ ctypes ctypes/macholib \ idlelib idlelib/Icons \ - distutils distutils/command $(XMLLIBSUBDIRS) \ + distutils distutils/command \ importlib \ turtledemo \ multiprocessing multiprocessing/dummy \ @@ -1365,6 +1365,10 @@ ifeq (@CURSES@,yes) LIBSUBDIRS += curses endif +ifeq (@EXPAT@,yes) +LIBSUBDIRS += $(XMLLIBSUBDIRS) +endif + ifeq (@LIB2TO3@,yes) LIBSUBDIRS += lib2to3 lib2to3/fixes lib2to3/pgen2 TESTSUBDIRS += lib2to3/tests \ diff --git a/configure.ac b/configure.ac index 69dd6ca..214a666 100644 --- a/configure.ac +++ b/configure.ac @@ -2969,13 +2969,21 @@ PKG_PROG_PKG_CONFIG AC_SUBST(DISABLED_EXTENSIONS) # Check for use of the system expat library -AC_MSG_CHECKING(for --with-system-expat) -AC_ARG_WITH(system_expat, - AS_HELP_STRING([--with-system-expat], [build pyexpat module using an installed expat library]), +AC_MSG_CHECKING(for --with-expat) +AC_ARG_WITH(expat, + AS_HELP_STRING([--with-expat], [select which expat version to use: system, builtin, none]), [], - [with_system_expat="no"]) + [with_expat="builtin"]) -AC_MSG_RESULT($with_system_expat) +AC_MSG_RESULT($with_expat) + +if test "$with_expat" != "none"; then + EXPAT=yes +else + DISABLED_EXTENSIONS="${DISABLED_EXTENSIONS} pyexpat" + EXPAT=no +fi +AC_SUBST(EXPAT) # Check for use of the system libffi library AC_MSG_CHECKING(for --with-system-ffi) diff --git a/setup.py b/setup.py index 104ed28..85d823f 100644 --- a/setup.py +++ b/setup.py @@ -1529,7 +1529,7 @@ class PyBuildExt(build_ext): # # More information on Expat can be found at www.libexpat.org. # - if '--with-system-expat' in sysconfig.get_config_var("CONFIG_ARGS"): + if '--with-expat=system' in sysconfig.get_config_var("CONFIG_ARGS"): expat_inc = [] define_macros = [] extra_compile_args = [] -- 2.7.4 From 5fa5150e9bc718971947bc04a767042abdc74706 Mon Sep 17 00:00:00 2001 From: Thomas Petazzoni Date: Wed, 23 Dec 2015 11:49:55 +0100 Subject: [PATCH 20/33] Add an option to disable CJK codecs Signed-off-by: Thomas Petazzoni --- configure.ac | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/configure.ac b/configure.ac index 214a666..34dc0b8 100644 --- a/configure.ac +++ b/configure.ac @@ -3246,6 +3246,12 @@ if test "$SQLITE3" = "no" ; then DISABLED_EXTENSIONS="${DISABLED_EXTENSIONS} _sqlite3" fi +AC_ARG_ENABLE(codecs-cjk, + AS_HELP_STRING([--disable-codecs-cjk], [disable CJK codecs]), + [ if test "$enableval" = "no"; then + DISABLED_EXTENSIONS="${DISABLED_EXTENSIONS} _codecs_kr _codecs_jp _codecs_cn _codecs_tw _codecs_hk _codecs_iso2022" + fi]) + AC_SUBST(TK) AC_ARG_ENABLE(tk, AS_HELP_STRING([--disable-tk], [disable tk]), -- 2.7.4 From cf2f18c7682004a8351141c460e300eaedc1782a Mon Sep 17 00:00:00 2001 From: Thomas Petazzoni Date: Wed, 23 Dec 2015 11:50:11 +0100 Subject: [PATCH 21/33] Add an option to disable NIS NIS is not necessarily available in uClibc, so we need an option to not compile support for it. Signed-off-by: Thomas Petazzoni --- configure.ac | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/configure.ac b/configure.ac index 34dc0b8..ba1b1fc 100644 --- a/configure.ac +++ b/configure.ac @@ -3252,6 +3252,12 @@ AC_ARG_ENABLE(codecs-cjk, DISABLED_EXTENSIONS="${DISABLED_EXTENSIONS} _codecs_kr _codecs_jp _codecs_cn _codecs_tw _codecs_hk _codecs_iso2022" fi]) +AC_ARG_ENABLE(nis, + AS_HELP_STRING([--disable-nis], [disable NIS]), + [ if test "$enableval" = "no"; then + DISABLED_EXTENSIONS="${DISABLED_EXTENSIONS} nis" + fi]) + AC_SUBST(TK) AC_ARG_ENABLE(tk, AS_HELP_STRING([--disable-tk], [disable tk]), -- 2.7.4 From dcd9e9ff46b624222092e7a0ac5d5a9f80cce9b1 Mon Sep 17 00:00:00 2001 From: Thomas Petazzoni Date: Wed, 23 Dec 2015 11:50:27 +0100 Subject: [PATCH 22/33] Add an option to disable unicodedata Signed-off-by: Thomas Petazzoni --- configure.ac | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/configure.ac b/configure.ac index ba1b1fc..f902d14 100644 --- a/configure.ac +++ b/configure.ac @@ -3258,6 +3258,12 @@ AC_ARG_ENABLE(nis, DISABLED_EXTENSIONS="${DISABLED_EXTENSIONS} nis" fi]) +AC_ARG_ENABLE(unicodedata, + AS_HELP_STRING([--disable-unicodedata], [disable unicodedata]), + [ if test "$enableval" = "no"; then + DISABLED_EXTENSIONS="${DISABLED_EXTENSIONS} unicodedata" + fi]) + AC_SUBST(TK) AC_ARG_ENABLE(tk, AS_HELP_STRING([--disable-tk], [disable tk]), -- 2.7.4 From 5d7312323a17f7d7dd495a1459905dff8f09880d Mon Sep 17 00:00:00 2001 From: Maxime Ripard Date: Wed, 22 Feb 2017 17:45:14 -0800 Subject: [PATCH 23/33] Add an option to disable IDLE IDLE is an IDE embedded into python, written using Tk, so it doesn't make much sense to have it into our build. Signed-off-by: Maxime Ripard [ Andrey Smirnov: ported to Python 3.6 ] Signed-off-by: Andrey Smirnov --- Makefile.pre.in | 7 ++++++- configure.ac | 6 ++++++ setup.py | 4 +++- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 6b262f4..31dc7c3 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1238,7 +1238,9 @@ bininstall: altbininstall -rm -f $(DESTDIR)$(LIBPC)/python3.pc (cd $(DESTDIR)$(LIBPC); $(LN) -s python-$(VERSION).pc python3.pc) -rm -f $(DESTDIR)$(BINDIR)/idle3 +ifeq (@IDLE@,yes) (cd $(DESTDIR)$(BINDIR); $(LN) -s idle$(VERSION) idle3) +endif -rm -f $(DESTDIR)$(BINDIR)/pydoc3 ifeq (@PYDOC@,yes) (cd $(DESTDIR)$(BINDIR); $(LN) -s pydoc$(VERSION) pydoc3) @@ -1283,7 +1285,6 @@ LIBSUBDIRS= site-packages \ html json http dbm xmlrpc \ logging csv wsgiref urllib \ ctypes ctypes/macholib \ - idlelib idlelib/Icons \ distutils distutils/command \ importlib \ turtledemo \ @@ -1369,6 +1370,10 @@ ifeq (@EXPAT@,yes) LIBSUBDIRS += $(XMLLIBSUBDIRS) endif +ifeq (@IDLE@,yes) +LIBSUBDIRS += idlelib idlelib/Icons +endif + ifeq (@LIB2TO3@,yes) LIBSUBDIRS += lib2to3 lib2to3/fixes lib2to3/pgen2 TESTSUBDIRS += lib2to3/tests \ diff --git a/configure.ac b/configure.ac index f902d14..799ebd6 100644 --- a/configure.ac +++ b/configure.ac @@ -3300,6 +3300,12 @@ AC_ARG_ENABLE(lib2to3, AS_HELP_STRING([--disable-lib2to3], [disable lib2to3]), [ LIB2TO3="${enableval}" ], [ LIB2TO3=yes ]) +AC_SUBST(IDLE) + +AC_ARG_ENABLE(idle3, + AS_HELP_STRING([--disable-idle3], [disable idle3 IDE]), + [ IDLE="${enableval}" ], [ IDLE=yes ]) + # Check for enable-ipv6 AH_TEMPLATE(ENABLE_IPV6, [Define if --enable-ipv6 is specified]) AC_MSG_CHECKING([if --enable-ipv6 is specified]) diff --git a/setup.py b/setup.py index 85d823f..f377381 100644 --- a/setup.py +++ b/setup.py @@ -2377,11 +2377,13 @@ def main(): import warnings warnings.filterwarnings("ignore",category=DeprecationWarning) - scripts = ['Tools/scripts/idle3', 'Lib/smtpd.py'] + scripts = [ 'Lib/smtpd.py'] if not '--disable-pydoc' in sysconfig.get_config_var("CONFIG_ARGS"): scripts += [ 'Tools/scripts/pydoc3' ] if not '--disable-lib2to3' in sysconfig.get_config_var("CONFIG_ARGS"): scripts += [ 'Tools/scripts/2to3' ] + if not '--disable-idle3' in sysconfig.get_config_var("CONFIG_ARGS"): + scripts += [ 'Tools/scripts/idle3' ] setup(# PyPI Metadata (PEP 301) name = "Python", -- 2.7.4 From 9395a0bd66319ef5745902bcd4e8dda0bd2b78bb Mon Sep 17 00:00:00 2001 From: Thomas Petazzoni Date: Wed, 23 Dec 2015 11:51:31 +0100 Subject: [PATCH 24/33] Add an option to disable decimal This patch replaces the existing --with-system-libmpdec option with a --with-libmpdec={system,builtin,none} option, which allows to tell Python whether we want to use the system libmpdec (already installed), the libmpdec builtin the Python sources, or no libmpdec at all. Signed-off-by: Thomas Petazzoni [aduskett@gmail.com: Update for python 3.7.0] Signed-off-by: Adam Duskett --- configure.ac | 17 ++++++++++++----- setup.py | 2 +- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/configure.ac b/configure.ac index 799ebd6..6f674d8 100644 --- a/configure.ac +++ b/configure.ac @@ -3020,13 +3020,20 @@ fi AC_SUBST(LIBFFI_INCLUDEDIR) # Check for use of the system libmpdec library -AC_MSG_CHECKING(for --with-system-libmpdec) -AC_ARG_WITH(system_libmpdec, - AS_HELP_STRING([--with-system-libmpdec], [build _decimal module using an installed libmpdec library]), +AC_MSG_CHECKING(for --with-libmpdec) +AC_ARG_WITH(libmpdec, + AS_HELP_STRING([--with-libmpdec], [select which libmpdec version to use: system, builtin, none]), [], - [with_system_libmpdec="no"]) + [with_libmpdec="builtin"]) -AC_MSG_RESULT($with_system_libmpdec) +AC_MSG_RESULT($with_libmpdec) +if test "$with_libmpdec" != "none"; then + MPDEC=yes +else + DISABLED_EXTENSIONS="${DISABLED_EXTENSIONS} _decimal" + MPDEC=no +fi +AC_SUBST(MPDEC) # Check for support for loadable sqlite extensions AC_MSG_CHECKING(for --enable-loadable-sqlite-extensions) diff --git a/setup.py b/setup.py index f377381..aadfb75 100644 --- a/setup.py +++ b/setup.py @@ -2054,7 +2054,7 @@ class PyBuildExt(build_ext): def _decimal_ext(self): extra_compile_args = [] undef_macros = [] - if '--with-system-libmpdec' in sysconfig.get_config_var("CONFIG_ARGS"): + if '--with-libmpdec=system' in sysconfig.get_config_var("CONFIG_ARGS"): include_dirs = [] libraries = [':libmpdec.so.2'] sources = ['_decimal/_decimal.c'] -- 2.7.4 From b34d8abb5aa757839df0b6caefd4100a0d1710b3 Mon Sep 17 00:00:00 2001 From: Thomas Petazzoni Date: Wed, 23 Dec 2015 11:51:58 +0100 Subject: [PATCH 25/33] Add an option to disable the ossaudiodev module Signed-off-by: Thomas Petazzoni --- configure.ac | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/configure.ac b/configure.ac index 6f674d8..7d438ed 100644 --- a/configure.ac +++ b/configure.ac @@ -3035,6 +3035,12 @@ else fi AC_SUBST(MPDEC) +AC_ARG_ENABLE(ossaudiodev, + AS_HELP_STRING([--disable-ossaudiodev], [disable OSSAUDIODEV]), + [ if test "$enableval" = "no"; then + DISABLED_EXTENSIONS="${DISABLED_EXTENSIONS} ossaudiodev" + fi]) + # Check for support for loadable sqlite extensions AC_MSG_CHECKING(for --enable-loadable-sqlite-extensions) AC_ARG_ENABLE(loadable-sqlite-extensions, -- 2.7.4 From 3b5c8b96475c047ae59bea38285f53f6fa38a3ae Mon Sep 17 00:00:00 2001 From: Nicolas Cavallari Date: Wed, 22 Feb 2017 17:55:59 -0800 Subject: [PATCH 26/33] Add an option to disable openssl support. Signed-off-by: Nicolas Cavallari --- configure.ac | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/configure.ac b/configure.ac index 7d438ed..32657f5 100644 --- a/configure.ac +++ b/configure.ac @@ -3277,6 +3277,12 @@ AC_ARG_ENABLE(unicodedata, DISABLED_EXTENSIONS="${DISABLED_EXTENSIONS} unicodedata" fi]) +AC_ARG_ENABLE(openssl, + AS_HELP_STRING([--disable-openssl], [disable openssl support]), + [ if test "$enableval" = "no"; then + DISABLED_EXTENSIONS="${DISABLED_EXTENSIONS} ssl _ssl _hashlib" + fi]) + AC_SUBST(TK) AC_ARG_ENABLE(tk, AS_HELP_STRING([--disable-tk], [disable tk]), -- 2.7.4 From d97fb86e2794a44076462a6e07829bc4df162284 Mon Sep 17 00:00:00 2001 From: Thomas Petazzoni Date: Tue, 7 Mar 2017 23:29:05 +0100 Subject: [PATCH 27/33] Add an option to disable the readline module Signed-off-by: Thomas Petazzoni --- configure.ac | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/configure.ac b/configure.ac index 32657f5..d568f25 100644 --- a/configure.ac +++ b/configure.ac @@ -3283,6 +3283,12 @@ AC_ARG_ENABLE(openssl, DISABLED_EXTENSIONS="${DISABLED_EXTENSIONS} ssl _ssl _hashlib" fi]) +AC_ARG_ENABLE(readline, + AS_HELP_STRING([--disable-readline], [disable readline]), + [ if test "$enableval" = "no"; then + DISABLED_EXTENSIONS="${DISABLED_EXTENSIONS} readline" + fi]) + AC_SUBST(TK) AC_ARG_ENABLE(tk, AS_HELP_STRING([--disable-tk], [disable tk]), -- 2.7.4 From 0d2cf7a3c9a43debfef8d9eb4dae68df145ff41c Mon Sep 17 00:00:00 2001 From: Thomas Petazzoni Date: Tue, 7 Mar 2017 23:31:11 +0100 Subject: [PATCH 28/33] Add options to disable zlib, bzip2 and xz modules Signed-off-by: Thomas Petazzoni --- configure.ac | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/configure.ac b/configure.ac index d568f25..6d8c6ec 100644 --- a/configure.ac +++ b/configure.ac @@ -3289,6 +3289,24 @@ AC_ARG_ENABLE(readline, DISABLED_EXTENSIONS="${DISABLED_EXTENSIONS} readline" fi]) +AC_ARG_ENABLE(bzip2, + AS_HELP_STRING([--disable-bzip2], [disable bzip2]), + [ if test "$enableval" = "no"; then + DISABLED_EXTENSIONS="${DISABLED_EXTENSIONS} _bz2" + fi]) + +AC_ARG_ENABLE(zlib, + AS_HELP_STRING([--disable-zlib], [disable zlib]), + [ if test "$enableval" = "no"; then + DISABLED_EXTENSIONS="${DISABLED_EXTENSIONS} zlib" + fi]) + +AC_ARG_ENABLE(xz, + AS_HELP_STRING([--disable-xz], [disable xz]), + [ if test "$enableval" = "no"; then + DISABLED_EXTENSIONS="${DISABLED_EXTENSIONS} _lzma" + fi]) + AC_SUBST(TK) AC_ARG_ENABLE(tk, AS_HELP_STRING([--disable-tk], [disable tk]), -- 2.7.4 From 06fe0264238cc01fc8e89634b3c3849864bc9bf1 Mon Sep 17 00:00:00 2001 From: Matt Weber Date: Fri, 6 Oct 2017 09:54:15 -0500 Subject: [PATCH 29/33] python-config.sh: don't reassign ${prefix} When prefix is set to a path like /usr during crossbuild the sed operations end up executing twice, once for the prefix reassignment and another for includedir if it is set as a string including the ${prefix} variable. This results in an issue when the build directory is under /usr. This patch updates the remaining location which uses the prefix variable to also sed and update to use the real path. Upstream bug report: https://bugs.python.org/issue31713 Buildroot bug: https://bugs.busybox.net/show_bug.cgi?id=10361 Fixes failures like the following: dbus-python-1.2.4 | NOK | http://autobuild.buildroot.net/results/758858efa97b6273c1b470513f5492258a6d8853 Signed-off-by: Matthew Weber --- Misc/python-config.sh.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Misc/python-config.sh.in b/Misc/python-config.sh.in index db91bd6..c12640b 100644 --- a/Misc/python-config.sh.in +++ b/Misc/python-config.sh.in @@ -31,7 +31,7 @@ prefix_real=$(installed_prefix "$0") # locations. Keep prefix & exec_prefix using their original values in case # they are referenced in other configure variables, to prevent double # substitution, issue #22140. -prefix=$(echo "$prefix_build" | sed "s#^$prefix_build#$prefix_real#") +prefix=$prefix_build exec_prefix=$(echo "$exec_prefix_build" | sed "s#^$exec_prefix_build#$prefix_real#") exec_prefix_real=${prefix_real} includedir=$(echo "@includedir@" | sed "s#^$prefix_build#$prefix_real#") @@ -48,7 +48,7 @@ LDLIBRARY="@LDLIBRARY@" OPT="@OPT@" PY_ENABLE_SHARED="@PY_ENABLE_SHARED@" LDVERSION="@LDVERSION@" -LIBDEST=${prefix_real}/lib/python${VERSION} +LIBDEST=$( echo "${prefix}/lib/python${VERSION}" | sed "s#^$prefix_build#$prefix_real#") LIBPL=$(echo "@LIBPL@" | sed "s#^$prefix_build#$prefix_real#") SO="@EXT_SUFFIX@" PYTHONFRAMEWORK="@PYTHONFRAMEWORK@" -- 2.7.4 From 15f1dd515d45edbaff26b77c8bb144cd5ee8ea7d Mon Sep 17 00:00:00 2001 From: Adam Duskett Date: Fri, 20 Jul 2018 10:17:39 -0400 Subject: [PATCH 30/33] Fix cross compiling the uuid module Python 3.7 has a new _uuid module, however, the include directory search path for uuid.h is hardcoded to /usr/include/uuid, which should not be used when cross-compiling. To fix this, use the same solution as the one used by the NIS detection: append "uuid" to each of the include directories in "inc_dirs", instead of hardcoding /usr/include/uuid. Signed-off-by: Adam Duskett [Thomas: drop STAGING_DIR based solution, use a solution similar to the one used for the NIS detection.] Signed-off-by: Thomas Petazzoni --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index aadfb75..94a327b 100644 --- a/setup.py +++ b/setup.py @@ -1671,7 +1671,8 @@ class PyBuildExt(build_ext): missing.append('_tkinter') # Build the _uuid module if possible - uuid_incs = find_file("uuid.h", inc_dirs, ["/usr/include/uuid"]) + uuid_incs = find_file("uuid.h", inc_dirs, + [os.path.join(inc_dir, 'uuid') for inc_dir in inc_dirs]) if uuid_incs is not None: if self.compiler.find_library_file(lib_dirs, 'uuid'): uuid_libs = ['uuid'] -- 2.7.4 From 3b7a6b54a04564ba05f8f628a0114e45d1c1ea77 Mon Sep 17 00:00:00 2001 From: Thomas Petazzoni Date: Sat, 18 Aug 2018 10:54:56 +0200 Subject: [PATCH 31/33] Add an option to disable uuid module Signed-off-by: Thomas Petazzoni --- configure.ac | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/configure.ac b/configure.ac index 6d8c6ec..91c55da 100644 --- a/configure.ac +++ b/configure.ac @@ -3325,6 +3325,15 @@ if test "$CURSES" = "no"; then DISABLED_EXTENSIONS="${DISABLED_EXTENSIONS} _curses _curses_panel" fi +AC_SUBST(UUID) +AC_ARG_ENABLE(uuid, + AS_HELP_STRING([--disable-uuid], [disable uuid]), + [ UUID="${enableval}" ], [ UUID=yes ]) + +if test "$UUID" = "no"; then + DISABLED_EXTENSIONS="${DISABLED_EXTENSIONS} _uuid" +fi + AC_SUBST(PYDOC) AC_ARG_ENABLE(pydoc, -- 2.7.4 From b2bf4f1cf6141ff19ef3a6e61d054dabea83d3c9 Mon Sep 17 00:00:00 2001 From: Adam Duskett Date: Thu, 16 Aug 2018 14:52:37 -0700 Subject: [PATCH 32/33] fix building on older distributions Python > 3.6.3 calls os.replace in the update_file.py script, during the regen-importlib phase of the build process. According to Doc/whatsnew/3.3.rst line 1631, os.replace acts in the same way as os.rename, however, it is now cross-platform compatible for Windows. Because BuildRoot is guaranteed only to be built in POSIX environment, it is safe to change os.replace back to os.rename. This change fixes building on older systems such as CentOS7, that only come with python 2. Signed-off-by: Adam Duskett --- Tools/scripts/update_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/scripts/update_file.py b/Tools/scripts/update_file.py index 224585c..ef458c0 100644 --- a/Tools/scripts/update_file.py +++ b/Tools/scripts/update_file.py @@ -16,7 +16,7 @@ def main(old_path, new_path): with open(new_path, 'rb') as f: new_contents = f.read() if old_contents != new_contents: - os.replace(new_path, old_path) + os.rename(new_path, old_path) else: os.unlink(new_path) -- 2.7.4 From b532835bd1f1d169e2339f0b1324558c153cac99 Mon Sep 17 00:00:00 2001 From: Peter Korsgaard Date: Fri, 2 Aug 2019 15:53:16 +0200 Subject: [PATCH 33/33] configure.ac: fixup $CC --print-multiarch output for musl/uclibc GCC 8+ toolchains GCC commit 6834b83784dcf0364eb820e8 (multiarch support for non-glibc linux systems), which is part of GCC 8+, changed the multiarch logic to use $arch-linux-musl / $arch-linux-uclibc rather than $arch-linux-gnu. This then causes the python3 configure script to error out: checking for the platform triplet based on compiler characteristics... powerpc-linux-gnu configure: error: internal configure error for the platform triplet, please file a bug report http://autobuild.buildroot.net/results/cb4/cb49c539501342e45cbe5ade82e588fcdf51f05b As it requires that the --print-multiarch output (if not empty) matches the deduced triplet (which always uses -linux-gnu). It isn't quite clear why --print-multiarch returns something for a non-multiarch toolchain on some architectures (E.G. PowerPC), but as a workaround, rewrite the --print-multiarch output to match older GCC versions to keep the configure script happy. Signed-off-by: Peter Korsgaard --- configure.ac | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 91c55da..371e199 100644 --- a/configure.ac +++ b/configure.ac @@ -724,7 +724,9 @@ then fi -MULTIARCH=$($CC --print-multiarch 2>/dev/null) +# GCC 8+ returns $arch-linux-{musl,uclibc} for musl/uClibc based +# toolchains confusing python. Fix that up +MULTIARCH=$($CC --print-multiarch 2>/dev/null | sed -E 's/-linux-(musl|uclibc)*$/-linux-gnu/') AC_SUBST(MULTIARCH) AC_MSG_CHECKING([for the platform triplet based on compiler characteristics]) -- 2.7.4