refactored loops to call build_loop()

fixed bypassing tasks logging with [?]
fixed sip using PyQt5.zip instead of .sip, doesn't change anything though, the module name in that header was unused
python: don't copy openssl dlls, the openssl project already does it
python: don't upgrade the solution, not needed
python: don't need a bypass file for the zip file, just check if the zip exists

src/tasks/ncc.cpp

@@ -27,12 +27,16 @@ fs::path ncc::source_path()
 
 void ncc::do_clean(clean c)
 {
+	// delete the whole directory
 	if (is_set(c, clean::reclone))
 	{
 		git_wrap::delete_directory(cx(), source_path());
+
+		// no need to do anything else
 		return;
 	}
 
+	// msbuild clean
 	if (is_set(c, clean::rebuild))
 		run_tool(create_msbuild_tool(msbuild::clean));
 }
@@ -47,11 +51,13 @@ void ncc::do_fetch()
 
 void ncc::do_build_and_install()
 {
+	// build
 	run_tool(msbuild()
 		.solution(source_path() / "NexusClient.sln")
 		.targets({"NexusClientCLI"})
 		.platform("Any CPU"));
 
+	// run publish.bat, which copies a bunch of stuff to install/bin/NCC
 	const auto publish = source_path() / "publish.bat";
 
 	run_tool(process_runner(process()

src/tasks/nmm.cpp

@@ -26,50 +26,45 @@ fs::path nmm::source_path()
 
 void nmm::do_clean(clean c)
 {
+	// delete the whole directory
 	if (is_set(c, clean::reclone))
 	{
 		git_wrap::delete_directory(cx(), source_path());
+
+		// no need to do anything else
 		return;
 	}
 
+	// msbuild clean
 	if (is_set(c, clean::rebuild))
 		run_tool(create_msbuild_tool(msbuild::clean));
 }
 
 void nmm::do_fetch()
 {
+	// clone/pull
 	run_tool(make_git()
 		.url(make_git_url("Nexus-Mods", "Nexus-Mod-Manager"))
 		.branch(version())
 		.root(source_path()));
 
+	// run nuget
 	run_tool(nuget(source_path() / "NexusClient.sln"));
 }
 
 void nmm::do_build_and_install()
 {
-	// nmm sometimes fails with files being locked
-	const int max_tries = 3;
-
-	for (int tries=0; tries<max_tries; ++tries)
+	build_loop(cx(), [&](bool mp)
 	{
+		// msbuild defaults to multiprocess, give allow_failure for multiprocess
+		// builds and force single_job for the last single process build
+
 		const int exit_code = run_tool(create_msbuild_tool(
-			msbuild::build, msbuild::allow_failure));
+			msbuild::build,
+			mp ? msbuild::allow_failure : msbuild::single_job));
 
-		if (exit_code == 0)
-			return;
-
-		cx().debug(context::generic,
-			"msbuild multiprocess sometimes fails with nmm because of race "
-			"conditions; trying again");
-	}
-
-	cx().debug(context::generic,
-		"msbuild multiprocess has failed more than {} times for nmm, "
-		"restarting one last time single process; that one should work",
-		max_tries);
-
-	run_tool(create_msbuild_tool(msbuild::build, msbuild::single_job));
+		return (exit_code == 0);
+	});
 }
 
 msbuild nmm::create_msbuild_tool(msbuild::ops o, msbuild::flags_t f)

src/tasks/openssl.cpp

@@ -5,6 +5,48 @@
 namespace mob::tasks
 {
 
+namespace
+{
+
+url source_url()
+{
+	return
+		"https://www.openssl.org/source/"
+		"openssl-" + openssl::version() + ".tar.gz";
+}
+
+url prebuilt_url()
+{
+	return make_prebuilt_url("openssl-prebuilt-" + openssl::version() + ".7z");
+}
+
+std::string version_no_patch_underscores()
+{
+	auto v = openssl::parsed_version();
+
+	std::string s = v.major;
+
+	if (v.minor != "")
+		s += "_" + v.minor;
+
+	return s;
+}
+
+// the filenames of the dlls, but without extension, because this is used for
+// both dlls and pdbs
+//
+std::vector<std::string> output_names()
+{
+	return
+	{
+		"libcrypto-" + version_no_patch_underscores() + "-x64",
+		"libssl-" + version_no_patch_underscores() + "-x64"
+	};
+}
+
+}	// namespace
+
+
 openssl::openssl()
 	: basic_task("openssl")
 {
@@ -39,20 +81,23 @@ void openssl::do_clean(clean c)
 {
 	if (prebuilt())
 	{
+		// delete prebuilt download
 		if (is_set(c, clean::redownload))
 			run_tool(downloader(prebuilt_url(), downloader::clean));
 	}
 	else
 	{
+		// delete source download
 		if (is_set(c, clean::redownload))
 			run_tool(downloader(source_url(), downloader::clean));
 	}
 
+	// there's no easy way to clean anything for openssl, it puts files all
+	// over the place, just delete the whole thing
 	if (is_any_set(c, clean::reextract|clean::reconfigure|clean::rebuild))
 	{
 		cx().trace(context::reextract, "deleting {}", source_path());
 		op::delete_directory(cx(), source_path(), op::optional);
-		return;
 	}
 }
 
@@ -94,18 +139,26 @@ void openssl::fetch_from_source()
 
 void openssl::build_and_install_prebuilt()
 {
+	// nothing to built for prebuilt, just copy
 	copy_files();
 }
 
 void openssl::build_and_install_from_source()
 {
+	// running the Configure perl script generates a file `makefile`; since
+	// Configuring takes forever and will fully run every time, don't run it if
+	// the makefile already exists
 	if (fs::exists(source_path() / "makefile"))
 		cx().trace(context::bypass, "openssl already configured");
 	else
 		configure();
 
+	// run the install_engines target in the makefile, this builds everything
+	// required
 	install_engines();
 
+	// applink.c is required when building python from source, the .vcxproj
+	// assumes it's in the include path for whatever reason, so copy it there
 	op::copy_file_to_dir_if_better(cx(),
 		source_path() / "ms" / "applink.c",
 		include_path());
@@ -130,38 +183,28 @@ void openssl::configure()
 
 void openssl::install_engines()
 {
-	const int max_tries = 3;
-
-	for (int tries=0; tries<max_tries; ++tries)
+	build_loop(cx(), [&](bool mp)
 	{
+		// jom defaults to multiprocess, give allow_failure for multiprocess
+		// builds and force single_job for the last single process build
+
 		const int exit_code = run_tool(jom()
 			.path(source_path())
 			.target("install_engines")
-			.flag(jom::allow_failure));
+			.flag(mp ? jom::allow_failure : jom::single_job));
 
-		if (exit_code == 0)
-			return;
-
-		cx().debug(context::generic,
-			"jom /J regularly fails with openssh because of race conditions; "
-			"trying again");
-	}
-
-	cx().debug(context::generic,
-		"jom /J has failed more than {} times, "
-		"restarting one last time without /J; that one should work",
-		max_tries);
-
-	run_tool(jom()
-		.path(source_path())
-		.target("install_engines")
-		.flag(jom::single_job));
+		return (exit_code == 0);
+	});
 }
 
 void openssl::copy_files()
 {
+	// duplicate the dlls to both bin/ and bin/dlls, they're needed by both
+	// MO and Qt
 	copy_dlls_to(conf().path().install_bin());
 	copy_dlls_to(conf().path().install_dlls());
+
+	// pdbs
 	copy_pdbs_to(conf().path().install_pdbs());
 }
 
@@ -188,27 +231,6 @@ fs::path openssl::include_path()
 	return openssl::source_path() / "include";
 }
 
-url openssl::source_url()
-{
-	return
-		"https://www.openssl.org/source/"
-		"openssl-" + version() + ".tar.gz";
-}
-
-url openssl::prebuilt_url()
-{
-	return make_prebuilt_url("openssl-prebuilt-" + version() + ".7z");
-}
-
-std::vector<std::string> openssl::output_names()
-{
-	return
-	{
-		"libcrypto-" + version_no_patch_underscores() + "-x64",
-		"libssl-" + version_no_patch_underscores() + "-x64"
-	};
-}
-
 openssl::version_info openssl::parsed_version()
 {
 	// 1.2.3d
@@ -232,16 +254,4 @@ openssl::version_info openssl::parsed_version()
 	return {m[1], m[2], m[3]};
 }
 
-std::string openssl::version_no_patch_underscores()
-{
-	auto v = parsed_version();
-
-	std::string s = v.major;
-
-	if (v.minor != "")
-		s += "_" + v.minor;
-
-	return s;
-}
-
 }	// namespace

src/tasks/pyqt.cpp

@@ -2,9 +2,78 @@
 #include "tasks.h"
 #include "../core/process.h"
 
+// build process for python, sip and pyqt; if one is built from source, all
+// three need to be built from source, plus openssl because python needs it
+//
+//  1) build openssl
+//
+//  2) build python, needs openssl
+//
+//  3) build sip, needs python:
+//      - download and extract source archive
+//      - run `python setup.py install` in sip's directory, this generates
+//        sip-install.exe and sip-module.exe in python-XX/Scripts (among others)
+//      - run `sip-module.exe --sip-h` to generate a sip.h file in sip's source
+//        directory
+//      - that header file is copied into python/include and is included by
+//        by plugin_python in sipapiaccess.h
+//
+// 4) build pyqt, needs sip
+//      - download and extract source archive
+//      - use `pip install` to install PyQt-builder
+//      - run `sip-install.exe` with the list of required modules, creating a
+//        folder for each module in PyQt5-XX/build/ with `.pyd` files
+//      - run `sip-module.exe --sdist`, which creates
+//        downloads/PyQt5_sip-XXX.tar.gz
+//      - run `pip install` with that file, which creates
+//        `python-XX/Lib/site-packages/PyQt5/sip.cp32-win_amd64.pyd`
+//      - for installation, a bunch of files from site-packages/PyQt5/ are
+//        copied into bin/plugins/data/PyQt5, including a .pyi file from sip
+
+
 namespace mob::tasks
 {
 
+namespace
+{
+
+url source_url()
+{
+	return
+		"https://pypi.io/packages/source/P/PyQt5/"
+		"PyQt5-" + pyqt::version() + ".tar.gz";
+}
+
+url prebuilt_url()
+{
+	return make_prebuilt_url("PyQt5_gpl-prebuilt-" + pyqt::version() + ".7z");
+}
+
+// file created by sip-module.exe
+//
+fs::path sip_install_file()
+{
+	return "PyQt5_sip-" + sip::version_for_pyqt() + ".tar.gz";
+}
+
+std::vector<std::string> modules()
+{
+	return
+	{
+		"QtCore",
+		"QtGui",
+		"QtWidgets",
+		"QtOpenGL",
+		"QtSvg",
+		"_QOpenGLFunctions_2_0",
+		"_QOpenGLFunctions_2_1",
+		"_QOpenGLFunctions_4_1_Core"
+	};
+}
+
+}	// namespace
+
+
 pyqt::pyqt()
 	: basic_task("pyqt")
 {
@@ -35,28 +104,39 @@ fs::path pyqt::build_path()
 	return source_path() / "build";
 }
 
+std::string pyqt::pyqt_sip_module_name()
+{
+	return "PyQt5.sip";
+}
+
 void pyqt::do_clean(clean c)
 {
 	if (prebuilt())
 	{
+		// delete prebuilt download
 		if (is_set(c, clean::redownload))
 			run_tool(downloader(prebuilt_url(), downloader::clean));
 	}
 	else
 	{
+		// delete source download
 		if (is_set(c, clean::redownload))
 			run_tool(downloader(source_url(), downloader::clean));
 	}
 
+	// delete whole directory
 	if (is_set(c, clean::reextract))
 	{
 		cx().trace(context::reextract, "deleting {}", source_path());
 		op::delete_directory(cx(), source_path(), op::optional);
+
+		// no need to do anything else
 		return;
 	}
 
 	if (!prebuilt())
 	{
+		// delete the pyqt-sip file that's created when building from source
 		if (is_set(c, clean::rebuild))
 		{
 			op::delete_file(cx(),
@@ -93,11 +173,14 @@ void pyqt::fetch_prebuilt()
 
 void pyqt::build_and_install_prebuilt()
 {
+	// copy the prebuilt files directly into the python directory, they're
+	// required by sip, which is always built from source
 	op::copy_glob_to_dir_if_better(cx(),
 		source_path() / "*",
 		python::source_path(),
 		op::copy_files|op::copy_dirs);
 
+	// copy files to build/install for MO
 	copy_files();
 }
 
@@ -112,23 +195,31 @@ void pyqt::fetch_from_source()
 
 void pyqt::build_and_install_from_source()
 {
+	// use pip to install the pyqt builder
 	run_tool(pip(pip::install)
 		.package("PyQt-builder")
 		.version(builder_version()));
 
+	// patch for builder.py
 	run_tool(patcher()
 		.task(name())
 		.file("builder.py.manual_patch")
 		.root(python::site_packages_path() / "pyqtbuild"));
 
+	// build modules and generate the PyQt5_sip-XX.tar.gz file
 	sip_build();
 
+	// run pip install for the PyQt5_sip-XX.tar.gz file
 	install_sip_file();
+
+	// copy files to build/install for MO
 	copy_files();
 }
 
 void pyqt::sip_build()
 {
+	// put qt and python in the path, set CL and LIB, which are used by the
+	// visual c++ compiler that's eventually spawned, and set PYTHONHOME
 	auto pyqt_env = env::vs_x64()
 		.append_path({
 			qt::bin_path(),
@@ -139,7 +230,8 @@ void pyqt::sip_build()
 		.set("LIB", ";" + path_to_utf8(conf().path().install_libs()), env::append)
 		.set("PYTHONHOME", path_to_utf8(python::source_path()));
 
-
+	// create a bypass file, because pyqt always tries to build stuff and it
+	// takes forever
 	bypass_file built_bypass(cx(), source_path(), "built");
 
 	if (built_bypass.exists())
@@ -153,6 +245,7 @@ void pyqt::sip_build()
 		// here instead
 		op::delete_directory(cx(), source_path() / "build", op::optional);
 
+		// build modules
 		run_tool(process_runner(process()
 			.binary(sip::sip_install_exe())
 			.arg("--confirm-license")
@@ -166,19 +259,23 @@ void pyqt::sip_build()
 			.cwd(source_path())
 			.env(pyqt_env)));
 
+		// done, create the bypass file
 		built_bypass.create();
 	}
 
+	// generate the PyQt5_sip-XX.tar.gz file
 	run_tool(process_runner(process()
 		.binary(sip::sip_module_exe())
 		.arg("--sdist")
-		.arg("PyQt5.sip")
+		.arg(pyqt_sip_module_name())
 		.cwd(conf().path().cache())
 		.env(pyqt_env)));
 }
 
 void pyqt::install_sip_file()
 {
+	// create a bypass file, because pyqt always tries to install stuff and it
+	// takes forever
 	bypass_file installed_bypass(cx(), source_path(), "installed");
 
 	if (installed_bypass.exists())
@@ -187,17 +284,27 @@ void pyqt::install_sip_file()
 	}
 	else
 	{
+		// run `pip install` on the generated PyQt5_sip-XX.tar.gz file
 		run_tool(pip(pip::install)
 			.file(conf().path().cache() / sip_install_file()));
 
+		// done, create the bypass file
 		installed_bypass.create();
 	}
 }
 
 void pyqt::copy_files()
 {
-	const fs::path site_packages_pyqt = python::site_packages_path() / "PyQt5";
-	const fs::path pyqt_plugin = conf().path().install_plugins() / "data" / "PyQt5";
+	// pyqt puts its files in python-XX/Lib/site-packages/PyQt5
+	const fs::path site_packages_pyqt =
+		python::site_packages_path() / "PyQt5";
+
+	// target directory is bin/plugins/data/PyQt5
+	const fs::path pyqt_plugin =
+		conf().path().install_plugins() / "data" / "PyQt5";
+
+
+	// copying a bunch of files from site-packages into the plugins directory
 
 	op::copy_file_to_dir_if_better(cx(),
 		site_packages_pyqt / "__init__.py",
@@ -220,11 +327,15 @@ void pyqt::copy_files()
 			op::optional);
 	}
 
+	// also copy this from sip's module/source/XX/ directory
 	op::copy_file_to_dir_if_better(cx(),
 		sip::module_source_path() / "sip.pyi",
 		pyqt_plugin);
 
-	// these are needed by PyQt5 while building several projects
+
+	// copying some dlls from Qt's installation directory into
+	// python-XX/PCBuild/amd64, those are needed by PyQt5 when building several
+	// projects
 
 	op::copy_file_to_dir_if_better(cx(),
 		qt::bin_path() / "Qt5Core.dll",
@@ -237,36 +348,4 @@ void pyqt::copy_files()
 		op::unsafe);   // source file is outside prefix
 }
 
-url pyqt::source_url()
-{
-	return
-		"https://pypi.io/packages/source/P/PyQt5/"
-		"PyQt5-" + version() + ".tar.gz";
-}
-
-url pyqt::prebuilt_url()
-{
-	return make_prebuilt_url("PyQt5_gpl-prebuilt-" + version() + ".7z");
-}
-
-fs::path pyqt::sip_install_file()
-{
-	return "PyQt5_sip-" + sip::version_for_pyqt() + ".tar.gz";
-}
-
-std::vector<std::string> pyqt::modules()
-{
-	return
-	{
-		"QtCore",
-		"QtGui",
-		"QtWidgets",
-		"QtOpenGL",
-		"QtSvg",
-		"_QOpenGLFunctions_2_0",
-		"_QOpenGLFunctions_2_1",
-		"_QOpenGLFunctions_4_1_Core"
-	};
-}
-
 }	// namespace

src/tasks/python.cpp

@@ -2,9 +2,55 @@
 #include "tasks.h"
 #include "../core/process.h"
 
+// see the top of pyqt.cpp for some stuff about python/sip/pyqt
+
 namespace mob::tasks
 {
 
+namespace
+{
+
+std::string version_without_v()
+{
+	const auto v = python::parsed_version();
+
+	// 3.8.1
+	std::string s = v.major + "." + v.minor;
+	if (v.patch != "")
+		s += "." + v.patch;
+
+	return s;
+}
+
+std::string version_for_dll()
+{
+	const auto v = python::parsed_version();
+
+	// 38
+	return v.major + v.minor;
+}
+
+url prebuilt_url()
+{
+	return make_prebuilt_url("python-prebuilt-" + version_without_v() + ".7z");
+}
+
+fs::path solution_file()
+{
+	return python::source_path() / "PCBuild" / "pcbuild.sln";
+}
+
+fs::path python_core_zip_file()
+{
+	return
+		python::build_path() /
+		"pythoncore" /
+		("python" + version_for_dll() + ".zip");
+}
+
+}	// namespace
+
+
 python::python()
 	: basic_task("python")
 {
@@ -41,18 +87,6 @@ python::version_info python::parsed_version()
 	return v;
 }
 
-std::string python::version_without_v()
-{
-	const auto v = parsed_version();
-
-	// 3.8.1
-	std::string s = v.major + "." + v.minor;
-	if (v.patch != "")
-		s += "." + v.patch;
-
-	return s;
-}
-
 fs::path python::source_path()
 {
 	return conf().path().build() / ("python-" + version_without_v());
@@ -67,24 +101,29 @@ void python::do_clean(clean c)
 {
 	if (prebuilt())
 	{
+		// delete download
 		if (is_set(c, clean::redownload))
 			run_tool(downloader(prebuilt_url(), downloader::clean));
 
+		// delete the whole directory
 		if (is_set(c, clean::reextract))
 		{
 			cx().trace(context::reextract, "deleting {}", source_path());
 			op::delete_directory(cx(), source_path(), op::optional);
-			return;
 		}
 	}
 	else
 	{
+		// delete the whole directory
 		if (is_set(c, clean::reclone))
 		{
 			git_wrap::delete_directory(cx(), source_path());
+
+			// no need to do anything else
 			return;
 		}
 
+		// msbuild clean
 		if (is_set(c, clean::rebuild))
 			run_tool(create_msbuild_tool(msbuild::clean));
 	}
@@ -117,11 +156,6 @@ void python::fetch_prebuilt()
 
 void python::build_and_install_prebuilt()
 {
-	op::copy_glob_to_dir_if_better(cx(),
-		openssl::bin_path() / "*.dll",
-		python::build_path(),
-		op::copy_files);
-
 	install_pip();
 	copy_files();
 }
@@ -132,18 +166,20 @@ void python::fetch_from_source()
 		.url(make_git_url("python", "cpython"))
 		.branch(version())
 		.root(source_path()));
-
-	run_tool(vs(vs::upgrade)
-		.solution(solution_file()));
 }
 
 void python::build_and_install_from_source()
 {
+	// build
 	run_tool(create_msbuild_tool());
+
+	// package stuff into pythoncore.zip
 	package();
 
+	// install pip for other tasks that need it
 	install_pip();
 
+	// boost.python expects pyconfig.h to be in the include path
 	op::copy_file_to_dir_if_better(cx(),
 		source_path() / "PC" / "pyconfig.h",
 		include_path());
@@ -153,37 +189,35 @@ void python::build_and_install_from_source()
 
 void python::package()
 {
-	bypass_file packaged_bypass(cx(), build_path(), "packaged");
-
-	if (packaged_bypass.exists())
+	if (fs::exists(python_core_zip_file()))
 	{
 		cx().trace(context::bypass, "python already packaged");
+		return;
 	}
-	else
-	{
-		const auto bat = source_path() / "python.bat";
 
-		run_tool(process_runner(process()
-			.binary(bat)
-			.arg(fs::path("PC/layout"))
-			.arg("--source", source_path())
-			.arg("--build", build_path())
-			.arg("--temp", (build_path() / "pythoncore_temp"))
-			.arg("--copy", (build_path() / "pythoncore"))
-			.arg("--preset-embed")
-			.cwd(source_path())));
+	const auto bat = source_path() / "python.bat";
 
-		op::touch(cx(), build_path() / "_mob_packaged");
-	}
+	// package libs into pythonXX.zip
+	run_tool(process_runner(process()
+		.binary(bat)
+		.arg(fs::path("PC/layout"))
+		.arg("--source", source_path())
+		.arg("--build", build_path())
+		.arg("--temp", (build_path() / "pythoncore_temp"))
+		.arg("--copy", (build_path() / "pythoncore"))
+		.arg("--preset-embed")
+		.cwd(source_path())));
 }
 
 void python::copy_files()
 {
+	// libs
 	op::copy_glob_to_dir_if_better(cx(),
 		build_path() / "*.lib",
 		conf().path().install_libs(),
 		op::copy_files);
 
+	// dlls
 	op::copy_glob_to_dir_if_better(cx(),
 		build_path() / "libffi*.dll",
 		conf().path().install_bin(),
@@ -193,17 +227,20 @@ void python::copy_files()
 		build_path() / ("python" + version_for_dll() + ".dll"),
 		conf().path().install_bin());
 
+	// pdbs
 	op::copy_file_to_dir_if_better(cx(),
 		build_path() / ("python" + version_for_dll() + ".pdb"),
 		conf().path().install_pdbs());
 
+	// pyd files
 	op::copy_glob_to_dir_if_better(cx(),
 		build_path() / "pythoncore/*.pyd",
 		conf().path().install_pythoncore(),
 		op::copy_files);
 
+	// pythonXX.zip -> pythoncore.zip
 	op::copy_file_to_file_if_better(cx(),
-		build_path() / "pythoncore" / ("python" + version_for_dll() + ".zip"),
+		python_core_zip_file(),
 		conf().path().install_bin() / "pythoncore.zip",
 		op::copy_files);
 }
@@ -250,22 +287,4 @@ fs::path python::site_packages_path()
 	return source_path() / "Lib" / "site-packages";
 }
 
-url python::prebuilt_url()
-{
-	return make_prebuilt_url("python-prebuilt-" + version_without_v() + ".7z");
-}
-
-fs::path python::solution_file()
-{
-	return source_path() / "PCBuild" / "pcbuild.sln";
-}
-
-std::string python::version_for_dll()
-{
-	const auto v = parsed_version();
-
-	// 38
-	return v.major + v.minor;
-}
-
 }	// namespace

src/tasks/sip.cpp

@@ -149,13 +149,15 @@ void sip::generate()
 		op::replace_file(cx(), src, dest, backup);
 	}
 
+	// generate sip.h, will be copied to python's include directory, used
+	// by plugin_python
 	run_tool(process_runner(process()
 		.binary(sip_module_exe())
 		.chcp(850)
 		.stdout_encoding(encodings::acp)
 		.stderr_encoding(encodings::acp)
 		.arg("--sip-h")
-		.arg("PyQt5.zip")
+		.arg(pyqt::pyqt_sip_module_name())
 		.cwd(source_path())));
 }
 

src/tasks/task.cpp

@@ -330,17 +330,17 @@ bool task::get_prebuilt() const
 
 void task::run()
 {
-	if (!enabled())
-	{
-		cx().debug(context::generic, "task is disabled");
-		return;
-	}
-
 	// make sure there's a context for this thread; run() can be called from
 	// the main thread or from parallel_tasks, for example, so it might be in
 	// a new thread or not
 	running_from_thread(name(), [&]
 	{
+		if (!enabled())
+		{
+			cx().debug(context::generic, "task is disabled");
+			return;
+		}
+
 		cx().info(context::generic, "running task");
 
 		// clean task if needed

src/tasks/tasks.h

@@ -402,6 +402,7 @@ public:
 
 	static fs::path source_path();
 	static fs::path include_path();
+	static fs::path build_path();
 	static fs::path bin_path();
 
 protected:
@@ -420,15 +421,12 @@ private:
 	void copy_files();
 	void copy_dlls_to(const fs::path& dir);
 	void copy_pdbs_to(const fs::path& dir);
-
-	static url source_url();
-	static url prebuilt_url();
-	static fs::path build_path();
-	static std::vector<std::string> output_names();
-	static std::string version_no_patch_underscores();
 };
 
 
+// when building from source, builds both pyqt5 and pyqt5-sip; when using the
+// prebuilt, the downloaded zip contains both
+//
 class pyqt : public basic_task<pyqt>
 {
 public:
@@ -439,6 +437,11 @@ public:
 	static bool prebuilt();
 
 	static fs::path source_path();
+	static fs::path build_path();
+
+	// "PyQt5.sip", used both in pyqt and sip
+	//
+	static std::string pyqt_sip_module_name();
 
 protected:
 	void do_clean(clean c) override;
@@ -454,12 +457,6 @@ private:
 	void sip_build();
 	void install_sip_file();
 	void copy_files();
-
-	static url source_url();
-	static url prebuilt_url();
-	static fs::path sip_install_file();
-	static fs::path build_path();
-	static std::vector<std::string> modules();
 };
 
 
@@ -477,13 +474,30 @@ public:
 
 	static std::string version();
 	static bool prebuilt();
-
 	static version_info parsed_version();
+
+	// build/python-XX
+	//
 	static fs::path source_path();
+
+	// build/python-XX/PCBuild/amd64
+	//
 	static fs::path build_path();
+
+	// build/python-XX/PCBuild/amd64/python.exe
+	//
 	static fs::path python_exe();
+
+	// build/python-XX/Include
+	//
 	static fs::path include_path();
+
+	// build/python-XX/Scripts
+	//
 	static fs::path scripts_path();
+
+	// build/python-XX/Lib/site-packages
+	//
 	static fs::path site_packages_path();
 
 protected:
@@ -502,11 +516,6 @@ private:
 	void copy_files();
 
 	msbuild create_msbuild_tool(msbuild::ops o=msbuild::build);
-
-	static std::string version_without_v();
-	static url prebuilt_url();
-	static fs::path solution_file();
-	static std::string version_for_dll();
 };
 
 

src/tools/tools.cpp

@@ -459,4 +459,33 @@ void iscc::do_run()
 		.arg(iss_));
 }
 
+
+void build_loop(const context& cx, std::function<bool (bool)> f)
+{
+	// building sometimes fails with files being locked
+	const int max_tries = 3;
+
+	for (int tries=0; tries<max_tries; ++tries)
+	{
+		// try a multiprocess build
+		if (f(true))
+		{
+			// building succeeded, done
+			return;
+		}
+
+		cx.debug(context::generic,
+			"multiprocess build sometimes fails because of race "
+			"conditions; trying again");
+	}
+
+	cx.debug(context::generic,
+		"multiprocess build has failed more than {} times, "
+		"restarting one last time single process; that one should work",
+		max_tries);
+
+	// do one last single process build
+	f(false);
+}
+
 }	// namespace

src/tools/tools.h

@@ -917,6 +917,23 @@ private:
 	fs::path iss_;
 };
 
+
+// some tasks use a multiprocess version of their build tool to make things
+// faster, but the build process might fail because the projects are not really
+// set up correctly for multiprocess (locked files, etc.)
+//
+// that's the case for nmm, openssl and couple of others
+//
+// so `f` will first be called in a loop a couple of times with its parameter as
+// `true`; it should run the build multiprocess and return whether the build
+// succeeded, which will end the loop and return immediately
+//
+// if `f` returned false every time in the loop, it will be called one last time
+// with its parameter as `false`, which should build one final time with any
+// multiprocess flags
+//
+void build_loop(const context& cx, std::function<bool (bool)> f);
+
 }	// namespace