diff --git a/src/tasks/ncc.cpp b/src/tasks/ncc.cpp index cb831ad..b97a571 100644 --- a/src/tasks/ncc.cpp +++ b/src/tasks/ncc.cpp @@ -27,12 +27,16 @@ fs::path ncc::source_path() void ncc::do_clean(clean c) { + // delete the whole directory if (is_set(c, clean::reclone)) { git_wrap::delete_directory(cx(), source_path()); + + // no need to do anything else return; } + // msbuild clean if (is_set(c, clean::rebuild)) run_tool(create_msbuild_tool(msbuild::clean)); } @@ -47,11 +51,13 @@ void ncc::do_fetch() void ncc::do_build_and_install() { + // build run_tool(msbuild() .solution(source_path() / "NexusClient.sln") .targets({"NexusClientCLI"}) .platform("Any CPU")); + // run publish.bat, which copies a bunch of stuff to install/bin/NCC const auto publish = source_path() / "publish.bat"; run_tool(process_runner(process() diff --git a/src/tasks/nmm.cpp b/src/tasks/nmm.cpp index 102f275..709c4bf 100644 --- a/src/tasks/nmm.cpp +++ b/src/tasks/nmm.cpp @@ -26,50 +26,45 @@ fs::path nmm::source_path() void nmm::do_clean(clean c) { + // delete the whole directory if (is_set(c, clean::reclone)) { git_wrap::delete_directory(cx(), source_path()); + + // no need to do anything else return; } + // msbuild clean if (is_set(c, clean::rebuild)) run_tool(create_msbuild_tool(msbuild::clean)); } void nmm::do_fetch() { + // clone/pull run_tool(make_git() .url(make_git_url("Nexus-Mods", "Nexus-Mod-Manager")) .branch(version()) .root(source_path())); + // run nuget run_tool(nuget(source_path() / "NexusClient.sln")); } void nmm::do_build_and_install() { - // nmm sometimes fails with files being locked - const int max_tries = 3; - - for (int tries=0; tries output_names() +{ + return + { + "libcrypto-" + version_no_patch_underscores() + "-x64", + "libssl-" + version_no_patch_underscores() + "-x64" + }; +} + +} // namespace + + openssl::openssl() : basic_task("openssl") { @@ -39,20 +81,23 @@ void openssl::do_clean(clean c) { if (prebuilt()) { + // delete prebuilt download if (is_set(c, clean::redownload)) run_tool(downloader(prebuilt_url(), downloader::clean)); } else { + // delete source download if (is_set(c, clean::redownload)) run_tool(downloader(source_url(), downloader::clean)); } + // there's no easy way to clean anything for openssl, it puts files all + // over the place, just delete the whole thing if (is_any_set(c, clean::reextract|clean::reconfigure|clean::rebuild)) { cx().trace(context::reextract, "deleting {}", source_path()); op::delete_directory(cx(), source_path(), op::optional); - return; } } @@ -94,18 +139,26 @@ void openssl::fetch_from_source() void openssl::build_and_install_prebuilt() { + // nothing to built for prebuilt, just copy copy_files(); } void openssl::build_and_install_from_source() { + // running the Configure perl script generates a file `makefile`; since + // Configuring takes forever and will fully run every time, don't run it if + // the makefile already exists if (fs::exists(source_path() / "makefile")) cx().trace(context::bypass, "openssl already configured"); else configure(); + // run the install_engines target in the makefile, this builds everything + // required install_engines(); + // applink.c is required when building python from source, the .vcxproj + // assumes it's in the include path for whatever reason, so copy it there op::copy_file_to_dir_if_better(cx(), source_path() / "ms" / "applink.c", include_path()); @@ -130,38 +183,28 @@ void openssl::configure() void openssl::install_engines() { - const int max_tries = 3; - - for (int tries=0; tries openssl::output_names() -{ - return - { - "libcrypto-" + version_no_patch_underscores() + "-x64", - "libssl-" + version_no_patch_underscores() + "-x64" - }; -} - openssl::version_info openssl::parsed_version() { // 1.2.3d @@ -232,16 +254,4 @@ openssl::version_info openssl::parsed_version() return {m[1], m[2], m[3]}; } -std::string openssl::version_no_patch_underscores() -{ - auto v = parsed_version(); - - std::string s = v.major; - - if (v.minor != "") - s += "_" + v.minor; - - return s; -} - } // namespace diff --git a/src/tasks/pyqt.cpp b/src/tasks/pyqt.cpp index 12fc30c..11fb379 100644 --- a/src/tasks/pyqt.cpp +++ b/src/tasks/pyqt.cpp @@ -2,9 +2,78 @@ #include "tasks.h" #include "../core/process.h" +// build process for python, sip and pyqt; if one is built from source, all +// three need to be built from source, plus openssl because python needs it +// +// 1) build openssl +// +// 2) build python, needs openssl +// +// 3) build sip, needs python: +// - download and extract source archive +// - run `python setup.py install` in sip's directory, this generates +// sip-install.exe and sip-module.exe in python-XX/Scripts (among others) +// - run `sip-module.exe --sip-h` to generate a sip.h file in sip's source +// directory +// - that header file is copied into python/include and is included by +// by plugin_python in sipapiaccess.h +// +// 4) build pyqt, needs sip +// - download and extract source archive +// - use `pip install` to install PyQt-builder +// - run `sip-install.exe` with the list of required modules, creating a +// folder for each module in PyQt5-XX/build/ with `.pyd` files +// - run `sip-module.exe --sdist`, which creates +// downloads/PyQt5_sip-XXX.tar.gz +// - run `pip install` with that file, which creates +// `python-XX/Lib/site-packages/PyQt5/sip.cp32-win_amd64.pyd` +// - for installation, a bunch of files from site-packages/PyQt5/ are +// copied into bin/plugins/data/PyQt5, including a .pyi file from sip + + namespace mob::tasks { +namespace +{ + +url source_url() +{ + return + "https://pypi.io/packages/source/P/PyQt5/" + "PyQt5-" + pyqt::version() + ".tar.gz"; +} + +url prebuilt_url() +{ + return make_prebuilt_url("PyQt5_gpl-prebuilt-" + pyqt::version() + ".7z"); +} + +// file created by sip-module.exe +// +fs::path sip_install_file() +{ + return "PyQt5_sip-" + sip::version_for_pyqt() + ".tar.gz"; +} + +std::vector modules() +{ + return + { + "QtCore", + "QtGui", + "QtWidgets", + "QtOpenGL", + "QtSvg", + "_QOpenGLFunctions_2_0", + "_QOpenGLFunctions_2_1", + "_QOpenGLFunctions_4_1_Core" + }; +} + +} // namespace + + pyqt::pyqt() : basic_task("pyqt") { @@ -35,28 +104,39 @@ fs::path pyqt::build_path() return source_path() / "build"; } +std::string pyqt::pyqt_sip_module_name() +{ + return "PyQt5.sip"; +} + void pyqt::do_clean(clean c) { if (prebuilt()) { + // delete prebuilt download if (is_set(c, clean::redownload)) run_tool(downloader(prebuilt_url(), downloader::clean)); } else { + // delete source download if (is_set(c, clean::redownload)) run_tool(downloader(source_url(), downloader::clean)); } + // delete whole directory if (is_set(c, clean::reextract)) { cx().trace(context::reextract, "deleting {}", source_path()); op::delete_directory(cx(), source_path(), op::optional); + + // no need to do anything else return; } if (!prebuilt()) { + // delete the pyqt-sip file that's created when building from source if (is_set(c, clean::rebuild)) { op::delete_file(cx(), @@ -93,11 +173,14 @@ void pyqt::fetch_prebuilt() void pyqt::build_and_install_prebuilt() { + // copy the prebuilt files directly into the python directory, they're + // required by sip, which is always built from source op::copy_glob_to_dir_if_better(cx(), source_path() / "*", python::source_path(), op::copy_files|op::copy_dirs); + // copy files to build/install for MO copy_files(); } @@ -112,23 +195,31 @@ void pyqt::fetch_from_source() void pyqt::build_and_install_from_source() { + // use pip to install the pyqt builder run_tool(pip(pip::install) .package("PyQt-builder") .version(builder_version())); + // patch for builder.py run_tool(patcher() .task(name()) .file("builder.py.manual_patch") .root(python::site_packages_path() / "pyqtbuild")); + // build modules and generate the PyQt5_sip-XX.tar.gz file sip_build(); + // run pip install for the PyQt5_sip-XX.tar.gz file install_sip_file(); + + // copy files to build/install for MO copy_files(); } void pyqt::sip_build() { + // put qt and python in the path, set CL and LIB, which are used by the + // visual c++ compiler that's eventually spawned, and set PYTHONHOME auto pyqt_env = env::vs_x64() .append_path({ qt::bin_path(), @@ -139,7 +230,8 @@ void pyqt::sip_build() .set("LIB", ";" + path_to_utf8(conf().path().install_libs()), env::append) .set("PYTHONHOME", path_to_utf8(python::source_path())); - + // create a bypass file, because pyqt always tries to build stuff and it + // takes forever bypass_file built_bypass(cx(), source_path(), "built"); if (built_bypass.exists()) @@ -153,6 +245,7 @@ void pyqt::sip_build() // here instead op::delete_directory(cx(), source_path() / "build", op::optional); + // build modules run_tool(process_runner(process() .binary(sip::sip_install_exe()) .arg("--confirm-license") @@ -166,19 +259,23 @@ void pyqt::sip_build() .cwd(source_path()) .env(pyqt_env))); + // done, create the bypass file built_bypass.create(); } + // generate the PyQt5_sip-XX.tar.gz file run_tool(process_runner(process() .binary(sip::sip_module_exe()) .arg("--sdist") - .arg("PyQt5.sip") + .arg(pyqt_sip_module_name()) .cwd(conf().path().cache()) .env(pyqt_env))); } void pyqt::install_sip_file() { + // create a bypass file, because pyqt always tries to install stuff and it + // takes forever bypass_file installed_bypass(cx(), source_path(), "installed"); if (installed_bypass.exists()) @@ -187,17 +284,27 @@ void pyqt::install_sip_file() } else { + // run `pip install` on the generated PyQt5_sip-XX.tar.gz file run_tool(pip(pip::install) .file(conf().path().cache() / sip_install_file())); + // done, create the bypass file installed_bypass.create(); } } void pyqt::copy_files() { - const fs::path site_packages_pyqt = python::site_packages_path() / "PyQt5"; - const fs::path pyqt_plugin = conf().path().install_plugins() / "data" / "PyQt5"; + // pyqt puts its files in python-XX/Lib/site-packages/PyQt5 + const fs::path site_packages_pyqt = + python::site_packages_path() / "PyQt5"; + + // target directory is bin/plugins/data/PyQt5 + const fs::path pyqt_plugin = + conf().path().install_plugins() / "data" / "PyQt5"; + + + // copying a bunch of files from site-packages into the plugins directory op::copy_file_to_dir_if_better(cx(), site_packages_pyqt / "__init__.py", @@ -220,11 +327,15 @@ void pyqt::copy_files() op::optional); } + // also copy this from sip's module/source/XX/ directory op::copy_file_to_dir_if_better(cx(), sip::module_source_path() / "sip.pyi", pyqt_plugin); - // these are needed by PyQt5 while building several projects + + // copying some dlls from Qt's installation directory into + // python-XX/PCBuild/amd64, those are needed by PyQt5 when building several + // projects op::copy_file_to_dir_if_better(cx(), qt::bin_path() / "Qt5Core.dll", @@ -237,36 +348,4 @@ void pyqt::copy_files() op::unsafe); // source file is outside prefix } -url pyqt::source_url() -{ - return - "https://pypi.io/packages/source/P/PyQt5/" - "PyQt5-" + version() + ".tar.gz"; -} - -url pyqt::prebuilt_url() -{ - return make_prebuilt_url("PyQt5_gpl-prebuilt-" + version() + ".7z"); -} - -fs::path pyqt::sip_install_file() -{ - return "PyQt5_sip-" + sip::version_for_pyqt() + ".tar.gz"; -} - -std::vector pyqt::modules() -{ - return - { - "QtCore", - "QtGui", - "QtWidgets", - "QtOpenGL", - "QtSvg", - "_QOpenGLFunctions_2_0", - "_QOpenGLFunctions_2_1", - "_QOpenGLFunctions_4_1_Core" - }; -} - } // namespace diff --git a/src/tasks/python.cpp b/src/tasks/python.cpp index 14d76cb..3834cca 100644 --- a/src/tasks/python.cpp +++ b/src/tasks/python.cpp @@ -2,9 +2,55 @@ #include "tasks.h" #include "../core/process.h" +// see the top of pyqt.cpp for some stuff about python/sip/pyqt + namespace mob::tasks { +namespace +{ + +std::string version_without_v() +{ + const auto v = python::parsed_version(); + + // 3.8.1 + std::string s = v.major + "." + v.minor; + if (v.patch != "") + s += "." + v.patch; + + return s; +} + +std::string version_for_dll() +{ + const auto v = python::parsed_version(); + + // 38 + return v.major + v.minor; +} + +url prebuilt_url() +{ + return make_prebuilt_url("python-prebuilt-" + version_without_v() + ".7z"); +} + +fs::path solution_file() +{ + return python::source_path() / "PCBuild" / "pcbuild.sln"; +} + +fs::path python_core_zip_file() +{ + return + python::build_path() / + "pythoncore" / + ("python" + version_for_dll() + ".zip"); +} + +} // namespace + + python::python() : basic_task("python") { @@ -41,18 +87,6 @@ python::version_info python::parsed_version() return v; } -std::string python::version_without_v() -{ - const auto v = parsed_version(); - - // 3.8.1 - std::string s = v.major + "." + v.minor; - if (v.patch != "") - s += "." + v.patch; - - return s; -} - fs::path python::source_path() { return conf().path().build() / ("python-" + version_without_v()); @@ -67,24 +101,29 @@ void python::do_clean(clean c) { if (prebuilt()) { + // delete download if (is_set(c, clean::redownload)) run_tool(downloader(prebuilt_url(), downloader::clean)); + // delete the whole directory if (is_set(c, clean::reextract)) { cx().trace(context::reextract, "deleting {}", source_path()); op::delete_directory(cx(), source_path(), op::optional); - return; } } else { + // delete the whole directory if (is_set(c, clean::reclone)) { git_wrap::delete_directory(cx(), source_path()); + + // no need to do anything else return; } + // msbuild clean if (is_set(c, clean::rebuild)) run_tool(create_msbuild_tool(msbuild::clean)); } @@ -117,11 +156,6 @@ void python::fetch_prebuilt() void python::build_and_install_prebuilt() { - op::copy_glob_to_dir_if_better(cx(), - openssl::bin_path() / "*.dll", - python::build_path(), - op::copy_files); - install_pip(); copy_files(); } @@ -132,18 +166,20 @@ void python::fetch_from_source() .url(make_git_url("python", "cpython")) .branch(version()) .root(source_path())); - - run_tool(vs(vs::upgrade) - .solution(solution_file())); } void python::build_and_install_from_source() { + // build run_tool(create_msbuild_tool()); + + // package stuff into pythoncore.zip package(); + // install pip for other tasks that need it install_pip(); + // boost.python expects pyconfig.h to be in the include path op::copy_file_to_dir_if_better(cx(), source_path() / "PC" / "pyconfig.h", include_path()); @@ -153,37 +189,35 @@ void python::build_and_install_from_source() void python::package() { - bypass_file packaged_bypass(cx(), build_path(), "packaged"); - - if (packaged_bypass.exists()) + if (fs::exists(python_core_zip_file())) { cx().trace(context::bypass, "python already packaged"); + return; } - else - { - const auto bat = source_path() / "python.bat"; - run_tool(process_runner(process() - .binary(bat) - .arg(fs::path("PC/layout")) - .arg("--source", source_path()) - .arg("--build", build_path()) - .arg("--temp", (build_path() / "pythoncore_temp")) - .arg("--copy", (build_path() / "pythoncore")) - .arg("--preset-embed") - .cwd(source_path()))); + const auto bat = source_path() / "python.bat"; - op::touch(cx(), build_path() / "_mob_packaged"); - } + // package libs into pythonXX.zip + run_tool(process_runner(process() + .binary(bat) + .arg(fs::path("PC/layout")) + .arg("--source", source_path()) + .arg("--build", build_path()) + .arg("--temp", (build_path() / "pythoncore_temp")) + .arg("--copy", (build_path() / "pythoncore")) + .arg("--preset-embed") + .cwd(source_path()))); } void python::copy_files() { + // libs op::copy_glob_to_dir_if_better(cx(), build_path() / "*.lib", conf().path().install_libs(), op::copy_files); + // dlls op::copy_glob_to_dir_if_better(cx(), build_path() / "libffi*.dll", conf().path().install_bin(), @@ -193,17 +227,20 @@ void python::copy_files() build_path() / ("python" + version_for_dll() + ".dll"), conf().path().install_bin()); + // pdbs op::copy_file_to_dir_if_better(cx(), build_path() / ("python" + version_for_dll() + ".pdb"), conf().path().install_pdbs()); + // pyd files op::copy_glob_to_dir_if_better(cx(), build_path() / "pythoncore/*.pyd", conf().path().install_pythoncore(), op::copy_files); + // pythonXX.zip -> pythoncore.zip op::copy_file_to_file_if_better(cx(), - build_path() / "pythoncore" / ("python" + version_for_dll() + ".zip"), + python_core_zip_file(), conf().path().install_bin() / "pythoncore.zip", op::copy_files); } @@ -250,22 +287,4 @@ fs::path python::site_packages_path() return source_path() / "Lib" / "site-packages"; } -url python::prebuilt_url() -{ - return make_prebuilt_url("python-prebuilt-" + version_without_v() + ".7z"); -} - -fs::path python::solution_file() -{ - return source_path() / "PCBuild" / "pcbuild.sln"; -} - -std::string python::version_for_dll() -{ - const auto v = parsed_version(); - - // 38 - return v.major + v.minor; -} - } // namespace diff --git a/src/tasks/sip.cpp b/src/tasks/sip.cpp index e4d539b..519b8ee 100644 --- a/src/tasks/sip.cpp +++ b/src/tasks/sip.cpp @@ -149,13 +149,15 @@ void sip::generate() op::replace_file(cx(), src, dest, backup); } + // generate sip.h, will be copied to python's include directory, used + // by plugin_python run_tool(process_runner(process() .binary(sip_module_exe()) .chcp(850) .stdout_encoding(encodings::acp) .stderr_encoding(encodings::acp) .arg("--sip-h") - .arg("PyQt5.zip") + .arg(pyqt::pyqt_sip_module_name()) .cwd(source_path()))); } diff --git a/src/tasks/task.cpp b/src/tasks/task.cpp index fb203b4..af0b7fa 100644 --- a/src/tasks/task.cpp +++ b/src/tasks/task.cpp @@ -330,17 +330,17 @@ bool task::get_prebuilt() const void task::run() { - if (!enabled()) - { - cx().debug(context::generic, "task is disabled"); - return; - } - // make sure there's a context for this thread; run() can be called from // the main thread or from parallel_tasks, for example, so it might be in // a new thread or not running_from_thread(name(), [&] { + if (!enabled()) + { + cx().debug(context::generic, "task is disabled"); + return; + } + cx().info(context::generic, "running task"); // clean task if needed diff --git a/src/tasks/tasks.h b/src/tasks/tasks.h index 88d242a..eef1c27 100644 --- a/src/tasks/tasks.h +++ b/src/tasks/tasks.h @@ -402,6 +402,7 @@ public: static fs::path source_path(); static fs::path include_path(); + static fs::path build_path(); static fs::path bin_path(); protected: @@ -420,15 +421,12 @@ private: void copy_files(); void copy_dlls_to(const fs::path& dir); void copy_pdbs_to(const fs::path& dir); - - static url source_url(); - static url prebuilt_url(); - static fs::path build_path(); - static std::vector output_names(); - static std::string version_no_patch_underscores(); }; +// when building from source, builds both pyqt5 and pyqt5-sip; when using the +// prebuilt, the downloaded zip contains both +// class pyqt : public basic_task { public: @@ -439,6 +437,11 @@ public: static bool prebuilt(); static fs::path source_path(); + static fs::path build_path(); + + // "PyQt5.sip", used both in pyqt and sip + // + static std::string pyqt_sip_module_name(); protected: void do_clean(clean c) override; @@ -454,12 +457,6 @@ private: void sip_build(); void install_sip_file(); void copy_files(); - - static url source_url(); - static url prebuilt_url(); - static fs::path sip_install_file(); - static fs::path build_path(); - static std::vector modules(); }; @@ -477,13 +474,30 @@ public: static std::string version(); static bool prebuilt(); - static version_info parsed_version(); + + // build/python-XX + // static fs::path source_path(); + + // build/python-XX/PCBuild/amd64 + // static fs::path build_path(); + + // build/python-XX/PCBuild/amd64/python.exe + // static fs::path python_exe(); + + // build/python-XX/Include + // static fs::path include_path(); + + // build/python-XX/Scripts + // static fs::path scripts_path(); + + // build/python-XX/Lib/site-packages + // static fs::path site_packages_path(); protected: @@ -502,11 +516,6 @@ private: void copy_files(); msbuild create_msbuild_tool(msbuild::ops o=msbuild::build); - - static std::string version_without_v(); - static url prebuilt_url(); - static fs::path solution_file(); - static std::string version_for_dll(); }; diff --git a/src/tools/tools.cpp b/src/tools/tools.cpp index 078fcd7..8e12f9d 100644 --- a/src/tools/tools.cpp +++ b/src/tools/tools.cpp @@ -459,4 +459,33 @@ void iscc::do_run() .arg(iss_)); } + +void build_loop(const context& cx, std::function f) +{ + // building sometimes fails with files being locked + const int max_tries = 3; + + for (int tries=0; tries f); + } // namespace