diff --git a/patches/7z/7zip.mak.patch b/patches/7z/7zip.mak.patch new file mode 100644 index 0000000..b6bfa58 --- /dev/null +++ b/patches/7z/7zip.mak.patch @@ -0,0 +1,23 @@ +--- CPP/7zip/7zip.mak Mon Jun 23 11:56:48 2014 ++++ CPP/7zip/7zip.mak Mon Apr 27 15:38:13 2020 +@@ -1,5 +1,4 @@ +-OBJS = \ +- $O\StdAfx.obj \ ++OBJS_NEEDING_PCH = \ + $(CURRENT_OBJS) \ + $(COMMON_OBJS) \ + $(WIN_OBJS) \ +@@ -26,9 +25,11 @@ + $(ZIP_OBJS) \ + $(COMPRESS_OBJS) \ + $(CRYPTO_OBJS) \ +- $(C_OBJS) \ ++ $(C_OBJS) ++ ++OBJS = $(OBJS_NEEDING_PCH) \ + $(ASM_OBJS) \ +- $O\resource.res \ ++ $O\resource.res + + !include "../../../Build.mak" + diff --git a/patches/7z/Build.mak.patch b/patches/7z/Build.mak.patch new file mode 100644 index 0000000..20cdf1b --- /dev/null +++ b/patches/7z/Build.mak.patch @@ -0,0 +1,20 @@ +--- CPP/Build.mak Thu Feb 21 07:50:31 2019 ++++ CPP/Build.mak Mon Apr 27 15:38:30 2020 +@@ -137,6 +137,8 @@ + + all: $(PROGPATH) + ++$(OBJS_NEEDING_PCH): $O\StdAfx.obj ++ + clean: + -del /Q $(PROGPATH) $O\*.exe $O\*.dll $O\*.obj $O\*.lib $O\*.exp $O\*.res $O\*.pch $O\*.asm + +@@ -146,7 +148,7 @@ + if not exist "$O/asm" mkdir "$O/asm" + + $(PROGPATH): $O $O/asm $(OBJS) $(DEF_FILE) +- link $(LFLAGS) -out:$(PROGPATH) $(OBJS) $(LIBS) ++ link $(LFLAGS) -out:$(PROGPATH) $O\StdAfx.obj $(OBJS) $(LIBS) + + !IFNDEF NO_DEFAULT_RES + $O\resource.res: $(*B).rc diff --git a/src/conf.cpp b/src/conf.cpp index 5f7e4ca..e4c07bc 100644 --- a/src/conf.cpp +++ b/src/conf.cpp @@ -34,6 +34,54 @@ const std::string& get_conf(const std::string& name) } +fs::path find_root() +{ + debug("looking for root directory"); + + fs::path p = fs::absolute("third-party"); + debug("checking " + p.string()); + + if (!fs::exists(p)) + { + p = fs::absolute("../third-party"); + debug("checking " + p.string()); + + if (!fs::exists(p)) + { + p = fs::absolute("../../third-party"); + debug("checking " + p.string()); + + if (!fs::exists(p)) + { + p = fs::absolute("../../../third-party"); + debug("checking " + p.string()); + + if (!fs::exists(p)) + bail_out("root directory not found"); + } + } + } + + p = p.parent_path(); + + debug("found root directory at " + p.string()); + return p; +} + +fs::path find_in_root(const fs::path& file) +{ + static fs::path root = find_root(); + + fs::path p = root / file; + if (!fs::exists(p)) + bail_out(p.string() + " not found"); + + debug("found " + p.string()); + return p; +} + + + const std::string& versions::vs() { return get_conf("vs"); } const std::string& versions::vs_year() { return get_conf("vs_year"); } const std::string& versions::sevenzip() { return get_conf("sevenzip"); } @@ -50,6 +98,12 @@ fs::path paths::install() { return prefix() / get_conf("install"); } fs::path paths::install_bin( ) { return install() / get_conf("bin"); } fs::path paths::install_dlls() { return install_bin() / get_conf("dlls"); } +fs::path paths::patches() +{ + static fs::path p = find_in_root("patches"); + return p; +} + fs::path paths::program_files_x86() { @@ -117,9 +171,29 @@ fs::path paths::temp_file() } + +fs::path third_party::sevenz() +{ + static fs::path path = find_in_root("third-party/bin/7z.exe"); + return path; +} + +fs::path third_party::jom() +{ + static fs::path path = find_in_root("third-party/bin/jom.exe"); + return path; +} + +fs::path third_party::patch() +{ + static fs::path path = find_in_root("third-party/bin/patch.exe"); + return path; +} + + bool conf::verbose() { - return true; + return false; } bool conf::dry() @@ -127,31 +201,4 @@ bool conf::dry() return false; } - -fs::path find_sevenz() -{ - static fs::path path = [] - { - const fs::path sevenz = "7z.exe"; - const fs::path third_party = "third-party/bin"; - - fs::path final; - - if (fs::path p=third_party / sevenz; fs::exists(p)) - final = fs::absolute(p); - else if(p = ".." / third_party / sevenz; fs::exists(p)) - final = fs::absolute(p); - else if (p = "../../.." / third_party / sevenz; fs::exists(p)) - final = fs::absolute(p); - - if (final.empty()) - bail_out("7z.exe not found"); - - debug("found " + final.string()); - return final; - }(); - - return path; -} - } // namespace diff --git a/src/conf.h b/src/conf.h index 147c0ae..8fb78b1 100644 --- a/src/conf.h +++ b/src/conf.h @@ -19,6 +19,7 @@ struct paths { static fs::path prefix(); static fs::path cache(); + static fs::path patches(); static fs::path build(); static fs::path install(); static fs::path install_bin(); @@ -30,6 +31,13 @@ struct paths static fs::path temp_file(); }; +struct third_party +{ + static fs::path sevenz(); + static fs::path jom(); + static fs::path patch(); +}; + struct conf { static bool verbose(); diff --git a/src/main.cpp b/src/main.cpp index 1e519f5..68d35d5 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -54,19 +54,18 @@ public: if (interrupted_) return; - build(); - - if (interrupted_) - return; - - install(); + build_and_install(); } catch(bailed e) { - bailed_ = e; error(name_ + " bailed out, interrupting all tasks"); interrupt_all(); } + catch(std::exception& e) + { + error(name_ + " uncaught exception: " + e.what()); + interrupt_all(); + } }); } @@ -82,26 +81,18 @@ public: void join() { if (thread_.joinable()) - { thread_.join(); - if (bailed_) - throw *bailed_; - } } void fetch() { do_fetch(); + run_tool(paths::patches() / name_, source_path()); } - void build() + void build_and_install() { - do_build(); - } - - void install() - { - do_install(); + do_build_and_install(); } protected: @@ -111,8 +102,8 @@ protected: } virtual void do_fetch() {}; - virtual void do_build() {}; - virtual void do_install() {}; + virtual void do_build_and_install() {}; + virtual fs::path source_path() const = 0; template auto run_tool(Args&&... args) @@ -132,7 +123,6 @@ protected: private: std::string name_; std::thread thread_; - std::optional bailed_; std::atomic interrupted_; std::unique_ptr tool_; @@ -153,6 +143,11 @@ public: } protected: + fs::path source_path() const override + { + return paths::build() / ("7zip-" + versions::sevenzip()); + } + void do_fetch() override { const auto nodots = replace_all(versions::sevenzip(), ".", ""); @@ -160,25 +155,19 @@ protected: const auto file = run_tool( "https://www.7-zip.org/a/7z" + nodots + "-src.7z"); - run_tool(file, src_path()); + run_tool(file, source_path()); } - void do_build() + void do_build_and_install() override { - /*const fs::path src = - src_path() / "CPP" / "7zip" / "Bundles" / "Format7zF"; + const fs::path src = + source_path() / "CPP" / "7zip" / "Bundles" / "Format7zF"; run_tool(src, "/NOLOGO CPU=x64 NEW_COMPILER=1 " "MY_STATIC_LINK=1 NO_BUFFEROVERFLOWU=1"); - op::copy_file_to_dir(src / "x64/7z.dll", paths::install_dlls());*/ - } - -private: - fs::path src_path() const - { - return paths::build() / ("7zip-" + versions::sevenzip()); + op::copy_file_to_dir_if_better(src / "x64/7z.dll", paths::install_dlls()); } }; @@ -192,30 +181,27 @@ public: } protected: - void do_fetch() + fs::path source_path() const override + { + return paths::build() / ("zlib-" + versions::zlib()); + } + + void do_fetch() override { const auto file = run_tool( "http://zlib.net/zlib-" + versions::zlib() + ".tar.gz"); - run_tool(file, src_path()); + run_tool(file, source_path()); } - void do_build() + void do_build_and_install() override { - run_tool(src_path(), "", src_path()); - run_tool(src_path() / cmake_for_nmake::build_path()); - } + run_tool(source_path(), "", source_path()); + run_tool(source_path() / cmake_for_nmake::build_path()); - void do_install() - { - run_tool(src_path() / cmake_for_nmake::build_path()); - op::copy_file_to_dir(src_path() / cmake_for_nmake::build_path() / "zconf.h", src_path()); - } - -private: - fs::path src_path() - { - return paths::build() / ("zlib-" + versions::zlib()); + op::copy_file_to_dir_if_better( + source_path() / cmake_for_nmake::build_path() / "zconf.h", + source_path()); } }; @@ -229,7 +215,13 @@ public: } protected: - void do_fetch() + fs::path source_path() const override + { + const auto underscores = replace_all(versions::boost(), ".", "_"); + return paths::build() / ("boost_" + underscores); + } + + void do_fetch() override { const auto underscores = replace_all(versions::boost(), ".", "_"); @@ -237,25 +229,20 @@ protected: "https://github.com/ModOrganizer2/modorganizer-umbrella/" "releases/download/1.1/boost_prebuilt_" + underscores + ".7z"); - run_tool(file, src_path()); + run_tool(file, source_path()); } - void do_build() + void do_build_and_install() override { - op::copy_file_to_dir(lib_path() / python_dll(), paths::install_bin()); + op::copy_file_to_dir_if_better( + lib_path() / python_dll(), paths::install_bin()); } private: - fs::path src_path() const - { - const auto underscores = replace_all(versions::boost(), ".", "_"); - return paths::build() / ("boost_" + underscores); - } - fs::path lib_path() const { const std::string lib = "lib64-msvc-" + versions::boost_vs(); - return src_path() / lib / "lib"; + return source_path() / lib / "lib"; } std::string python_dll() const @@ -315,25 +302,24 @@ public: } protected: - void do_fetch() + fs::path source_path() const override + { + return paths::build() / ("fmt-" + versions::fmt()); + } + + void do_fetch() override { const auto file = run_tool( "https://github.com/fmtlib/fmt/releases/download/" + versions::fmt() + "/fmt-" + versions::fmt() + ".zip"); - run_tool(file, src_path()); + run_tool(file, source_path()); } - void do_build() + void do_build_and_install() override { - run_tool(src_path(), "-DFMT_TEST=OFF -DFMT_DOC=OFF"); - nmake(src_path() / cmake_for_nmake::build_path()); - } - -private: - fs::path src_path() - { - return paths::build() / ("fmt-" + versions::fmt()); + run_tool(source_path(), "-DFMT_TEST=OFF -DFMT_DOC=OFF"); + nmake(source_path() / cmake_for_nmake::build_path()); } }; @@ -349,11 +335,9 @@ public: protected: - void do_fetch() + void do_fetch() override { - //run_tool("dummy process", "grep 1"); std::apply([&](auto&&... args){ run_tool(args...); }, args_); - //run_tool( } private: diff --git a/src/op.cpp b/src/op.cpp index 9c78d12..c3a4e84 100644 --- a/src/op.cpp +++ b/src/op.cpp @@ -6,12 +6,18 @@ namespace builder { -process::process(handle_ptr h) - : handle_(std::move(h)), code_(0) +process::process() + : code_(0) +{ +} + +process::process(std::string cmd, handle_ptr h) + : cmd_(std::move(cmd)), handle_(std::move(h)), code_(0) { } process::process(process&& p) : + cmd_(std::move(p.cmd_)), handle_(std::move(p.handle_)), interrupt_(p.interrupt_.load()), code_(p.code_) @@ -20,6 +26,7 @@ process::process(process&& p) : process& process::operator=(process&& p) { + cmd_ = std::move(p.cmd_); handle_ = std::move(p.handle_); interrupt_ = p.interrupt_.load(); code_ = p.code_; @@ -86,6 +93,11 @@ void process::join() handle_ = {}; } +const std::string& process::cmd() const +{ + return cmd_; +} + int process::exit_code() const { return static_cast(code_); @@ -151,10 +163,70 @@ void op::remove_readonly(const fs::path& first) } } -void op::copy_file_to_dir(const fs::path& file, const fs::path& dir) +bool is_source_better(const fs::path& src, const fs::path& dest) { - info(file.string() + " -> " + dir.string()); + if (!fs::exists(dest)) + { + debug("target " + dest.string() + " doesn't exist; copying"); + return true; + } + std::error_code ec; + + const auto src_size = fs::file_size(src, ec); + if (ec) + { + warn("failed to get size of " + src.string() + "; forcing copy"); + return true; + } + + const auto dest_size = fs::file_size(dest, ec); + if (ec) + { + warn("failed to get size of " + dest.string() + "; forcing copy"); + return true; + } + + if (src_size != dest_size) + { + debug( + "src " + src.string() + " is " + std::to_string(src_size) + "), " + "dest " + dest.string() + " is " + std::to_string(dest_size) + "); " + "sizes different, copying"); + + return true; + } + + + const auto src_time = fs::last_write_time(src, ec); + if (ec) + { + warn("failed to get time of " + src.string() + "; forcing copy"); + return true; + } + + const auto dest_time = fs::last_write_time(dest, ec); + if (ec) + { + warn("failed to get time of " + dest.string() + "; forcing copy"); + return true; + } + + if (src_time > dest_time) + { + debug( + "src " + src.string() + " is newer than " + dest.string() + "; " + "copying"); + + return true; + } + + // same size, same date + return false; +} + +void op::copy_file_to_dir_if_better(const fs::path& file, const fs::path& dir) +{ check(file); check(dir); @@ -164,8 +236,18 @@ void op::copy_file_to_dir(const fs::path& file, const fs::path& dir) if (fs::exists(dir) && !fs::is_directory(dir)) bail_out("can't copy to " + dir.string() + ", not a directory"); - if (!conf::dry()) - do_copy_file_to_dir(file, dir); + const auto target = dir / file.filename(); + if (is_source_better(file, target)) + { + info(file.string() + " -> " + dir.string()); + + if (!conf::dry()) + do_copy_file_to_dir(file, dir); + } + else + { + debug("(skipped) " + file.string() + " -> " + dir.string()); + } } process op::run(const std::string& cmd, const fs::path& cwd) @@ -280,7 +362,7 @@ process op::do_run(const std::string& what, const fs::path& cwd) ::CloseHandle(pi.hThread); - return process(handle_ptr(pi.hProcess)); + return process(what, handle_ptr(pi.hProcess)); } void op::check(const fs::path& p) diff --git a/src/op.h b/src/op.h index 15875ea..b3eb53a 100644 --- a/src/op.h +++ b/src/op.h @@ -8,7 +8,9 @@ namespace builder class process { public: - process(handle_ptr h={}); + process(); + process(std::string cmd, handle_ptr h); + process(process&&); process& operator=(process&&); ~process(); @@ -18,9 +20,12 @@ public: void interrupt(); void join(); + + const std::string& cmd() const; int exit_code() const; private: + std::string cmd_; handle_ptr handle_; std::atomic interrupt_; DWORD code_; @@ -35,7 +40,7 @@ public: static void delete_directory(const fs::path& p); static void delete_file(const fs::path& p); static void remove_readonly(const fs::path& first); - static void copy_file_to_dir(const fs::path& file, const fs::path& dir); + static void copy_file_to_dir_if_better(const fs::path& file, const fs::path& dir); static process run(const std::string& cmd, const fs::path& cwd={}); private: diff --git a/src/tools.cpp b/src/tools.cpp index 0069a7f..3ff94d0 100644 --- a/src/tools.cpp +++ b/src/tools.cpp @@ -68,15 +68,19 @@ void vcvars() } } + process do_cmake( const fs::path& build, const fs::path& prefix, const std::string& args, const std::string& generator) { - std::string cmd = "cmake -G \"" + generator + "\""; + std::string cmd = "cmake -G \"" + generator + "\" -DCMAKE_INSTALL_MESSAGE=NEVER"; if (!prefix.empty()) cmd += " -DCMAKE_INSTALL_PREFIX=\"" + prefix.string() + "\""; + if (!conf::verbose()) + cmd += " --log-level=WARNING"; + if (!args.empty()) cmd += " " + args; @@ -90,10 +94,9 @@ process do_nmake( const std::string& what, const std::string& args) { std::ostringstream oss; + oss << "\"" << third_party::jom().string() << "\""; - oss << "nmake"; - - if (conf::verbose()) + if (!conf::verbose()) oss << " /C /S"; if (!args.empty()) @@ -144,7 +147,7 @@ process_runner::process_runner(std::string name, std::string cmd, fs::path cwd) void process_runner::do_run() { - set(op::run(cmd_, cwd_)); + execute_and_join(op::run(cmd_, cwd_)); } void process_runner::do_interrupt() @@ -152,13 +155,26 @@ void process_runner::do_interrupt() process_.interrupt(); } -void process_runner::set(process p) +int process_runner::execute_and_join(process p, bool check_exit_code) { process_ = std::move(p); + join(check_exit_code); + return process_.exit_code(); +} + +void process_runner::join(bool check_exit_code) +{ process_.join(); - if (process_.exit_code() != 0) - bail_out("command returned " + std::to_string(p.exit_code())); + if (check_exit_code) + { + if (process_.exit_code() != 0) + { + bail_out( + process_.cmd() + " returned " + + std::to_string(process_.exit_code())); + } + } } @@ -192,8 +208,6 @@ decompresser::decompresser(fs::path file, fs::path where) : void decompresser::do_run() { - info("decompress " + file_.string() + " into " + where_.string()); - if (fs::exists(interrupt_file())) { debug("found interrupt file " + interrupt_file().string()); @@ -205,7 +219,11 @@ void decompresser::do_run() return; } - const auto sevenz = "\"" + find_sevenz().string() + "\""; + info("decompress " + file_.string() + " into " + where_.string()); + + op::touch(interrupt_file()); + + const auto sevenz = "\"" + third_party::sevenz().string() + "\""; //op::delete_directory(where_); op::create_directories(where_); @@ -225,11 +243,9 @@ void decompresser::do_run() "\"" + file_.string() + "\" " + redir_nul()); } - set(std::move(p)); + execute_and_join(std::move(p)); - if (interrupted()) - op::touch(interrupt_file()); - else + if (!interrupted()) op::delete_file(interrupt_file()); } @@ -239,6 +255,67 @@ fs::path decompresser::interrupt_file() const } +patcher::patcher(fs::path patch_dir, fs::path output_dir) : + process_runner("patcher"), + patches_(std::move(patch_dir)), output_(std::move(output_dir)) +{ +} + +void patcher::do_run() +{ + if (!fs::exists(patches_)) + return; + + std::ostringstream oss; + + oss + << "\"" << third_party::patch().string() << "\" " + << "--read-only=ignore " + << "--strip=0 " + << "--directory=\"" << output_.string() << "\" "; + + if (!conf::verbose()) + oss << "--quiet "; + + const std::string base = oss.str(); + + for (auto e : fs::directory_iterator(patches_)) + { + if (!e.is_regular_file()) + continue; + + const auto p = e.path(); + if (p.extension() != ".patch") + { + warn( + "file without .patch extension " + p.string() + " " + "in patches directory " + patches_.string()); + + continue; + } + + const std::string input = "--input=\"" + p.string() + "\""; + const std::string check = base + " --dry-run --force --reverse " + input; + const std::string apply = base + " --forward --batch " + input; + + { + // check + if (execute_and_join(op::run(check), false) == 0) + { + debug("patch " + p.string() + " already applied"); + continue; + } + } + + { + // apply + info("applying patch " + p.string()); + execute_and_join(op::run(apply)); + } + } +} + + cmake_for_nmake::cmake_for_nmake(fs::path r, std::string a, fs::path p) : process_runner("cmake_for_nmake"), root_(std::move(r)), args_(std::move(a)), prefix_(std::move(p)) @@ -254,7 +331,7 @@ void cmake_for_nmake::do_run() { const auto build = root_ / build_path(); const std::string g = "NMake Makefiles"; - set(do_cmake(build, prefix_, args_, g)); + execute_and_join(do_cmake(build, prefix_, args_, g)); } @@ -273,7 +350,7 @@ void cmake_for_vs::do_run() { const auto build = root_ / build_path(); const std::string g = "Visual Studio " + versions::vs() + " " + versions::vs_year(); - set(do_cmake(build, prefix_, args_, g)); + execute_and_join(do_cmake(build, prefix_, args_, g)); } @@ -284,7 +361,7 @@ nmake::nmake(fs::path dir, std::string args) void nmake::do_run() { - set(do_nmake(dir_, "", args_)); + execute_and_join(do_nmake(dir_, "", args_)); } nmake_install::nmake_install(fs::path dir, std::string args) : @@ -295,7 +372,7 @@ nmake_install::nmake_install(fs::path dir, std::string args) : void nmake_install::do_run() { - set(do_nmake(dir_, "install", args_)); + execute_and_join(do_nmake(dir_, "install", args_)); } } // namespace diff --git a/src/tools.h b/src/tools.h index 0fd0e85..cfc726c 100644 --- a/src/tools.h +++ b/src/tools.h @@ -7,7 +7,7 @@ namespace builder { void vcvars(); -fs::path find_sevenz(); + class tool { @@ -51,13 +51,14 @@ class process_runner : public tool { public: process_runner(std::string name, std::string cmd, fs::path cwd={}); + void join(bool check_exit_code=true); protected: process_runner(std::string name); void do_run() override; void do_interrupt() override; - void set(process p); + int execute_and_join(process p, bool check_exit_code=true); private: std::string cmd_; @@ -82,6 +83,20 @@ private: }; +class patcher : public process_runner +{ +public: + patcher(fs::path patch_dir, fs::path output_dir); + +protected: + void do_run() override; + +private: + fs::path patches_; + fs::path output_; +}; + + class cmake_for_nmake : public process_runner { public: diff --git a/src/utility.h b/src/utility.h index 16644cc..f774185 100644 --- a/src/utility.h +++ b/src/utility.h @@ -62,6 +62,12 @@ void error(Args&&... args) out(level::error, std::forward(args)...); } +template +void warn(Args&&... args) +{ + out(level::warning, std::forward(args)...); +} + template void info(Args&&... args) { diff --git a/third-party/bin/jom.exe b/third-party/bin/jom.exe new file mode 100644 index 0000000..14a199a Binary files /dev/null and b/third-party/bin/jom.exe differ diff --git a/third-party/bin/msys-2.0.dll b/third-party/bin/msys-2.0.dll new file mode 100644 index 0000000..de4bf16 Binary files /dev/null and b/third-party/bin/msys-2.0.dll differ diff --git a/third-party/bin/patch.exe b/third-party/bin/patch.exe new file mode 100644 index 0000000..c11b0be Binary files /dev/null and b/third-party/bin/patch.exe differ