From 2246d52197ba42b56167e2315b3d4afbc16702eb Mon Sep 17 00:00:00 2001 From: isanae <14251494+isanae@users.noreply.github.com> Date: Sat, 16 May 2020 17:55:39 -0400 Subject: [PATCH] release command --- src/commands.cpp | 515 +++++++++++++++++++++++++++++++++++++++------- src/commands.h | 66 +++++- src/conf.cpp | 12 +- src/main.cpp | 27 ++- src/op.cpp | 80 +++++++ src/op.h | 10 + src/tasks/tasks.h | 3 +- vs/props.props | 2 +- 8 files changed, 619 insertions(+), 96 deletions(-) diff --git a/src/commands.cpp b/src/commands.cpp index d155873..7474012 100644 --- a/src/commands.cpp +++ b/src/commands.cpp @@ -13,11 +13,34 @@ std::string version() return "mob 1.0"; } +void help(const clipp::group& g, const std::string& more) +{ +#pragma warning(suppress: 4548) + auto usage_df = clipp::doc_formatting() + .first_column(4) + .doc_column(30); + + auto options_df = clipp::doc_formatting() + .first_column(4) + .doc_column(30); + + u8cout + << "Usage:\n" << clipp::usage_lines(g, "mob", usage_df) + << "\n\n" + << "Options:\n" + << clipp::documentation(g, options_df) + << "\n"; + + if (!more.empty()) + u8cout << "\n" << more << "\n"; +} + + command::common_options command::common; -command::command() - : picked_(false), code_(0) +command::command(flags f) + : picked_(false), help_(false), flags_(f), code_(0) { } @@ -25,7 +48,7 @@ clipp::group command::common_options_group() { auto& o = common; - return "Options" % ( + return (clipp::option("-i", "--ini") & clipp::value("FILE") >> o.ini) % ("path to the ini file"), @@ -52,8 +75,7 @@ clipp::group command::common_options_group() (clipp::repeatable(clipp::option("-s", "--set") >> o.set & clipp::opt_value("OPTION", o.options))) % "sets an option, such as 'versions/openssl=1.2'; -s with no " - "arguments lists the available options" - ); + "arguments lists the available options"; } void command::force_exit_code(int code) @@ -71,8 +93,26 @@ bool command::picked() const return picked_; } +bool command::wants_help() const +{ + return help_; +} + +clipp::group command::group() +{ + return (do_group(), + (clipp::option("-h", "--help") >> help_) + % ("shows this message")); +} + int command::run() { + if (help_) + { + help(group(), do_doc()); + return 0; + } + auto& o = common; if (o.file_log_level == -1) @@ -101,6 +141,17 @@ int command::run() if (!o.prefix.empty()) o.options.push_back("paths/prefix=" + o.prefix); + do_pre_run(); + + if (flags_ & requires_options) + { + init_options(command::common.ini, command::common.options); + dump_options(); + + if (!verify_options()) + return 1; + } + const auto r = do_run(); if (code_) @@ -110,13 +161,10 @@ int command::run() } - -clipp::group version_command::group() +clipp::group version_command::do_group() { return clipp::group( - "version command" % ( - (clipp::command("version", "-v", "--version")).set(picked_) - )); + clipp::command("version", "-v", "--version").set(picked_)); } int version_command::do_run() @@ -126,12 +174,10 @@ int version_command::do_run() } -clipp::group help_command::group() +clipp::group help_command::do_group() { return clipp::group( - "help command" % ( - (clipp::command("-h", "--help")).set(picked_) - )); + clipp::command("-h", "--help").set(picked_)); } int help_command::do_run() @@ -139,40 +185,27 @@ int help_command::do_run() #pragma warning(suppress: 4548) auto doc = (command::common_options_group(), (clipp::value("command"))); - auto usage_df = clipp::doc_formatting() - .first_column(4) - .doc_column(30); - - auto options_df = clipp::doc_formatting() - .first_column(0) - .doc_column(30); - - u8cout - << "Usage:\n" << clipp::usage_lines(doc, "mob", usage_df) - << "\n\n" - << "Commands:\n" - << " help shows this message\n" - << " version shows the version\n" - << " build build tasks\n" - << " list lists available tasks\n" - << " devbuild create a devbuild\n" - << "\n" - << clipp::documentation(doc, options_df) - << "\n\n" - << "Invoking `mob` without a command builds everything. Do\n" - << "`mob build ...` to build specific tasks. See\n" - << "`mob command --help` for more information about a command.\n"; + help(doc, + "Commands:\n" + " help shows this message\n" + " version shows the version\n" + " list lists available tasks\n" + " options lists available options and default values\n" + " build builds tasks\n" + " release creates a release or a devbuild" + "\n\n" + "Invoking `mob` without a command builds everything. Do " + "`mob build ...` to build specific tasks. See\n" + "`mob command --help` for more information about a command."); return 0; } -clipp::group options_command::group() +clipp::group options_command::do_group() { return clipp::group( - "options command" % ( - (clipp::command("-h", "--help")).set(picked_) - )); + clipp::command("options").set(picked_)); } int options_command::do_run() @@ -181,10 +214,23 @@ int options_command::do_run() return 0; } - -clipp::group build_command::group() +std::string options_command::do_doc() { - return "build command" % ( + return + "Lists all available options in the form of `section/key = default`.\n" + "They can be changed in the INI file by setting `key = value` within\n" + "`[section]` or with `-s section/key=value`."; +} + + +build_command::build_command() + : command(requires_options) +{ +} + +clipp::group build_command::do_group() +{ + return (clipp::command("build")).set(picked_), (clipp::option("-g", "--redownload") >> redownload_) @@ -202,30 +248,25 @@ clipp::group build_command::group() (clipp::opt_values( clipp::match::prefix_not("-"), "task", tasks_)) % "tasks to run; specify 'super' to only build modorganizer " - "projects" - ); + "projects"; +} + +void build_command::do_pre_run() +{ + if (redownload_ || clean_) + common.options.push_back("options/redownload=true"); + + if (reextract_ || clean_) + common.options.push_back("options/reextract=true"); + + if (rebuild_ || clean_) + common.options.push_back("options/rebuild=true"); } int build_command::do_run() { try { - if (redownload_ || clean_) - common.options.push_back("options/redownload=true"); - - if (reextract_ || clean_) - common.options.push_back("options/reextract=true"); - - if (rebuild_ || clean_) - common.options.push_back("options/rebuild=true"); - - - init_options(command::common.ini, command::common.options); - dump_options(); - - if (!verify_options()) - return 1; - curl_init curl; if (!tasks_.empty()) @@ -244,12 +285,10 @@ int build_command::do_run() } -clipp::group list_command::group() +clipp::group list_command::do_group() { return clipp::group( - "list command" % ( - clipp::command("list").set(picked_) - )); + clipp::command("list").set(picked_)); } int list_command::do_run() @@ -258,19 +297,347 @@ int list_command::do_run() return 0; } - -clipp::group devbuild_command::group() +std::string list_command::do_doc() { - return clipp::group( - "devbuild command" % ( - clipp::command("devbuild").set(picked_) - )); + return + "Lists all available tasks. The special task name `super` is a\n" + "shorthand for all modorganizer tasks."; } -int devbuild_command::do_run() + +release_command::release_command() + : command(requires_options) { - verify_options(); +} + +void release_command::make_bin() +{ + const auto out = out_ / make_filename(""); + u8cout << "making binary archive " << path_to_utf8(out) << "\n"; + + op::archive_from_glob(gcx(), + paths::install_bin() / "*", out, {"__pycache__"}); +} + +void release_command::make_pdbs() +{ + const auto out = out_ / make_filename("pdbs"); + u8cout << "making pdbs archive " << path_to_utf8(out) << "\n"; + + op::archive_from_glob(gcx(), + paths::install_pdbs() / "*", out, {"__pycache__"}); +} + +void release_command::make_src() +{ + const auto out = out_ / make_filename("src"); + u8cout << "making src archive " << path_to_utf8(out) << "\n"; + + const std::vector ignore = + { + "\\..+", // dot files + ".*\\.log", + ".*\\.tlog", + ".*\\.dll", + ".*\\.exe", + ".*\\.lib", + ".*\\.obj", + ".*\\.ts", + ".*\\.aps", + "vsbuild" + }; + + const std::vector ignore_re(ignore.begin(), ignore.end()); + + std::vector files; + std::size_t total_size = 0; + + if (!fs::exists(modorganizer::super_path())) + { + gcx().bail_out(context::generic, + "modorganizer super path not found: {}", + modorganizer::super_path()); + } + + walk_dir(modorganizer::super_path(), files, ignore_re, total_size); + + // should be below 20MB + const std::size_t max_expected_size = 20 * 1024 * 1024; + if (total_size >= max_expected_size) + { + gcx().warning(context::generic, + "total size of source files would be {}, expected something " + "below {}, something might be wrong", + total_size, max_expected_size); + + if (!force_) + { + gcx().bail_out(context::generic, + "bailing out, use --force to ignore"); + } + } + + op::archive_from_files(gcx(), + files, modorganizer::super_path(), out); +} + +void release_command::walk_dir( + const fs::path& dir, std::vector& files, + const std::vector& ignore_re, std::size_t& total_size) +{ + for (auto e : fs::directory_iterator(dir)) + { + const auto p = e.path(); + const auto filename = path_to_utf8(p.filename()); + + bool ignored = false; + + for (auto&& re : ignore_re) + { + if (std::regex_match(filename, re)) + { + ignored = true; + break; + } + } + + if (ignored) + continue; + + if (e.is_directory()) + { + walk_dir(e.path(), files, ignore_re, total_size); + } + else if (e.is_regular_file()) + { + total_size += fs::file_size(p); + files.push_back(p); + } + } +} + +fs::path release_command::make_filename(const std::string& what) const +{ + std::string filename = "Mod.Organizer"; + + if (!version_.empty()) + filename += "-" + version_; + + if (!suffix_.empty()) + filename += "-" + suffix_; + + if (!what.empty()) + filename += "-" + what; + + filename += ".7z"; + + return filename; +} + +clipp::group release_command::do_group() +{ + return clipp::group( + clipp::command("release").set(picked_), + + ( + clipp::option("--no-bin").set(bin_, false) | + clipp::option("--bin").set(bin_, true) + ) % "sets whether the binary archive is created [default: yes]", + + ( + clipp::option("--no-pdbs").set(pdbs_, false) | + clipp::option("--pdbs").set(pdbs_, true) + ) % "sets whether the PDBs archive is created [default: yes]", + + ( + clipp::option("--no-src").set(src_, false) | + clipp::option("--src").set(src_, true) + ) % "sets whether the source archive is created [default: yes]", + + clipp::option("--version-from-exe").set(version_exe_) + % "retrieves version information from ModOrganizer.exe " + "[default]", + + clipp::option("--version-from-rc").set(version_rc_) + % "retrieves version information from modorganizer/src/version.rc", + + (clipp::option("--rc") + & clipp::value("PATH") >> utf8_rc_path_) + % "overrides the path to version.rc", + + (clipp::option("--version") + & clipp::value("VERSION") >> version_) + % "overrides the version string", + + (clipp::option("--output-dir") + & clipp::value("PATH") >> utf8out_) + % "sets the output directory to use instead of `$prefix/releases`", + + (clipp::option("--suffix") + & clipp::value("SUFFIX") >> suffix_) + % "optional suffix to add to the archive filenames", + + clipp::option("--force").set(force_) + % "ignores file size warnings which could indicate bad paths or " + "unexpected files being pulled into the archives" + ); +} + +int release_command::do_run() +{ + out_ = fs::path(utf8_to_utf16(utf8out_)); + if (out_.empty()) + out_ = paths::prefix() / "releases"; + else if (out_.is_relative()) + out_ = paths::prefix() / out_; + + rc_path_ = fs::path(utf8_to_utf16(utf8_rc_path_)); + if (rc_path_.empty()) + { + rc_path_ = + modorganizer::super_path() / "modorganizer" / "src" / "version.rc"; + } + + if (version_.empty()) + { + if (version_rc_) + version_ = version_from_rc(); + else + version_ = version_from_exe(); + } + + u8cout << + ">> don't forget to update the version number before making a " + "release\n"; + + if (bin_) + make_bin(); + + if (pdbs_) + make_pdbs(); + + if (src_) + make_src(); + return 0; } +std::string release_command::do_doc() +{ + return + "Creates three archives in the current directory: one from \n" + "`install/bin/*`, one from `install/pdbs/*` and another with the \n" + "sources of projects from modorganizer_super.\n" + "\n" + "The archive filename is `Mod.Organizer-version-suffix-what.7z`,\n" + "where:\n" + " - `version` is taken from `ModOrganizer.exe`, `version.rc`\n" + " or from --version;\n" + " - `suffix` is the optional `--suffix` argument;\n" + " - `what` is either nothing, `src` or `pdbs`."; +} + +std::string release_command::version_from_exe() const +{ + const auto exe = paths::install_bin() / "ModOrganizer.exe"; + + // getting version info size + DWORD dummy = 0; + const DWORD size = GetFileVersionInfoSizeW(exe.native().c_str(), &dummy); + + if (size == 0) + { + const auto e = GetLastError(); + gcx().bail_out(context::generic, + "can't get file version info size from {}, {}", + exe, error_message(e)); + } + + // getting version info + auto buffer = std::make_unique(size); + + if (!GetFileVersionInfoW(exe.native().c_str(), 0, size, buffer.get())) + { + const auto e = GetLastError(); + gcx().bail_out(context::generic, + "can't get file version info from {}, {}", + exe, error_message(e)); + } + + struct LANGANDCODEPAGE + { + WORD wLanguage; + WORD wCodePage; + }; + + void* value_pointer = nullptr; + unsigned int value_size = 0; + + // getting list of available languages + auto ret = VerQueryValueW( + buffer.get(), L"\\VarFileInfo\\Translation", + &value_pointer, &value_size); + + if (!ret || !value_pointer || value_size == 0) + { + const auto e = GetLastError(); + gcx().bail_out(context::generic, + "VerQueryValueW() for translations failed on {}, {}", + exe, error_message(e)); + } + + // number of languages + const auto count = value_size / sizeof(LANGANDCODEPAGE); + if (count == 0) + gcx().bail_out(context::generic, "no languages found in {}", exe); + + // using the first language in the list to get FileVersion + const auto* lcp = static_cast(value_pointer); + + const auto sub_block = ::fmt::format( + L"\\StringFileInfo\\{:04x}{:04x}\\FileVersion", + lcp->wLanguage, lcp->wCodePage); + + ret = VerQueryValueW( + buffer.get(), sub_block.c_str(), &value_pointer, &value_size); + + if (!ret || !value_pointer || value_size == 0) + { + gcx().bail_out(context::generic, + "language {} not found in {}", sub_block, exe); + } + + // value_size includes the null terminator + return utf16_to_utf8(std::wstring( + static_cast(value_pointer), + value_size - 1)); +} + +std::string release_command::version_from_rc() const +{ + // matching: #define VER_FILEVERSION_STR "2.2.1\0" + std::regex re(R"(#define VER_FILEVERSION_STR "(.+)\\0")"); + + const std::string rc = op::read_text_file( + gcx(), encodings::utf8, rc_path_); + + std::smatch m; + std::string v; + + for_each_line(rc, [&](std::string_view line) + { + std::string line_s(line); + if (std::regex_match(line_s, m, re)) + v = m[1]; + }); + + if (v.empty()) + { + gcx().bail_out(context::generic, + "can't find version string in {}", rc_path_); + } + + return v; +} + } // namespace diff --git a/src/commands.h b/src/commands.h index 80788f8..a5ab6c0 100644 --- a/src/commands.h +++ b/src/commands.h @@ -6,7 +6,6 @@ namespace mob class command { public: - command(); virtual ~command() = default; struct common_options @@ -24,21 +23,35 @@ public: static common_options common; static clipp::group common_options_group(); - virtual clipp::group group() = 0; void force_exit_code(int code); void force_pick(); bool picked() const; + bool wants_help() const; + clipp::group group(); int run(); protected: - bool picked_; + enum flags + { + noflags = 0x00, + requires_options = 0x01, + }; + bool picked_; + bool help_; + + command(flags f=noflags); + + virtual clipp::group do_group() = 0; + virtual void do_pre_run() {}; virtual int do_run() = 0; + virtual std::string do_doc() { return {}; } private: + flags flags_; std::optional code_; }; @@ -46,9 +59,9 @@ private: class version_command : public command { public: - clipp::group group() override; protected: + clipp::group do_group() override; int do_run() override; }; @@ -56,9 +69,9 @@ protected: class help_command : public command { public: - clipp::group group() override; protected: + clipp::group do_group() override; int do_run() override; }; @@ -66,19 +79,22 @@ protected: class options_command : public command { public: - clipp::group group() override; protected: + clipp::group do_group() override; int do_run() override; + std::string do_doc() override; }; class build_command : public command { public: - clipp::group group() override; + build_command(); protected: + clipp::group do_group() override; + void do_pre_run() override; int do_run() override; private: @@ -93,20 +109,50 @@ private: class list_command : public command { public: - clipp::group group() override; protected: + clipp::group do_group() override; int do_run() override; + std::string do_doc() override; }; -class devbuild_command : public command +class release_command : public command { public: - clipp::group group() override; + release_command(); + + void make_bin(); + void make_pdbs(); + void make_src(); protected: + clipp::group do_group() override; int do_run() override; + std::string do_doc() override; + +private: + bool bin_ = true; + bool src_ = true; + bool pdbs_ = true; + std::string utf8out_; + fs::path out_; + std::string version_; + bool version_exe_ = false; + bool version_rc_ = false; + std::string utf8_rc_path_; + fs::path rc_path_; + bool force_; + std::string suffix_; + + fs::path make_filename(const std::string& what) const; + + void walk_dir( + const fs::path& dir, std::vector& files, + const std::vector& ignore_re, std::size_t& total_size); + + std::string version_from_exe() const; + std::string version_from_rc() const; }; } // namespace diff --git a/src/conf.cpp b/src/conf.cpp index e03f678..b49c54f 100644 --- a/src/conf.cpp +++ b/src/conf.cpp @@ -329,19 +329,19 @@ void set_option(const std::string& s) void dump_available_options() { for (auto&& [k, v] : g_options) - u8cout << "options/" << k << "\n"; + u8cout << "options/" << k << " = " << v << "\n"; for (auto&& [k, v] : g_tools) - u8cout << "tools/" << k << "\n"; + u8cout << "tools/" << k << " = " << v << "\n"; for (auto&& [k, v] : g_prebuilt) - u8cout << "prebuilt/" << k << "\n"; + u8cout << "prebuilt/" << k << " = " << v << "\n"; for (auto&& [k, v] : g_versions) - u8cout << "versions/" << k << "\n"; + u8cout << "versions/" << k << " = " << v << "\n"; for (auto&& [k, v] : g_paths) - u8cout << "paths/" << k << "\n"; + u8cout << "paths/" << k << " = " << v << "\n"; } bool try_parts(fs::path& check, const std::vector& parts) @@ -931,7 +931,7 @@ void init_options(const fs::path& ini, const std::vector& opts) make_canonical_path("install", paths::prefix(), "install"); make_canonical_path("install_bin", paths::install(), "bin"); make_canonical_path("install_libs", paths::install(), "libs"); - make_canonical_path("install_pdbs", paths::install(), "pdbs"); + make_canonical_path("install_pdbs", paths::install(), "pdb"); make_canonical_path("install_dlls", paths::install_bin(), "dlls"); make_canonical_path("install_loot", paths::install_bin(), "loot"); make_canonical_path("install_plugins", paths::install_bin(), "plugins"); diff --git a/src/main.cpp b/src/main.cpp index 7246a1e..a1a7765 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -22,7 +22,7 @@ std::shared_ptr handle_command_line(const std::vector& arg std::make_unique(), build, std::make_unique(), - std::make_unique(), + std::make_unique(), }; @@ -43,14 +43,36 @@ std::shared_ptr handle_command_line(const std::vector& arg if (!pr) { - pr = clipp::parse(args, command::common_options_group()); + // some commands have mandatory options, like devbuild, which requires + // the build number + // + // doing `devbuild -h` therefore fails to parse because of the missing + // build number + // + // but options are actually still set correctly, so -h can be checked + // manually here + for (auto&& c : commands) + { + if (c->picked() && c->wants_help()) + return std::move(c); + } + + // another way parsing can fail is if no commands are given, like just + // running `mob` as-is; clipp doesn't really have a ways of setting + // a default command to use, to the arguments can be reparsed with + // only the common options + cli = command::common_options_group(); + pr = clipp::parse(args, cli); if (!pr) { + // bad command line help->force_exit_code(1); return help; } + // if this parsing works, it means the user invoked `mob` without a + // command; default to `build` build->force_pick(); } @@ -60,7 +82,6 @@ std::shared_ptr handle_command_line(const std::vector& arg if (c->picked()) return std::move(c); } - u8cout << "!!\n"; return {}; } diff --git a/src/op.cpp b/src/op.cpp index a18217b..65fe884 100644 --- a/src/op.cpp +++ b/src/op.cpp @@ -3,6 +3,7 @@ #include "utility.h" #include "conf.h" #include "context.h" +#include "process.h" namespace mob::op { @@ -473,6 +474,85 @@ void write_text_file( "finished writing {} bytes to {}", bytes.size(), p); } +void archive_from_glob( + const context& cx, + const fs::path& src_glob, const fs::path& dest_file, + const std::vector& ignore) +{ + cx.trace(context::fs, "archiving {} into {}", src_glob, dest_file); + + if (conf::dry()) + return; + + op::create_directories(cx, dest_file.parent_path()); + + auto p = process() + .binary(tools::sevenz::binary()) + .arg("a") + .arg(dest_file) + .arg("-r") + .arg("-mx=5") + .arg(src_glob); + + for (auto&& i : ignore) + p.arg("-xr!", i, process::nospace); + + p.run(); + p.join(); +} + +void archive_from_files( + const context& cx, + const std::vector& files, const fs::path& files_root, + const fs::path& dest_file) +{ + cx.trace(context::fs, + "archiving {} files rooted in {} into {}", + files.size(), files_root, dest_file); + + if (conf::dry()) + return; + + std::string list_file_text; + std::error_code ec; + + for (auto&& f : files) + { + fs::path rf = fs::relative(f, files_root, ec); + + if (ec) + { + cx.bail_out(context::fs, + "file {} is not in root {}", f, files_root); + } + + list_file_text += path_to_utf8(rf) + "\n"; + } + + const auto list_file = make_temp_file(); + guard g([&] + { + if (fs::exists(list_file)) + { + std::error_code ec; + fs::remove(list_file, ec); + } + }); + + op::write_text_file(gcx(), encodings::utf8, list_file, list_file_text); + op::create_directories(cx, dest_file.parent_path()); + + auto p = process() + .binary(tools::sevenz::binary()) + .arg("a") + .arg(dest_file) + .arg("@", list_file, process::nospace) + .cwd(files_root); + + p.run(); + p.join(); +} + void do_touch(const context& cx, const fs::path& p) { diff --git a/src/op.h b/src/op.h index 991fef2..a29c638 100644 --- a/src/op.h +++ b/src/op.h @@ -62,4 +62,14 @@ void write_text_file( const context& cx, encodings e, const fs::path& p, std::string_view utf8, flags f=noflags); +void archive_from_glob( + const context& cx, + const fs::path& src_glob, const fs::path& dest_file, + const std::vector& ignore); + +void archive_from_files( + const context& cx, + const std::vector& files, const fs::path& files_root, + const fs::path& dest_file); + } // namespace diff --git a/src/tasks/tasks.h b/src/tasks/tasks.h index 3b21d97..a2f93b6 100644 --- a/src/tasks/tasks.h +++ b/src/tasks/tasks.h @@ -261,6 +261,7 @@ public: static bool prebuilt(); static fs::path source_path(); + static fs::path super_path(); bool is_super() const override; @@ -274,8 +275,6 @@ private: void initialize_super(const fs::path& super_root); fs::path this_source_path() const; - - static fs::path super_path(); }; diff --git a/vs/props.props b/vs/props.props index 31f266e..046c06c 100644 --- a/vs/props.props +++ b/vs/props.props @@ -25,7 +25,7 @@ ..\third-party\lib;%(AdditionalLibraryDirectories) - Ws2_32.lib;Crypt32.lib;Wldap32.lib;Shlwapi.lib;DbgHelp.lib;%(AdditionalDependencies) + Ws2_32.lib;Crypt32.lib;Wldap32.lib;Shlwapi.lib;DbgHelp.lib;Version.lib;%(AdditionalDependencies)