diff --git a/mob.ini b/mob.ini index 973afff..9421299 100644 --- a/mob.ini +++ b/mob.ini @@ -19,10 +19,11 @@ mo_branch = master no_pull = false ignore_ts = false revert_ts = false + git_url_prefix = https://github.com/ -git_shallow = true -git_username = -git_email = +git_shallow = true +git_username = +git_email = set_origin_remote = false remote_org = @@ -36,6 +37,12 @@ git_shallow = false [usvfs:task] git_shallow = false +[translations:task] +enabled = false + +[installer:task] +enabled = false + [tools] sevenz = 7z.exe jom = jom.exe @@ -50,15 +57,19 @@ vswhere = vswhere.exe nasm = nasm.exe tx = tx.py37-x64.exe lrelease = lrelease.exe +iscc = ISCC.exe vcvars = [transifex] -enabled = true -key = -team = mod-organizer-2-team -project = mod-organizer-2 -url = https://www.transifex.com -minimum = 60 +enabled = true +key = +team = mod-organizer-2-team +project = mod-organizer-2 +url = https://www.transifex.com +minimum = 60 +force = false +configure = true +pull = true [prebuilt] boost = true @@ -103,25 +114,27 @@ ss_paper_mono_6788 = 2.1 ss_dark_mode_1809_6788 = 2.0 [paths] -third_party = -prefix = -cache = -patches = -licenses = -build = -install = -install_bin = -install_libs = -install_pdbs = -install_dlls = -install_loot = -install_plugins = -install_stylesheets = -install_licenses = -install_pythoncore = -vs = -qt_install = -qt_bin = -pf_x86 = -pf_x64 = -temp_dir = +third_party = +prefix = +cache = +patches = +licenses = +build = +install = +install_bin = +install_installer = +install_libs = +install_pdbs = +install_dlls = +install_loot = +install_plugins = +install_stylesheets = +install_licenses = +install_pythoncore = +install_translations = +vs = +qt_install = +qt_bin = +pf_x86 = +pf_x64 = +temp_dir = diff --git a/src/commands.cpp b/src/commands.cpp index 1f95208..87ceeab 100644 --- a/src/commands.cpp +++ b/src/commands.cpp @@ -326,7 +326,7 @@ int help_command::do_run() "be loaded, but the one in the current directory will override the\n" "the other. Additional INIs can be specified with --ini, those will\n" "be loaded after the two mentioned above. Use --no-default-inis to\n" - "only disable auto detection and only use --ini."); + "disable auto detection and only use --ini."); return 0; } @@ -1357,21 +1357,29 @@ clipp::group tx_command::do_group() "get" % (clipp::command("get").set(mode_, modes::get), - (clipp::option("-m", "--minimum") - & clipp::value("PERCENT") >> min_) - % "minimum translation threshold to download [0-100]", - (clipp::option("-k", "--key") & clipp::value("APIKEY") >> key_) % "API key", - (clipp::option("-f", "--force").set(force_) - % "don't check timestamps, re-download all translation files"), + (clipp::option("-t", "--team") + & clipp::value("TEAM") >> team_) + % "team name", + + (clipp::option("-p", "--project") + & clipp::value("PROJECT") >> project_) + % "project name", (clipp::option("-u", "--url") & clipp::value("URL") >> url_) % "project URL", + (clipp::option("-m", "--minimum") + & clipp::value("PERCENT").set(min_)) + % "minimum translation threshold to download [0-100]", + + (clipp::option("-f", "--force").call([&]{ force_ = true; })) + % "don't check timestamps, re-download all translation files", + (clipp::value("path") >> path_) % "path that will contain the .tx directory" ) @@ -1390,6 +1398,29 @@ clipp::group tx_command::do_group() ); } +void tx_command::convert_cl_to_conf() +{ + command::convert_cl_to_conf(); + + if (!key_.empty()) + common.options.push_back("transifex/key=" + key_); + + if (!team_.empty()) + common.options.push_back("transifex/team=" + team_); + + if (!project_.empty()) + common.options.push_back("transifex/project=" + project_); + + if (!url_.empty()) + common.options.push_back("transifex/url=" + url_); + + if (min_ >= 0) + common.options.push_back("transifex/minimum=" + std::to_string(min_)); + + if (force_) + common.options.push_back("transifex/force=" + std::to_string(*force_)); +} + int tx_command::do_run() { switch (mode_) @@ -1414,8 +1445,7 @@ int tx_command::do_run() std::string tx_command::do_doc() { return - "Values for --key, --minimum and --url will be taken from the INI\n" - "file if not specified.\n" + "Some values will be taken from the INI file if not specified.\n" "\n" "Commands:\n" "get\n" @@ -1430,40 +1460,20 @@ std::string tx_command::do_doc() void tx_command::do_get() { - if (min_ < 0) - { - const auto s = conf::get_global("transifex", "minimum"); + const url u = + conf::get_global("transifex", "url") + "/" + + conf::get_global("transifex", "team") + "/" + + conf::get_global("transifex", "project"); - try - { - min_ = std::stoi(s); - } - catch(std::exception&) - { - gcx().bail_out(context::generic, - "bad transifex minimum percentage '{}'", s); - } - } + const std::string key = conf::get_global("transifex", "key"); - if (key_.empty()) - key_ = conf::get_global("transifex", "key"); - - if (url_.empty()) - { - url_ = - conf::get_global("transifex", "url") + "/" + - conf::get_global("transifex", "team") + "/" + - conf::get_global("transifex", "project"); - } - - if (key_.empty() && !this_env::get_opt("TX_TOKEN")) + if (key.empty() && !this_env::get_opt("TX_TOKEN")) { u8cout << "(no key was in the INI, --key wasn't given and TX_TOKEN env\n" "variable doesn't exist, this will probably fail)\n\n"; } - context cxcopy = gcx(); u8cout << "initializing\n"; @@ -1475,17 +1485,17 @@ void tx_command::do_get() transifex(transifex::config) .stdout_level(context::level::info) .root(path_) - .api_key(key_) - .url(url_) + .api_key(key) + .url(u) .run(cxcopy); u8cout << "pulling\n"; transifex(transifex::pull) .stdout_level(context::level::info) .root(path_) - .api_key(key_) - .minimum(min_) - .force(force_) + .api_key(key) + .minimum(conf::get_global_int("transifex", "minimum")) + .force(conf::get_global_bool("transifex", "force")) .run(cxcopy); } @@ -1495,97 +1505,28 @@ void tx_command::do_build() if (fs::exists(root / ".tx") && fs::exists(root / "translations")) root = root / "translations"; + translations::projects ps(root); + fs::path dest = dest_; op::create_directories(gcx(), dest, op::unsafe); - std::set warned; + for (auto&& w : ps.warnings()) + u8cerr << w << "\n"; - for (auto e : fs::directory_iterator(root)) + thread_pool tp; + + for (auto& p : ps.get()) { - if (!e.is_directory()) - continue; - - const auto dir = path_to_utf8(e.path().filename()); - const auto dir_cs = split(dir, "."); - if (dir_cs.size() != 2) + for (auto& lg : p.langs) { - u8cerr << "bad directory name '" << dir << "'; skipping\n"; - continue; - } - - const auto project = trim_copy(dir_cs[1]); - if (project.empty()) - { - u8cerr << "bad directory name '" << dir << "'; skipping\n"; - continue; - } - - u8cout << project << "\n"; - - const bool is_gamebryo_plugin = [&] - { - auto tasks = find_tasks(project); - if (tasks.empty()) + tp.add([&, cxcopy=gcx()]() mutable { - u8cerr - << "directory '" << dir << "' was parsed as project " - << "'" << project << "', but there's no task with this " - << "name\n"; - - return false; - } - - const task& t = *tasks[0]; - - if (!t.is_super()) - return false; - - const auto& mo_task = static_cast(t); - return mo_task.is_gamebryo_plugin(); - }(); - - - const fs::path gamebryo_dir = - conf::get_global("transifex", "project") + "." + - "game_gamebryo"; - - - for (auto f : fs::directory_iterator(e.path())) - { - if (!f.is_regular_file()) - continue; - - lrelease lr; - lr - .project(project) - .add_source(f.path()) - .out(dest_); - - if (is_gamebryo_plugin) - { - const auto gb_f = root / gamebryo_dir / f.path().filename(); - - if (fs::exists(gb_f)) - { - lr.add_source(gb_f); - } - else - { - if (!warned.contains(gb_f)) - { - u8cerr - << "this is a gamebryo plugin but there is no " - << "'" << path_to_utf8(gb_f) << "'; " - << "the .qm file will be missing some " - << "translations (will only warn once)\n"; - - warned.insert(gb_f); - } - } - } - - context cxcopy = gcx(); - lr.run(cxcopy); + lrelease() + .project(p.name) + .sources(lg.ts_files) + .out(dest) + .run(cxcopy); + }); } } } diff --git a/src/commands.h b/src/commands.h index 123f581..7f42231 100644 --- a/src/commands.h +++ b/src/commands.h @@ -287,6 +287,7 @@ public: protected: clipp::group do_group() override; + void convert_cl_to_conf() override; int do_run() override; std::string do_doc() override; @@ -299,10 +300,9 @@ private: }; modes mode_ = modes::none; + std::string key_, team_, project_, url_; int min_ = -1; - std::string key_; - std::string url_; - bool force_ = false; + std::optional force_; std::string path_; std::string dest_; diff --git a/src/conf.cpp b/src/conf.cpp index d1ef7a3..868f1da 100644 --- a/src/conf.cpp +++ b/src/conf.cpp @@ -19,6 +19,11 @@ std::string master_ini_filename() return "mob.ini"; } +bool bool_from_string(const std::string& s) +{ + return (s == "true" || s == "yes" || s == "1"); +} + std::string conf::get_global(const std::string& section, const std::string& key) { @@ -66,6 +71,26 @@ void conf::set_global( kitor->second = value; } +int conf::get_global_int(const std::string& section, const std::string& key) +{ + const auto s = conf::get_global(section, key); + + try + { + return std::stoi(s); + } + catch(std::exception&) + { + gcx().bail_out(context::conf, "bad int for {}/{}", section, key); + } +} + +bool conf::get_global_bool(const std::string& section, const std::string& key) +{ + const auto s = conf::get_global(section, key); + return bool_from_string(s); +} + void conf::add_global( const std::string& section, const std::string& key, const std::string& value) @@ -141,7 +166,7 @@ void conf::set_for_task( bool conf::prebuilt_by_name(const std::string& task) { const std::string s = get_global("prebuilt", task); - return (s == "true" || s == "yes" || s == "1"); + return bool_from_string(s); } fs::path conf::path_by_name(const std::string& name) @@ -167,7 +192,7 @@ std::string conf::global_by_name(const std::string& name) bool conf::bool_global_by_name(const std::string& name) { const std::string s = global_by_name(name); - return (s == "true" || s == "yes" || s == "1"); + return bool_from_string(s); } std::string conf::task_option_by_name( @@ -180,7 +205,7 @@ bool conf::bool_task_option_by_name( const std::vector& task_names, const std::string& name) { const std::string s = task_option_by_name(task_names, name); - return (s == "true" || s == "yes" || s == "1"); + return bool_from_string(s); } void conf::set_output_log_level(const std::string& s) @@ -224,7 +249,7 @@ void conf::set_file_log_level(const std::string& s) void conf::set_dry(const std::string& s) { - dry_ = (s == "true" || s == "yes" || s == "1"); + dry_ = bool_from_string(s); } @@ -662,7 +687,7 @@ void ini_error(const fs::path& ini, std::size_t line, const std::string& what) { gcx().bail_out(context::conf, "{}:{}: {}", - ini.filename(), (line + 1), what); + path_to_utf8(ini), (line + 1), what); } std::vector read_ini(const fs::path& ini) @@ -680,9 +705,6 @@ std::vector read_ini(const fs::path& ini) if (!in) break; - if (line.empty() || line[0] == '#' || line[0] == ';') - continue; - lines.push_back(std::move(line)); } @@ -706,6 +728,12 @@ void parse_section( const auto& line = lines[i]; + if (line.empty() || line[0] == '#' || line[0] == ';') + { + ++i; + continue; + } + const auto sep = line.find("="); if (sep == std::string::npos) ini_error(ini, i, "bad line '" + line + "'"); @@ -758,6 +786,11 @@ void parse_ini(const fs::path& ini, bool add) break; const auto& line = lines[i]; + if (line.empty() || line[0] == '#' || line[0] == ';') + { + ++i; + continue; + } if (line.starts_with("[") && line.ends_with("]")) { @@ -963,6 +996,41 @@ std::vector find_inis( return map(v, [&](auto&& p){ return p.second; }); } +fs::path find_iscc() +{ + const auto tasks = find_tasks("installer"); + MOB_ASSERT(tasks.size() == 1); + + if (!tasks[0]->enabled()) + return {}; + + const auto iscc = conf::tool_by_name("iscc"); + if (iscc.is_absolute()) + { + if (!fs::exists(iscc)) + { + gcx().bail_out(context::conf, + "{} doesn't exist (from ini, absolute path)", iscc); + } + + return iscc; + } + + for (int v : {5, 6, 7, 8}) + { + const fs::path inno = ::fmt::format("inno setup {}", v); + + for (fs::path pf : {paths::pf_x86(), paths::pf_x64()}) + { + fs::path p = pf / inno / iscc; + if (fs::exists(p)) + return fs::canonical(fs::absolute(p)); + } + } + + gcx().bail_out(context::conf, "can't find {} anywhere", iscc); +} + void init_options( const std::vector& inis, const std::vector& opts) { @@ -1037,19 +1105,22 @@ void init_options( find_vcvars(); validate_qt(); - if (!paths::prefix().empty()) - make_canonical_path("prefix", fs::current_path(), ""); + this_env::append_to_path(conf::path_by_name("qt_bin")); - make_canonical_path("cache", paths::prefix(), "downloads"); - make_canonical_path("build", paths::prefix(), "build"); - make_canonical_path("install", paths::prefix(), "install"); - make_canonical_path("install_bin", paths::install(), "bin"); - make_canonical_path("install_libs", paths::install(), "libs"); - make_canonical_path("install_pdbs", paths::install(), "pdb"); - make_canonical_path("install_dlls", paths::install_bin(), "dlls"); - make_canonical_path("install_loot", paths::install_bin(), "loot"); - make_canonical_path("install_plugins", paths::install_bin(), "plugins"); - make_canonical_path("install_licenses", paths::install_bin(), "licenses"); + if (!paths::prefix().empty()) + make_canonical_path("prefix", fs::current_path(), ""); + + make_canonical_path("cache", paths::prefix(), "downloads"); + make_canonical_path("build", paths::prefix(), "build"); + make_canonical_path("install", paths::prefix(), "install"); + make_canonical_path("install_installer", paths::install(), "installer"); + make_canonical_path("install_bin", paths::install(), "bin"); + make_canonical_path("install_libs", paths::install(), "libs"); + make_canonical_path("install_pdbs", paths::install(), "pdb"); + make_canonical_path("install_dlls", paths::install_bin(), "dlls"); + make_canonical_path("install_loot", paths::install_bin(), "loot"); + make_canonical_path("install_plugins", paths::install_bin(), "plugins"); + make_canonical_path("install_licenses", paths::install_bin(), "licenses"); make_canonical_path( "install_pythoncore", @@ -1058,6 +1129,12 @@ void init_options( make_canonical_path( "install_stylesheets", paths::install_bin(), "stylesheets"); + + make_canonical_path( + "install_translations", + paths::install_bin(), "translations"); + + conf::set_global("tools", "iscc", path_to_utf8(find_iscc())); } bool verify_options() diff --git a/src/conf.h b/src/conf.h index 902dec4..4015f17 100644 --- a/src/conf.h +++ b/src/conf.h @@ -9,6 +9,13 @@ public: static std::string get_global( const std::string& section, const std::string& key); + static int get_global_int( + const std::string& section, const std::string& key); + + static bool get_global_bool( + const std::string& section, const std::string& key); + + static void set_global( const std::string& section, const std::string& key, const std::string& value); @@ -98,6 +105,7 @@ struct paths VALUE(build); VALUE(install); + VALUE(install_installer); VALUE(install_bin); VALUE(install_libs); VALUE(install_pdbs); @@ -108,6 +116,7 @@ struct paths VALUE(install_stylesheets); VALUE(install_licenses); VALUE(install_pythoncore); + VALUE(install_translations); VALUE(pf_x86); VALUE(pf_x64); diff --git a/src/env.cpp b/src/env.cpp index 04b9dd0..cc44d60 100644 --- a/src/env.cpp +++ b/src/env.cpp @@ -449,6 +449,12 @@ void this_env::prepend_to_path(const fs::path& p) set("PATH", path_to_utf8(p) + ";", env::prepend); } +void this_env::append_to_path(const fs::path& p) +{ + gcx().trace(context::generic, "appending to PATH: {}", p); + set("PATH", ";" + path_to_utf8(p), env::append); +} + std::string this_env::get(const std::string& name) { auto v = get_impl(name); diff --git a/src/env.h b/src/env.h index f7e2b13..01d4fed 100644 --- a/src/env.h +++ b/src/env.h @@ -70,6 +70,7 @@ struct this_env env::flags f=env::replace); static void prepend_to_path(const fs::path& p); + static void append_to_path(const fs::path& p); static env get(); diff --git a/src/main.cpp b/src/main.cpp index 9a12e39..f201bd4 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -136,6 +136,9 @@ void add_tasks() .add_task({"modorganizer-bsapacker", "bsa_packer"}) .add_task("modorganizer-preview_bsa"); + // the gamebryo flag must be set for all game plugins that inherit from + // the gamebryo classes; this will merge the .ts file from gamebryo with + // the one from the specific plugin add_task(true) .add_task("modorganizer-game_oblivion", mo::gamebryo) .add_task("modorganizer-game_fallout3", mo::gamebryo) @@ -163,7 +166,8 @@ void add_tasks() .add_task("modorganizer-installer_fomod_csharp") .add_task("modorganizer-installer_ncc") .add_task("modorganizer-bsa_extractor") - .add_task("modorganizer-plugin_python"); + .add_task("modorganizer-plugin_python") + .add_task(); add_task(true) .add_task({"modorganizer-tool_configurator", "pycfg"}) @@ -175,6 +179,8 @@ void add_tasks() .add_task({"modorganizer-form43_checker", "form43checker"}) .add_task({"modorganizer-preview_dds", "ddspreview"}) .add_task({"modorganizer", "organizer"}); + + add_task(); } diff --git a/src/op.cpp b/src/op.cpp index 5131b52..f9f5ec1 100644 --- a/src/op.cpp +++ b/src/op.cpp @@ -84,12 +84,46 @@ void delete_file(const context& cx, const fs::path& p, flags f) } if (fs::exists(p) && !fs::is_regular_file(p)) - cx.bail_out(context::fs, "can't delete {}, not a file", p); + { + if (f & optional) + cx.warning(context::fs, "can't delete {}, not a file", p); + else + cx.bail_out(context::fs, "can't delete {}, not a file", p); + + return; + } if (!conf::dry()) do_delete_file(cx, p); } +void delete_file_glob(const context& cx, const fs::path& glob, flags f) +{ + cx.trace(context::fs, "deleting glob {}", glob); + + const auto parent = glob.parent_path(); + const auto wildcard = glob.filename().native(); + + if (!fs::exists(parent)) + return; + + for (auto&& e : fs::directory_iterator(parent)) + { + const auto p = e.path(); + const auto name = p.filename().native(); + + if (!PathMatchSpecW(name.c_str(), wildcard.c_str())) + { + cx.trace(context::fs, + "{} did not match {}; skipping", name, wildcard); + + continue; + } + + delete_file(cx, p, f); + } +} + void remove_readonly(const context& cx, const fs::path& first) { cx.trace(context::fs, "removing read-only from {}", first); diff --git a/src/op.h b/src/op.h index 95213aa..10f4552 100644 --- a/src/op.h +++ b/src/op.h @@ -30,6 +30,9 @@ void delete_directory( void delete_file( const context& cx, const fs::path& p, flags f=noflags); +void delete_file_glob( + const context& cx, const fs::path& glob, flags f=noflags); + void remove_readonly( const context& cx, const fs::path& first); diff --git a/src/process.cpp b/src/process.cpp index 35001b9..8d76cfc 100644 --- a/src/process.cpp +++ b/src/process.cpp @@ -706,6 +706,8 @@ void process::on_completed() code_ = 0xffff; } + read_pipes(false); + for (;;) { read_pipes(true); diff --git a/src/process.h b/src/process.h index ba3456d..9be4693 100644 --- a/src/process.h +++ b/src/process.h @@ -137,14 +137,17 @@ private: } } - if (line.empty() && finished) + if (line.empty()) { - line = { - reinterpret_cast(bytes.data() + byte_offset), - size - byte_offset - }; + if (finished) + { + line = { + reinterpret_cast(bytes.data() + byte_offset), + size - byte_offset + }; - byte_offset = bytes.size(); + byte_offset = bytes.size(); + } } else { diff --git a/src/tasks/installer.cpp b/src/tasks/installer.cpp new file mode 100644 index 0000000..51ea489 --- /dev/null +++ b/src/tasks/installer.cpp @@ -0,0 +1,60 @@ +#include "pch.h" +#include "tasks.h" + +namespace mob +{ + +installer::installer() + : basic_task("installer", "modorganizer-Installer") +{ +} + +bool installer::prebuilt() +{ + return false; +} + +std::string installer::version() +{ + return {}; +} + +fs::path installer::source_path() +{ + return modorganizer::super_path() / "installer"; +} + +void installer::do_clean(clean c) +{ + instrument([&] + { + if (is_set(c, clean::reclone)) + git::delete_directory(cx(), source_path()); + + if (is_set(c, clean::rebuild)) + op::delete_directory(cx(), paths::install_installer()); + }); +} + +void installer::do_fetch() +{ + const std::string repo = "modorganizer-Installer"; + + instrument([&] + { + run_tool(task_conf().make_git() + .url(task_conf().make_git_url(task_conf().mo_org(), repo)) + .branch(task_conf().mo_branch()) + .root(source_path())); + }); +} + +void installer::do_build_and_install() +{ + instrument([&] + { + run_tool(source_path() / "dist" / "MO2-Installer.iss"); + }); +} + +} // namespace diff --git a/src/tasks/tasks.h b/src/tasks/tasks.h index fc3b063..78b0f4f 100644 --- a/src/tasks/tasks.h +++ b/src/tasks/tasks.h @@ -72,7 +72,6 @@ public: static std::string version(); static bool prebuilt(); - static fs::path source_path(); protected: @@ -88,7 +87,6 @@ public: static std::string version(); static bool prebuilt(); - static fs::path source_path(); protected: @@ -107,7 +105,6 @@ public: static std::string version(); static bool prebuilt(); - static fs::path source_path(); protected: @@ -126,7 +123,6 @@ public: static std::string version(); static bool prebuilt(); - static fs::path source_path(); protected: @@ -152,7 +148,6 @@ public: static std::string version(); static bool prebuilt(); - static fs::path source_path(); protected: @@ -166,6 +161,22 @@ private: }; +class installer : public basic_task +{ +public: + installer(); + + static bool prebuilt(); + static std::string version(); + static fs::path source_path(); + +protected: + void do_clean(clean c) override; + void do_fetch() override; + void do_build_and_install() override; +}; + + class licenses : public basic_task { public: @@ -173,7 +184,6 @@ public: static std::string version(); static bool prebuilt(); - static fs::path source_path(); protected: @@ -188,7 +198,6 @@ public: static std::string version(); static bool prebuilt(); - static fs::path source_path(); protected: @@ -228,7 +237,6 @@ public: static std::string version(); static std::string hash(); static bool prebuilt(); - static fs::path source_path(); protected: @@ -249,7 +257,6 @@ public: static std::string version(); static bool prebuilt(); - static fs::path source_path(); protected: @@ -323,7 +330,6 @@ public: static std::string version(); static bool prebuilt(); - static fs::path source_path(); protected: @@ -343,7 +349,6 @@ public: static std::string version(); static bool prebuilt(); - static fs::path source_path(); protected: @@ -515,7 +520,6 @@ public: static std::string version(); static bool prebuilt(); - static fs::path source_path(); protected: @@ -539,7 +543,6 @@ public: static std::string version(); static bool prebuilt(); - static fs::path source_path(); protected: @@ -587,6 +590,63 @@ private: }; +class translations : public basic_task +{ +public: + class projects + { + public: + struct lang + { + std::string name; + std::vector ts_files; + + lang(std::string n); + }; + + struct project + { + std::string name; + std::vector langs; + + project(std::string n); + }; + + + projects(fs::path root); + + const std::vector& get() const; + const std::vector& warnings() const; + + private: + const fs::path root_; + std::vector projects_; + std::vector warnings_; + std::set warned_; + + void create(); + bool is_gamebryo_plugin( + const std::string& dir, const std::string& project); + void handle_project_dir(const fs::path& dir); + + lang handle_ts_file( + bool gamebryo, const std::string& project_name, const fs::path& f); + }; + + + translations(); + + static bool prebuilt(); + static std::string version(); + static fs::path source_path(); + +protected: + void do_clean(clean c) override; + void do_fetch() override; + void do_build_and_install() override; +}; + + class usvfs : public basic_task { public: @@ -594,7 +654,6 @@ public: static std::string version(); static bool prebuilt(); - static fs::path source_path(); protected: @@ -627,7 +686,6 @@ public: static std::string version(); static bool prebuilt(); - static fs::path source_path(); protected: diff --git a/src/tasks/translations.cpp b/src/tasks/translations.cpp new file mode 100644 index 0000000..d0682c0 --- /dev/null +++ b/src/tasks/translations.cpp @@ -0,0 +1,263 @@ +#include "pch.h" +#include "tasks.h" + +namespace mob +{ + +translations::projects::lang::lang(std::string n) + : name(std::move(n)) +{ +} + +translations::projects::project::project(std::string n) + : name(std::move(n)) +{ +} + + +translations::projects::projects(fs::path root) + : root_(std::move(root)) +{ + create(); +} + +const std::vector& +translations::projects::get() const +{ + return projects_; +} + +const std::vector& translations::projects::warnings() const +{ + return warnings_; +} + +void translations::projects::create() +{ + for (auto e : fs::directory_iterator(root_)) + { + if (!e.is_directory()) + continue; + + handle_project_dir(e.path()); + } +} + +bool translations::projects::is_gamebryo_plugin( + const std::string& dir, const std::string& project) +{ + auto tasks = find_tasks(project); + if (tasks.empty()) + { + warnings_.push_back(::fmt::format( + "directory '{}' was parsed as project '{}', but there's " + "no task with this name", dir, project)); + + return false; + } + + const task& t = *tasks[0]; + + if (!t.is_super()) + return false; + + const auto& mo_task = static_cast(t); + return mo_task.is_gamebryo_plugin(); +} + +void translations::projects::handle_project_dir(const fs::path& dir) +{ + const auto dir_name = path_to_utf8(dir.filename()); + const auto dir_cs = split(dir_name, "."); + + if (dir_cs.size() != 2) + { + warnings_.push_back(::fmt::format( + "bad directory name '{}'; skipping", dir_name)); + + return; + } + + const auto project_name = trim_copy(dir_cs[1]); + if (project_name.empty()) + { + warnings_.push_back(::fmt::format( + "bad directory name '{}', skipping", dir_name)); + + return; + } + + project p(project_name); + + const bool gamebryo = is_gamebryo_plugin(dir_name, project_name); + + for (auto f : fs::directory_iterator(dir)) + { + if (!f.is_regular_file()) + continue; + + p.langs.push_back(handle_ts_file(gamebryo, project_name, f.path())); + } + + projects_.push_back(p); +} + +translations::projects::lang translations::projects::handle_ts_file( + bool gamebryo, const std::string& project_name, const fs::path& f) +{ + lang lg(path_to_utf8(f.stem())); + + lg.ts_files.push_back(f); + + if (gamebryo) + { + const fs::path gamebryo_dir = + conf::get_global("transifex", "project") + "." + + "game_gamebryo"; + + const auto gb_f = root_ / gamebryo_dir / f.filename(); + + if (fs::exists(gb_f)) + { + lg.ts_files.push_back(gb_f); + } + else + { + if (!warned_.contains(gb_f)) + { + warned_.insert(gb_f); + + warnings_.push_back(::fmt::format( + "{} is a gamebryo plugin but there is no '{}'; the " + ".qm file will be missing some translations (will " + "only warn once)", + project_name, path_to_utf8(gb_f))); + } + } + } + + return lg; +} + + +translations::translations() + : basic_task("translations") +{ +} + +bool translations::prebuilt() +{ + return false; +} + +std::string translations::version() +{ + return {}; +} + +fs::path translations::source_path() +{ + return paths::build() / "transifex-translations"; +} + +void translations::do_clean(clean c) +{ + instrument([&] + { + if (is_set(c, clean::redownload)) + op::delete_directory(cx(), source_path(), op::optional); + + if (is_set(c, clean::rebuild)) + { + op::delete_file_glob( + cx(), paths::install_translations() / "*.qm", op::optional); + } + }); +} + +void translations::do_fetch() +{ + instrument([&] + { + const url u = + conf::get_global("transifex", "url") + "/" + + conf::get_global("transifex", "team") + "/" + + conf::get_global("transifex", "project"); + + const std::string key = conf::get_global("transifex", "key"); + + if (key.empty() && !this_env::get_opt("TX_TOKEN")) + { + cx().warning(context::generic, + "no key was in the INI and the TX_TOKEN env variable doesn't " + "exist, this will probably fail"); + } + + cx().debug(context::generic, "init tx"); + run_tool(transifex(transifex::init) + .root(source_path())); + + if (conf::get_global_bool("transifex", "configure")) + { + cx().debug(context::generic, "configuring"); + run_tool(transifex(transifex::config) + .root(source_path()) + .api_key(key) + .url(u)); + } + else + { + cx().trace(context::generic, "skipping configuring"); + } + + if (conf::get_global_bool("transifex", "pull")) + { + cx().debug(context::generic, "pulling"); + run_tool(transifex(transifex::pull) + .root(source_path()) + .api_key(key) + .minimum(conf::get_global_int("transifex", "minimum")) + .force(conf::get_global_bool("transifex", "force"))); + } + else + { + cx().trace(context::generic, "skipping pulling"); + } + }); +} + +void translations::do_build_and_install() +{ + instrument([&] + { + const auto root = source_path() / "translations"; + const auto dest = paths::install_translations(); + const projects ps(root); + + op::create_directories(cx(), dest); + + for (auto&& w : ps.warnings()) + cx().warning(context::generic, "{}", w); + + thread_pool tp; + + for (auto& p : ps.get()) + { + for (auto& lg : p.langs) + { + tp.add([&] + { + threaded_run(lg.name + "." + p.name, [&] + { + run_tool(lrelease() + .project(p.name) + .sources(lg.ts_files) + .out(dest)); + }); + }); + } + } + }); +} + +} // namespace diff --git a/src/tools/tools.cpp b/src/tools/tools.cpp index 66fe18f..a3f851e 100644 --- a/src/tools/tools.cpp +++ b/src/tools/tools.cpp @@ -392,7 +392,7 @@ lrelease::lrelease() fs::path lrelease::binary() { - return conf::path_by_name("qt_bin") / conf::tool_by_name("lrelease"); + return conf::tool_by_name("lrelease"); } lrelease& lrelease::project(const std::string& name) @@ -407,6 +407,12 @@ lrelease& lrelease::add_source(const fs::path& ts_file) return *this; } +lrelease& lrelease::sources(const std::vector& v) +{ + sources_ = v; + return *this; +} + lrelease& lrelease::out(const fs::path& dir) { out_ = dir; @@ -453,4 +459,33 @@ void lrelease::do_run() execute_and_join(); } + +iscc::iscc(fs::path iss) + : basic_process_runner("iscc"), iss_(std::move(iss)) +{ +} + +fs::path iscc::binary() +{ + return conf::tool_by_name("iscc"); +} + +iscc& iscc::iss(const fs::path& p) +{ + iss_ = p; + return *this; +} + +void iscc::do_run() +{ + if (iss_.empty()) + cx().bail_out(context::generic, "iscc missing iss file"); + + process_ + .binary(binary()) + .arg(iss_); + + execute_and_join(); +} + } // namespace diff --git a/src/tools/tools.h b/src/tools/tools.h index 098b16f..ef038f5 100644 --- a/src/tools/tools.h +++ b/src/tools/tools.h @@ -608,6 +608,7 @@ public: lrelease& project(const std::string& name); lrelease& add_source(const fs::path& ts_file); + lrelease& sources(const std::vector& v); lrelease& out(const fs::path& dir); fs::path qm_file() const; @@ -621,4 +622,21 @@ private: fs::path out_; }; + +class iscc : public basic_process_runner +{ +public: + static fs::path binary(); + + iscc(fs::path iss = {}); + + iscc& iss(const fs::path& p); + +protected: + void do_run() override; + +private: + fs::path iss_; +}; + } // namespace diff --git a/src/utility.cpp b/src/utility.cpp index 5d81e16..8d9535f 100644 --- a/src/utility.cpp +++ b/src/utility.cpp @@ -918,4 +918,63 @@ std::string path_to_utf8(fs::path p) return utf16_to_utf8(p.native()); } + + +thread_pool::thread_pool(std::size_t count) + : count_(std::max(1, count)) +{ + for (std::size_t i=0; i()); +} + +thread_pool::~thread_pool() +{ + join(); +} + +void thread_pool::join() +{ + for (auto&& t : threads_) + { + if (t->thread.joinable()) + t->thread.join(); + } +} + +void thread_pool::add(fun thread_fun) +{ + for (;;) + { + if (try_add(thread_fun)) + break; + + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } +} + +bool thread_pool::try_add(fun thread_fun) +{ + for (auto& t : threads_) + { + if (!t->running) + { + if (t->thread.joinable()) + t->thread.join(); + + t->running = true; + t->thread_fun = thread_fun; + + t->thread = std::thread([&] + { + t->thread_fun(); + t->running = false; + }); + + return true; + } + } + + return false; +} + } // namespace diff --git a/src/utility.h b/src/utility.h index b0bdd59..e4528bd 100644 --- a/src/utility.h +++ b/src/utility.h @@ -588,4 +588,35 @@ auto map(const std::vector& v, F&& f) return out; } + +class thread_pool +{ +public: + typedef std::function fun; + + thread_pool(std::size_t count=std::thread::hardware_concurrency()); + ~thread_pool(); + + // non-copyable + thread_pool(const thread_pool&) = delete; + thread_pool& operator=(const thread_pool&) = delete; + + void add(fun f); + void join(); + +private: + struct thread_info + { + std::atomic running = false; + fun thread_fun; + std::thread thread; + }; + + + const std::size_t count_; + std::vector> threads_; + + bool try_add(fun thread_fun); +}; + } // namespace diff --git a/vs/mob.vcxproj b/vs/mob.vcxproj index 1f64e16..a26a9b4 100644 --- a/vs/mob.vcxproj +++ b/vs/mob.vcxproj @@ -76,6 +76,7 @@ + @@ -92,6 +93,7 @@ + diff --git a/vs/mob.vcxproj.filters b/vs/mob.vcxproj.filters index 7230f53..5e10200 100644 --- a/vs/mob.vcxproj.filters +++ b/vs/mob.vcxproj.filters @@ -141,6 +141,12 @@ src + + src\tasks + + + src\tasks +