From 518b20db0d0d75f4bc47572788958134c86f8072 Mon Sep 17 00:00:00 2001 From: isanae <14251494+isanae@users.noreply.github.com> Date: Mon, 13 Jul 2020 08:17:09 -0400 Subject: [PATCH] translations task --- mob.ini | 72 +++++----- src/commands.cpp | 88 ++++++------ src/commands.h | 6 +- src/conf.cpp | 53 +++++-- src/conf.h | 8 ++ src/main.cpp | 6 +- src/tasks/tasks.h | 16 +++ src/tasks/translations.cpp | 276 +++++++++++++++++++++++++++++++++++++ src/tools/tools.cpp | 6 + src/tools/tools.h | 1 + src/utility.cpp | 59 ++++++++ src/utility.h | 31 +++++ vs/mob.vcxproj | 1 + vs/mob.vcxproj.filters | 3 + 14 files changed, 543 insertions(+), 83 deletions(-) create mode 100644 src/tasks/translations.cpp diff --git a/mob.ini b/mob.ini index 50533d5..9421299 100644 --- a/mob.ini +++ b/mob.ini @@ -19,10 +19,11 @@ mo_branch = master no_pull = false ignore_ts = false revert_ts = false + git_url_prefix = https://github.com/ -git_shallow = true -git_username = -git_email = +git_shallow = true +git_username = +git_email = set_origin_remote = false remote_org = @@ -36,6 +37,9 @@ git_shallow = false [usvfs:task] git_shallow = false +[translations:task] +enabled = false + [installer:task] enabled = false @@ -57,12 +61,15 @@ iscc = ISCC.exe vcvars = [transifex] -enabled = true -key = -team = mod-organizer-2-team -project = mod-organizer-2 -url = https://www.transifex.com -minimum = 60 +enabled = true +key = +team = mod-organizer-2-team +project = mod-organizer-2 +url = https://www.transifex.com +minimum = 60 +force = false +configure = true +pull = true [prebuilt] boost = true @@ -107,26 +114,27 @@ ss_paper_mono_6788 = 2.1 ss_dark_mode_1809_6788 = 2.0 [paths] -third_party = -prefix = -cache = -patches = -licenses = -build = -install = -install_bin = -install_installer = -install_libs = -install_pdbs = -install_dlls = -install_loot = -install_plugins = -install_stylesheets = -install_licenses = -install_pythoncore = -vs = -qt_install = -qt_bin = -pf_x86 = -pf_x64 = -temp_dir = +third_party = +prefix = +cache = +patches = +licenses = +build = +install = +install_bin = +install_installer = +install_libs = +install_pdbs = +install_dlls = +install_loot = +install_plugins = +install_stylesheets = +install_licenses = +install_pythoncore = +install_translations = +vs = +qt_install = +qt_bin = +pf_x86 = +pf_x64 = +temp_dir = diff --git a/src/commands.cpp b/src/commands.cpp index edd9112..5569ef6 100644 --- a/src/commands.cpp +++ b/src/commands.cpp @@ -1357,21 +1357,29 @@ clipp::group tx_command::do_group() "get" % (clipp::command("get").set(mode_, modes::get), - (clipp::option("-m", "--minimum") - & clipp::value("PERCENT") >> min_) - % "minimum translation threshold to download [0-100]", - (clipp::option("-k", "--key") & clipp::value("APIKEY") >> key_) % "API key", - (clipp::option("-f", "--force").set(force_) - % "don't check timestamps, re-download all translation files"), + (clipp::option("-t", "--team") + & clipp::value("TEAM") >> team_) + % "team name", + + (clipp::option("-p", "--project") + & clipp::value("PROJECT") >> project_) + % "project name", (clipp::option("-u", "--url") & clipp::value("URL") >> url_) % "project URL", + (clipp::option("-m", "--minimum") + & clipp::value("PERCENT").set(min_)) + % "minimum translation threshold to download [0-100]", + + (clipp::option("-f", "--force").call([&]{ force_ = true; })) + % "don't check timestamps, re-download all translation files", + (clipp::value("path") >> path_) % "path that will contain the .tx directory" ) @@ -1390,6 +1398,29 @@ clipp::group tx_command::do_group() ); } +void tx_command::convert_cl_to_conf() +{ + command::convert_cl_to_conf(); + + if (!key_.empty()) + common.options.push_back("transifex/key=" + key_); + + if (!team_.empty()) + common.options.push_back("transifex/team=" + team_); + + if (!project_.empty()) + common.options.push_back("transifex/project=" + project_); + + if (!url_.empty()) + common.options.push_back("transifex/url=" + url_); + + if (min_ >= 0) + common.options.push_back("transifex/minimum=" + std::to_string(min_)); + + if (force_) + common.options.push_back("transifex/force=" + std::to_string(*force_)); +} + int tx_command::do_run() { switch (mode_) @@ -1414,8 +1445,7 @@ int tx_command::do_run() std::string tx_command::do_doc() { return - "Values for --key, --minimum and --url will be taken from the INI\n" - "file if not specified.\n" + "Some values will be taken from the INI file if not specified.\n" "\n" "Commands:\n" "get\n" @@ -1430,40 +1460,20 @@ std::string tx_command::do_doc() void tx_command::do_get() { - if (min_ < 0) - { - const auto s = conf::get_global("transifex", "minimum"); + const url u = + conf::get_global("transifex", "url") + "/" + + conf::get_global("transifex", "team") + "/" + + conf::get_global("transifex", "project"); - try - { - min_ = std::stoi(s); - } - catch(std::exception&) - { - gcx().bail_out(context::generic, - "bad transifex minimum percentage '{}'", s); - } - } + const std::string key = conf::get_global("transifex", "key"); - if (key_.empty()) - key_ = conf::get_global("transifex", "key"); - - if (url_.empty()) - { - url_ = - conf::get_global("transifex", "url") + "/" + - conf::get_global("transifex", "team") + "/" + - conf::get_global("transifex", "project"); - } - - if (key_.empty() && !this_env::get_opt("TX_TOKEN")) + if (key.empty() && !this_env::get_opt("TX_TOKEN")) { u8cout << "(no key was in the INI, --key wasn't given and TX_TOKEN env\n" "variable doesn't exist, this will probably fail)\n\n"; } - context cxcopy = gcx(); u8cout << "initializing\n"; @@ -1475,17 +1485,17 @@ void tx_command::do_get() transifex(transifex::config) .stdout_level(context::level::info) .root(path_) - .api_key(key_) - .url(url_) + .api_key(key) + .url(u) .run(cxcopy); u8cout << "pulling\n"; transifex(transifex::pull) .stdout_level(context::level::info) .root(path_) - .api_key(key_) - .minimum(min_) - .force(force_) + .api_key(key) + .minimum(conf::get_global_int("transifex", "minimum")) + .force(conf::get_global_bool("transifex", "force")) .run(cxcopy); } diff --git a/src/commands.h b/src/commands.h index 123f581..7f42231 100644 --- a/src/commands.h +++ b/src/commands.h @@ -287,6 +287,7 @@ public: protected: clipp::group do_group() override; + void convert_cl_to_conf() override; int do_run() override; std::string do_doc() override; @@ -299,10 +300,9 @@ private: }; modes mode_ = modes::none; + std::string key_, team_, project_, url_; int min_ = -1; - std::string key_; - std::string url_; - bool force_ = false; + std::optional force_; std::string path_; std::string dest_; diff --git a/src/conf.cpp b/src/conf.cpp index 45fa356..868f1da 100644 --- a/src/conf.cpp +++ b/src/conf.cpp @@ -19,6 +19,11 @@ std::string master_ini_filename() return "mob.ini"; } +bool bool_from_string(const std::string& s) +{ + return (s == "true" || s == "yes" || s == "1"); +} + std::string conf::get_global(const std::string& section, const std::string& key) { @@ -66,6 +71,26 @@ void conf::set_global( kitor->second = value; } +int conf::get_global_int(const std::string& section, const std::string& key) +{ + const auto s = conf::get_global(section, key); + + try + { + return std::stoi(s); + } + catch(std::exception&) + { + gcx().bail_out(context::conf, "bad int for {}/{}", section, key); + } +} + +bool conf::get_global_bool(const std::string& section, const std::string& key) +{ + const auto s = conf::get_global(section, key); + return bool_from_string(s); +} + void conf::add_global( const std::string& section, const std::string& key, const std::string& value) @@ -141,7 +166,7 @@ void conf::set_for_task( bool conf::prebuilt_by_name(const std::string& task) { const std::string s = get_global("prebuilt", task); - return (s == "true" || s == "yes" || s == "1"); + return bool_from_string(s); } fs::path conf::path_by_name(const std::string& name) @@ -167,7 +192,7 @@ std::string conf::global_by_name(const std::string& name) bool conf::bool_global_by_name(const std::string& name) { const std::string s = global_by_name(name); - return (s == "true" || s == "yes" || s == "1"); + return bool_from_string(s); } std::string conf::task_option_by_name( @@ -180,7 +205,7 @@ bool conf::bool_task_option_by_name( const std::vector& task_names, const std::string& name) { const std::string s = task_option_by_name(task_names, name); - return (s == "true" || s == "yes" || s == "1"); + return bool_from_string(s); } void conf::set_output_log_level(const std::string& s) @@ -224,7 +249,7 @@ void conf::set_file_log_level(const std::string& s) void conf::set_dry(const std::string& s) { - dry_ = (s == "true" || s == "yes" || s == "1"); + dry_ = bool_from_string(s); } @@ -662,7 +687,7 @@ void ini_error(const fs::path& ini, std::size_t line, const std::string& what) { gcx().bail_out(context::conf, "{}:{}: {}", - ini.filename(), (line + 1), what); + path_to_utf8(ini), (line + 1), what); } std::vector read_ini(const fs::path& ini) @@ -680,9 +705,6 @@ std::vector read_ini(const fs::path& ini) if (!in) break; - if (line.empty() || line[0] == '#' || line[0] == ';') - continue; - lines.push_back(std::move(line)); } @@ -706,6 +728,12 @@ void parse_section( const auto& line = lines[i]; + if (line.empty() || line[0] == '#' || line[0] == ';') + { + ++i; + continue; + } + const auto sep = line.find("="); if (sep == std::string::npos) ini_error(ini, i, "bad line '" + line + "'"); @@ -758,6 +786,11 @@ void parse_ini(const fs::path& ini, bool add) break; const auto& line = lines[i]; + if (line.empty() || line[0] == '#' || line[0] == ';') + { + ++i; + continue; + } if (line.starts_with("[") && line.ends_with("]")) { @@ -1097,6 +1130,10 @@ void init_options( "install_stylesheets", paths::install_bin(), "stylesheets"); + make_canonical_path( + "install_translations", + paths::install_bin(), "translations"); + conf::set_global("tools", "iscc", path_to_utf8(find_iscc())); } diff --git a/src/conf.h b/src/conf.h index 83f317d..4015f17 100644 --- a/src/conf.h +++ b/src/conf.h @@ -9,6 +9,13 @@ public: static std::string get_global( const std::string& section, const std::string& key); + static int get_global_int( + const std::string& section, const std::string& key); + + static bool get_global_bool( + const std::string& section, const std::string& key); + + static void set_global( const std::string& section, const std::string& key, const std::string& value); @@ -109,6 +116,7 @@ struct paths VALUE(install_stylesheets); VALUE(install_licenses); VALUE(install_pythoncore); + VALUE(install_translations); VALUE(pf_x86); VALUE(pf_x64); diff --git a/src/main.cpp b/src/main.cpp index 85ed925..f201bd4 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -136,6 +136,9 @@ void add_tasks() .add_task({"modorganizer-bsapacker", "bsa_packer"}) .add_task("modorganizer-preview_bsa"); + // the gamebryo flag must be set for all game plugins that inherit from + // the gamebryo classes; this will merge the .ts file from gamebryo with + // the one from the specific plugin add_task(true) .add_task("modorganizer-game_oblivion", mo::gamebryo) .add_task("modorganizer-game_fallout3", mo::gamebryo) @@ -163,7 +166,8 @@ void add_tasks() .add_task("modorganizer-installer_fomod_csharp") .add_task("modorganizer-installer_ncc") .add_task("modorganizer-bsa_extractor") - .add_task("modorganizer-plugin_python"); + .add_task("modorganizer-plugin_python") + .add_task(); add_task(true) .add_task({"modorganizer-tool_configurator", "pycfg"}) diff --git a/src/tasks/tasks.h b/src/tasks/tasks.h index f6d946d..8a699f8 100644 --- a/src/tasks/tasks.h +++ b/src/tasks/tasks.h @@ -590,6 +590,22 @@ private: }; +class translations : public basic_task +{ +public: + translations(); + + static bool prebuilt(); + static std::string version(); + static fs::path source_path(); + +protected: + void do_clean(clean c) override; + void do_fetch() override; + void do_build_and_install() override; +}; + + class usvfs : public basic_task { public: diff --git a/src/tasks/translations.cpp b/src/tasks/translations.cpp new file mode 100644 index 0000000..da55005 --- /dev/null +++ b/src/tasks/translations.cpp @@ -0,0 +1,276 @@ +#include "pch.h" +#include "tasks.h" + +namespace mob +{ + +class translation_projects +{ +public: + struct lang + { + std::string name; + std::vector ts_files; + + lang(std::string n) + : name(std::move(n)) + { + } + }; + + struct project + { + std::string name; + std::vector langs; + + project(std::string n) + : name(std::move(n)) + { + } + }; + + + translation_projects(fs::path root) + : root_(std::move(root)) + { + get(); + } + + const std::vector& projects() const + { + return projects_; + } + + const std::vector& warnings() const + { + return warnings_; + } + + void get() + { + for (auto e : fs::directory_iterator(root_)) + { + if (!e.is_directory()) + continue; + + handle_project_dir(e.path()); + } + } + + bool is_gamebryo_plugin(const std::string& dir, const std::string& project) + { + auto tasks = find_tasks(project); + if (tasks.empty()) + { + warnings_.push_back(::fmt::format( + "directory '{}' was parsed as project '{}', but there's " + "no task with this name", dir, project)); + + return false; + } + + const task& t = *tasks[0]; + + if (!t.is_super()) + return false; + + const auto& mo_task = static_cast(t); + return mo_task.is_gamebryo_plugin(); + } + + void handle_project_dir(const fs::path& dir) + { + const auto dir_name = path_to_utf8(dir.filename()); + const auto dir_cs = split(dir_name, "."); + + if (dir_cs.size() != 2) + { + warnings_.push_back(::fmt::format( + "bad directory name '{}'; skipping", dir_name)); + + return; + } + + const auto project_name = trim_copy(dir_cs[1]); + if (project_name.empty()) + { + warnings_.push_back(::fmt::format( + "bad directory name '{}', skipping", dir_name)); + + return; + } + + project p(project_name); + + const bool gamebryo = is_gamebryo_plugin(dir_name, project_name); + + for (auto f : fs::directory_iterator(dir)) + { + if (!f.is_regular_file()) + continue; + + p.langs.push_back(handle_ts_file(gamebryo, project_name, f.path())); + } + + projects_.push_back(p); + } + + lang handle_ts_file( + bool gamebryo, const std::string& project_name, const fs::path& f) + { + lang lg(path_to_utf8(f.stem())); + + lg.ts_files.push_back(f); + + if (gamebryo) + { + const fs::path gamebryo_dir = + conf::get_global("transifex", "project") + "." + + "game_gamebryo"; + + const auto gb_f = root_ / gamebryo_dir / f.filename(); + + if (fs::exists(gb_f)) + { + lg.ts_files.push_back(gb_f); + } + else + { + if (!warned_.contains(gb_f)) + { + warned_.insert(gb_f); + + warnings_.push_back(::fmt::format( + "{} is a gamebryo plugin but there is no '{}'; the " + ".qm file will be missing some translations (will " + "only warn once)", + project_name, path_to_utf8(gb_f))); + } + } + } + + return lg; + } + +private: + const fs::path root_; + std::vector projects_; + std::vector warnings_; + std::set warned_; +}; + + + +translations::translations() + : basic_task("translations") +{ +} + +bool translations::prebuilt() +{ + return false; +} + +std::string translations::version() +{ + return {}; +} + +fs::path translations::source_path() +{ + return paths::build() / "transifex-translations"; +} + +void translations::do_clean(clean c) +{ + instrument([&] + { + }); +} + +void translations::do_fetch() +{ + instrument([&] + { + const url u = + conf::get_global("transifex", "url") + "/" + + conf::get_global("transifex", "team") + "/" + + conf::get_global("transifex", "project"); + + const std::string key = conf::get_global("transifex", "key"); + + if (key.empty() && !this_env::get_opt("TX_TOKEN")) + { + cx().warning(context::generic, + "no key was in the INI and the TX_TOKEN env variable doesn't " + "exist, this will probably fail"); + } + + cx().debug(context::generic, "init tx"); + run_tool(transifex(transifex::init) + .root(source_path())); + + if (conf::get_global_bool("transifex", "configure")) + { + cx().debug(context::generic, "configuring"); + run_tool(transifex(transifex::config) + .root(source_path()) + .api_key(key) + .url(u)); + } + else + { + cx().trace(context::generic, "skipping configuring"); + } + + if (conf::get_global_bool("transifex", "pull")) + { + cx().debug(context::generic, "pulling"); + run_tool(transifex(transifex::pull) + .root(source_path()) + .api_key(key) + .minimum(conf::get_global_int("transifex", "minimum")) + .force(conf::get_global_bool("transifex", "force"))); + } + else + { + cx().trace(context::generic, "skipping pulling"); + } + }); +} + +void translations::do_build_and_install() +{ + instrument([&] + { + thread_pool threads; + + const auto root = source_path() / "translations"; + const auto dest = paths::install_translations(); + const translation_projects tp(root); + + op::create_directories(cx(), dest); + + for (auto&& w : tp.warnings()) + cx().warning(context::generic, "{}", w); + + for (auto& p : tp.projects()) + { + for (auto& lg : p.langs) + { + threads.add([&] + { + threaded_run(lg.name + "." + p.name, [&] + { + run_tool(lrelease() + .project(p.name) + .sources(lg.ts_files) + .out(dest)); + }); + }); + } + } + }); +} + +} // namespace diff --git a/src/tools/tools.cpp b/src/tools/tools.cpp index 1dc0bac..a3f851e 100644 --- a/src/tools/tools.cpp +++ b/src/tools/tools.cpp @@ -407,6 +407,12 @@ lrelease& lrelease::add_source(const fs::path& ts_file) return *this; } +lrelease& lrelease::sources(const std::vector& v) +{ + sources_ = v; + return *this; +} + lrelease& lrelease::out(const fs::path& dir) { out_ = dir; diff --git a/src/tools/tools.h b/src/tools/tools.h index b3ef6a0..ef038f5 100644 --- a/src/tools/tools.h +++ b/src/tools/tools.h @@ -608,6 +608,7 @@ public: lrelease& project(const std::string& name); lrelease& add_source(const fs::path& ts_file); + lrelease& sources(const std::vector& v); lrelease& out(const fs::path& dir); fs::path qm_file() const; diff --git a/src/utility.cpp b/src/utility.cpp index 5d81e16..8d9535f 100644 --- a/src/utility.cpp +++ b/src/utility.cpp @@ -918,4 +918,63 @@ std::string path_to_utf8(fs::path p) return utf16_to_utf8(p.native()); } + + +thread_pool::thread_pool(std::size_t count) + : count_(std::max(1, count)) +{ + for (std::size_t i=0; i()); +} + +thread_pool::~thread_pool() +{ + join(); +} + +void thread_pool::join() +{ + for (auto&& t : threads_) + { + if (t->thread.joinable()) + t->thread.join(); + } +} + +void thread_pool::add(fun thread_fun) +{ + for (;;) + { + if (try_add(thread_fun)) + break; + + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } +} + +bool thread_pool::try_add(fun thread_fun) +{ + for (auto& t : threads_) + { + if (!t->running) + { + if (t->thread.joinable()) + t->thread.join(); + + t->running = true; + t->thread_fun = thread_fun; + + t->thread = std::thread([&] + { + t->thread_fun(); + t->running = false; + }); + + return true; + } + } + + return false; +} + } // namespace diff --git a/src/utility.h b/src/utility.h index b0bdd59..e4528bd 100644 --- a/src/utility.h +++ b/src/utility.h @@ -588,4 +588,35 @@ auto map(const std::vector& v, F&& f) return out; } + +class thread_pool +{ +public: + typedef std::function fun; + + thread_pool(std::size_t count=std::thread::hardware_concurrency()); + ~thread_pool(); + + // non-copyable + thread_pool(const thread_pool&) = delete; + thread_pool& operator=(const thread_pool&) = delete; + + void add(fun f); + void join(); + +private: + struct thread_info + { + std::atomic running = false; + fun thread_fun; + std::thread thread; + }; + + + const std::size_t count_; + std::vector> threads_; + + bool try_add(fun thread_fun); +}; + } // namespace diff --git a/vs/mob.vcxproj b/vs/mob.vcxproj index 538cf5e..a26a9b4 100644 --- a/vs/mob.vcxproj +++ b/vs/mob.vcxproj @@ -93,6 +93,7 @@ + diff --git a/vs/mob.vcxproj.filters b/vs/mob.vcxproj.filters index 0dcd03a..5e10200 100644 --- a/vs/mob.vcxproj.filters +++ b/vs/mob.vcxproj.filters @@ -144,6 +144,9 @@ src\tasks + + src\tasks +