mo2/mob
0
0
Fork 0
mirror of https://github.com/ModOrganizer2/mob.git synced 2026-07-27 14:07:05 -07:00
Code Issues Packages Projects Releases Wiki Activity

translations task

Browse Source
This commit is contained in:
isanae
2020-07-13 08:17:09 -04:00
parent 7b073c4c5d
commit 518b20db0d
14 changed files with 543 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mob.ini
+40 -32
View File
@@ -19,10 +19,11 @@ mo_branch = master
no_pull = false
ignore_ts = false
revert_ts = false
git_url_prefix = https://github.com/
git_shallow = true
git_username =
git_email =
git_shallow = true
git_username =
git_email =
set_origin_remote = false
remote_org =
@@ -36,6 +37,9 @@ git_shallow = false
[usvfs:task]
git_shallow = false
[translations:task]
enabled = false
[installer:task]
enabled = false
@@ -57,12 +61,15 @@ iscc = ISCC.exe
vcvars =
[transifex]
enabled = true
key =
team = mod-organizer-2-team
project = mod-organizer-2
url = https://www.transifex.com
minimum = 60
enabled = true
key =
team = mod-organizer-2-team
project = mod-organizer-2
url = https://www.transifex.com
minimum = 60
force = false
configure = true
pull = true
[prebuilt]
boost = true
@@ -107,26 +114,27 @@ ss_paper_mono_6788 = 2.1
ss_dark_mode_1809_6788 = 2.0
[paths]
third_party =
prefix =
cache =
patches =
licenses =
build =
install =
install_bin =
install_installer =
install_libs =
install_pdbs =
install_dlls =
install_loot =
install_plugins =
install_stylesheets =
install_licenses =
install_pythoncore =
vs =
qt_install =
qt_bin =
pf_x86 =
pf_x64 =
temp_dir =
third_party =
prefix =
cache =
patches =
licenses =
build =
install =
install_bin =
install_installer =
install_libs =
install_pdbs =
install_dlls =
install_loot =
install_plugins =
install_stylesheets =
install_licenses =
install_pythoncore =
install_translations =
vs =
qt_install =
qt_bin =
pf_x86 =
pf_x64 =
temp_dir =
src/commands.cpp
+49 -39
View File
@@ -1357,21 +1357,29 @@ clipp::group tx_command::do_group()
"get" %
(clipp::command("get").set(mode_, modes::get),
(clipp::option("-m", "--minimum")
& clipp::value("PERCENT") >> min_)
% "minimum translation threshold to download [0-100]",
(clipp::option("-k", "--key")
& clipp::value("APIKEY") >> key_)
% "API key",
(clipp::option("-f", "--force").set(force_)
% "don't check timestamps, re-download all translation files"),
(clipp::option("-t", "--team")
& clipp::value("TEAM") >> team_)
% "team name",
(clipp::option("-p", "--project")
& clipp::value("PROJECT") >> project_)
% "project name",
(clipp::option("-u", "--url")
& clipp::value("URL") >> url_)
% "project URL",
(clipp::option("-m", "--minimum")
& clipp::value("PERCENT").set(min_))
% "minimum translation threshold to download [0-100]",
(clipp::option("-f", "--force").call([&]{ force_ = true; }))
% "don't check timestamps, re-download all translation files",
(clipp::value("path") >> path_)
% "path that will contain the .tx directory"
)
@@ -1390,6 +1398,29 @@ clipp::group tx_command::do_group()
);
}
void tx_command::convert_cl_to_conf()
{
command::convert_cl_to_conf();
if (!key_.empty())
common.options.push_back("transifex/key=" + key_);
if (!team_.empty())
common.options.push_back("transifex/team=" + team_);
if (!project_.empty())
common.options.push_back("transifex/project=" + project_);
if (!url_.empty())
common.options.push_back("transifex/url=" + url_);
if (min_ >= 0)
common.options.push_back("transifex/minimum=" + std::to_string(min_));
if (force_)
common.options.push_back("transifex/force=" + std::to_string(*force_));
}
int tx_command::do_run()
{
switch (mode_)
@@ -1414,8 +1445,7 @@ int tx_command::do_run()
std::string tx_command::do_doc()
{
return
"Values for --key, --minimum and --url will be taken from the INI\n"
"file if not specified.\n"
"Some values will be taken from the INI file if not specified.\n"
"\n"
"Commands:\n"
"get\n"
@@ -1430,40 +1460,20 @@ std::string tx_command::do_doc()
void tx_command::do_get()
{
if (min_ < 0)
{
const auto s = conf::get_global("transifex", "minimum");
const url u =
conf::get_global("transifex", "url") + "/" +
conf::get_global("transifex", "team") + "/" +
conf::get_global("transifex", "project");
try
{
min_ = std::stoi(s);
}
catch(std::exception&)
{
gcx().bail_out(context::generic,
"bad transifex minimum percentage '{}'", s);
}
}
const std::string key = conf::get_global("transifex", "key");
if (key_.empty())
key_ = conf::get_global("transifex", "key");
if (url_.empty())
{
url_ =
conf::get_global("transifex", "url") + "/" +
conf::get_global("transifex", "team") + "/" +
conf::get_global("transifex", "project");
}
if (key_.empty() && !this_env::get_opt("TX_TOKEN"))
if (key.empty() && !this_env::get_opt("TX_TOKEN"))
{
u8cout <<
"(no key was in the INI, --key wasn't given and TX_TOKEN env\n"
"variable doesn't exist, this will probably fail)\n\n";
}
context cxcopy = gcx();
u8cout << "initializing\n";
@@ -1475,17 +1485,17 @@ void tx_command::do_get()
transifex(transifex::config)
.stdout_level(context::level::info)
.root(path_)
.api_key(key_)
.url(url_)
.api_key(key)
.url(u)
.run(cxcopy);
u8cout << "pulling\n";
transifex(transifex::pull)
.stdout_level(context::level::info)
.root(path_)
.api_key(key_)
.minimum(min_)
.force(force_)
.api_key(key)
.minimum(conf::get_global_int("transifex", "minimum"))
.force(conf::get_global_bool("transifex", "force"))
.run(cxcopy);
}
src/commands.h
+3 -3
View File
@@ -287,6 +287,7 @@ public:
protected:
clipp::group do_group() override;
void convert_cl_to_conf() override;
int do_run() override;
std::string do_doc() override;
@@ -299,10 +300,9 @@ private:
};
modes mode_ = modes::none;
std::string key_, team_, project_, url_;
int min_ = -1;
std::string key_;
std::string url_;
bool force_ = false;
std::optional<bool> force_;
std::string path_;
std::string dest_;
src/conf.cpp
+45 -8
View File
@@ -19,6 +19,11 @@ std::string master_ini_filename()
return "mob.ini";
}
bool bool_from_string(const std::string& s)
{
return (s == "true" || s == "yes" || s == "1");
}
std::string conf::get_global(const std::string& section, const std::string& key)
{
@@ -66,6 +71,26 @@ void conf::set_global(
kitor->second = value;
}
int conf::get_global_int(const std::string& section, const std::string& key)
{
const auto s = conf::get_global(section, key);
try
{
return std::stoi(s);
}
catch(std::exception&)
{
gcx().bail_out(context::conf, "bad int for {}/{}", section, key);
}
}
bool conf::get_global_bool(const std::string& section, const std::string& key)
{
const auto s = conf::get_global(section, key);
return bool_from_string(s);
}
void conf::add_global(
const std::string& section,
const std::string& key, const std::string& value)
@@ -141,7 +166,7 @@ void conf::set_for_task(
bool conf::prebuilt_by_name(const std::string& task)
{
const std::string s = get_global("prebuilt", task);
return (s == "true" || s == "yes" || s == "1");
return bool_from_string(s);
}
fs::path conf::path_by_name(const std::string& name)
@@ -167,7 +192,7 @@ std::string conf::global_by_name(const std::string& name)
bool conf::bool_global_by_name(const std::string& name)
{
const std::string s = global_by_name(name);
return (s == "true" || s == "yes" || s == "1");
return bool_from_string(s);
}
std::string conf::task_option_by_name(
@@ -180,7 +205,7 @@ bool conf::bool_task_option_by_name(
const std::vector<std::string>& task_names, const std::string& name)
{
const std::string s = task_option_by_name(task_names, name);
return (s == "true" || s == "yes" || s == "1");
return bool_from_string(s);
}
void conf::set_output_log_level(const std::string& s)
@@ -224,7 +249,7 @@ void conf::set_file_log_level(const std::string& s)
void conf::set_dry(const std::string& s)
{
dry_ = (s == "true" || s == "yes" || s == "1");
dry_ = bool_from_string(s);
}
@@ -662,7 +687,7 @@ void ini_error(const fs::path& ini, std::size_t line, const std::string& what)
{
gcx().bail_out(context::conf,
"{}:{}: {}",
ini.filename(), (line + 1), what);
path_to_utf8(ini), (line + 1), what);
}
std::vector<std::string> read_ini(const fs::path& ini)
@@ -680,9 +705,6 @@ std::vector<std::string> read_ini(const fs::path& ini)
if (!in)
break;
if (line.empty() || line[0] == '#' || line[0] == ';')
continue;
lines.push_back(std::move(line));
}
@@ -706,6 +728,12 @@ void parse_section(
const auto& line = lines[i];
if (line.empty() || line[0] == '#' || line[0] == ';')
{
++i;
continue;
}
const auto sep = line.find("=");
if (sep == std::string::npos)
ini_error(ini, i, "bad line '" + line + "'");
@@ -758,6 +786,11 @@ void parse_ini(const fs::path& ini, bool add)
break;
const auto& line = lines[i];
if (line.empty() || line[0] == '#' || line[0] == ';')
{
++i;
continue;
}
if (line.starts_with("[") && line.ends_with("]"))
{
@@ -1097,6 +1130,10 @@ void init_options(
"install_stylesheets",
paths::install_bin(), "stylesheets");
make_canonical_path(
"install_translations",
paths::install_bin(), "translations");
conf::set_global("tools", "iscc", path_to_utf8(find_iscc()));
}
src/conf.h
+8
View File
@@ -9,6 +9,13 @@ public:
static std::string get_global(
const std::string& section, const std::string& key);
static int get_global_int(
const std::string& section, const std::string& key);
static bool get_global_bool(
const std::string& section, const std::string& key);
static void set_global(
const std::string& section,
const std::string& key, const std::string& value);
@@ -109,6 +116,7 @@ struct paths
VALUE(install_stylesheets);
VALUE(install_licenses);
VALUE(install_pythoncore);
VALUE(install_translations);
VALUE(pf_x86);
VALUE(pf_x64);
src/main.cpp
+5 -1
View File
@@ -136,6 +136,9 @@ void add_tasks()
.add_task<mo>({"modorganizer-bsapacker", "bsa_packer"})
.add_task<mo>("modorganizer-preview_bsa");
// the gamebryo flag must be set for all game plugins that inherit from
// the gamebryo classes; this will merge the .ts file from gamebryo with
// the one from the specific plugin
add_task<parallel_tasks>(true)
.add_task<mo>("modorganizer-game_oblivion", mo::gamebryo)
.add_task<mo>("modorganizer-game_fallout3", mo::gamebryo)
@@ -163,7 +166,8 @@ void add_tasks()
.add_task<mo>("modorganizer-installer_fomod_csharp")
.add_task<mo>("modorganizer-installer_ncc")
.add_task<mo>("modorganizer-bsa_extractor")
.add_task<mo>("modorganizer-plugin_python");
.add_task<mo>("modorganizer-plugin_python")
.add_task<translations>();
add_task<parallel_tasks>(true)
.add_task<mo>({"modorganizer-tool_configurator", "pycfg"})
src/tasks/tasks.h
+16
View File
@@ -590,6 +590,22 @@ private:
};
class translations : public basic_task<translations>
{
public:
translations();
static bool prebuilt();
static std::string version();
static fs::path source_path();
protected:
void do_clean(clean c) override;
void do_fetch() override;
void do_build_and_install() override;
};
class usvfs : public basic_task<usvfs>
{
public:
src/tasks/translations.cpp
+276
View File
@@ -0,0 +1,276 @@
#include "pch.h"
#include "tasks.h"
namespace mob
{
class translation_projects
{
public:
struct lang
{
std::string name;
std::vector<fs::path> ts_files;
lang(std::string n)
: name(std::move(n))
{
}
};
struct project
{
std::string name;
std::vector<lang> langs;
project(std::string n)
: name(std::move(n))
{
}
};
translation_projects(fs::path root)
: root_(std::move(root))
{
get();
}
const std::vector<project>& projects() const
{
return projects_;
}
const std::vector<std::string>& warnings() const
{
return warnings_;
}
void get()
{
for (auto e : fs::directory_iterator(root_))
{
if (!e.is_directory())
continue;
handle_project_dir(e.path());
}
}
bool is_gamebryo_plugin(const std::string& dir, const std::string& project)
{
auto tasks = find_tasks(project);
if (tasks.empty())
{
warnings_.push_back(::fmt::format(
"directory '{}' was parsed as project '{}', but there's "
"no task with this name", dir, project));
return false;
}
const task& t = *tasks[0];
if (!t.is_super())
return false;
const auto& mo_task = static_cast<const modorganizer&>(t);
return mo_task.is_gamebryo_plugin();
}
void handle_project_dir(const fs::path& dir)
{
const auto dir_name = path_to_utf8(dir.filename());
const auto dir_cs = split(dir_name, ".");
if (dir_cs.size() != 2)
{
warnings_.push_back(::fmt::format(
"bad directory name '{}'; skipping", dir_name));
return;
}
const auto project_name = trim_copy(dir_cs[1]);
if (project_name.empty())
{
warnings_.push_back(::fmt::format(
"bad directory name '{}', skipping", dir_name));
return;
}
project p(project_name);
const bool gamebryo = is_gamebryo_plugin(dir_name, project_name);
for (auto f : fs::directory_iterator(dir))
{
if (!f.is_regular_file())
continue;
p.langs.push_back(handle_ts_file(gamebryo, project_name, f.path()));
}
projects_.push_back(p);
}
lang handle_ts_file(
bool gamebryo, const std::string& project_name, const fs::path& f)
{
lang lg(path_to_utf8(f.stem()));
lg.ts_files.push_back(f);
if (gamebryo)
{
const fs::path gamebryo_dir =
conf::get_global("transifex", "project") + "." +
"game_gamebryo";
const auto gb_f = root_ / gamebryo_dir / f.filename();
if (fs::exists(gb_f))
{
lg.ts_files.push_back(gb_f);
}
else
{
if (!warned_.contains(gb_f))
{
warned_.insert(gb_f);
warnings_.push_back(::fmt::format(
"{} is a gamebryo plugin but there is no '{}'; the "
".qm file will be missing some translations (will "
"only warn once)",
project_name, path_to_utf8(gb_f)));
}
}
}
return lg;
}
private:
const fs::path root_;
std::vector<project> projects_;
std::vector<std::string> warnings_;
std::set<fs::path> warned_;
};
translations::translations()
: basic_task("translations")
{
}
bool translations::prebuilt()
{
return false;
}
std::string translations::version()
{
return {};
}
fs::path translations::source_path()
{
return paths::build() / "transifex-translations";
}
void translations::do_clean(clean c)
{
instrument<times::clean>([&]
{
});
}
void translations::do_fetch()
{
instrument<times::fetch>([&]
{
const url u =
conf::get_global("transifex", "url") + "/" +
conf::get_global("transifex", "team") + "/" +
conf::get_global("transifex", "project");
const std::string key = conf::get_global("transifex", "key");
if (key.empty() && !this_env::get_opt("TX_TOKEN"))
{
cx().warning(context::generic,
"no key was in the INI and the TX_TOKEN env variable doesn't "
"exist, this will probably fail");
}
cx().debug(context::generic, "init tx");
run_tool(transifex(transifex::init)
.root(source_path()));
if (conf::get_global_bool("transifex", "configure"))
{
cx().debug(context::generic, "configuring");
run_tool(transifex(transifex::config)
.root(source_path())
.api_key(key)
.url(u));
}
else
{
cx().trace(context::generic, "skipping configuring");
}
if (conf::get_global_bool("transifex", "pull"))
{
cx().debug(context::generic, "pulling");
run_tool(transifex(transifex::pull)
.root(source_path())
.api_key(key)
.minimum(conf::get_global_int("transifex", "minimum"))
.force(conf::get_global_bool("transifex", "force")));
}
else
{
cx().trace(context::generic, "skipping pulling");
}
});
}
void translations::do_build_and_install()
{
instrument<times::build>([&]
{
thread_pool threads;
const auto root = source_path() / "translations";
const auto dest = paths::install_translations();
const translation_projects tp(root);
op::create_directories(cx(), dest);
for (auto&& w : tp.warnings())
cx().warning(context::generic, "{}", w);
for (auto& p : tp.projects())
{
for (auto& lg : p.langs)
{
threads.add([&]
{
threaded_run(lg.name + "." + p.name, [&]
{
run_tool(lrelease()
.project(p.name)
.sources(lg.ts_files)
.out(dest));
});
});
}
}
});
}
} // namespace
src/tools/tools.cpp
+6
View File
@@ -407,6 +407,12 @@ lrelease& lrelease::add_source(const fs::path& ts_file)
return *this;
}
lrelease& lrelease::sources(const std::vector<fs::path>& v)
{
sources_ = v;
return *this;
}
lrelease& lrelease::out(const fs::path& dir)
{
out_ = dir;
src/tools/tools.h
+1
View File
@@ -608,6 +608,7 @@ public:
lrelease& project(const std::string& name);
lrelease& add_source(const fs::path& ts_file);
lrelease& sources(const std::vector<fs::path>& v);
lrelease& out(const fs::path& dir);
fs::path qm_file() const;
src/utility.cpp
+59
View File
@@ -918,4 +918,63 @@ std::string path_to_utf8(fs::path p)
return utf16_to_utf8(p.native());
}
thread_pool::thread_pool(std::size_t count)
: count_(std::max<std::size_t>(1, count))
{
for (std::size_t i=0; i<count_; ++i)
threads_.emplace_back(std::make_unique<thread_info>());
}
thread_pool::~thread_pool()
{
join();
}
void thread_pool::join()
{
for (auto&& t : threads_)
{
if (t->thread.joinable())
t->thread.join();
}
}
void thread_pool::add(fun thread_fun)
{
for (;;)
{
if (try_add(thread_fun))
break;
std::this_thread::sleep_for(std::chrono::milliseconds(1));
}
}
bool thread_pool::try_add(fun thread_fun)
{
for (auto& t : threads_)
{
if (!t->running)
{
if (t->thread.joinable())
t->thread.join();
t->running = true;
t->thread_fun = thread_fun;
t->thread = std::thread([&]
{
t->thread_fun();
t->running = false;
});
return true;
}
}
return false;
}
} // namespace
src/utility.h
+31
View File
@@ -588,4 +588,35 @@ auto map(const std::vector<T>& v, F&& f)
return out;
}
class thread_pool
{
public:
typedef std::function<void ()> fun;
thread_pool(std::size_t count=std::thread::hardware_concurrency());
~thread_pool();
// non-copyable
thread_pool(const thread_pool&) = delete;
thread_pool& operator=(const thread_pool&) = delete;
void add(fun f);
void join();
private:
struct thread_info
{
std::atomic<bool> running = false;
fun thread_fun;
std::thread thread;
};
const std::size_t count_;
std::vector<std::unique_ptr<thread_info>> threads_;
bool try_add(fun thread_fun);
};
} // namespace
vs/mob.vcxproj
+1
View File
@@ -93,6 +93,7 @@
<ClCompile Include="..\src\tasks\spdlog.cpp" />
<ClCompile Include="..\src\tasks\stylesheets.cpp" />
<ClCompile Include="..\src\tasks\task.cpp" />
<ClCompile Include="..\src\tasks\translations.cpp" />
<ClCompile Include="..\src\tasks\usvfs.cpp" />
<ClCompile Include="..\src\tasks\zlib.cpp" />
<ClCompile Include="..\src\tools\cmake.cpp" />
vs/mob.vcxproj.filters
+3
View File
@@ -144,6 +144,9 @@
<ClCompile Include="..\src\tasks\installer.cpp">
<Filter>src\tasks</Filter>
</ClCompile>
<ClCompile Include="..\src\tasks\translations.cpp">
<Filter>src\tasks</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\src\pch.h">