added most modorganizer projects

silenced cmake warnings about unused variables
added 'super' task name for everything in modorganizer_super
pyqt didn't copy two dlls
clone first, then add submodules so it can be done in parallel, adding submodules needs to lock
missing boost libraries
process: reduced timeouts, fixed pipes not being completely read when the process terminates

src/context.cpp

@@ -57,7 +57,7 @@ std::string reason_string(context::reason r)
 
 std::string task_name(const std::string& name)
 {
-	const std::size_t longest = 7;
+	const std::size_t longest = 15;
 	const std::size_t total = 1 + longest + 2; // '[x] '
 
 	if (!name.empty())

src/main.cpp

@@ -87,7 +87,8 @@ std::optional<int> handle_command_line(int argc, char** argv)
 
 		(clipp::opt_values(
 			clipp::match::prefix_not("-"), "task", g_tasks_to_run))
-			% "tasks to run"
+			% "tasks to run; specify 'super' to only build modorganizer "
+		      "projects"
 	);
 
 
@@ -176,6 +177,49 @@ void add_tasks()
 
 	add_task<modorganizer>("cmake_common");
 	add_task<modorganizer>("modorganizer-uibase");
+	add_task<modorganizer>("modorganizer-game_features");
+	add_task<modorganizer>("modorganizer-archive");
+	add_task<modorganizer>("modorganizer-lootcli");
+	add_task<modorganizer>("modorganizer-esptk");
+	add_task<modorganizer>("modorganizer-bsatk");
+	add_task<modorganizer>("modorganizer-nxmhandler");
+	add_task<modorganizer>("modorganizer-helper");
+	add_task<modorganizer>("modorganizer-game_gamebryo");
+	add_task<modorganizer>("modorganizer-game_oblivion");
+	add_task<modorganizer>("modorganizer-game_fallout3");
+	add_task<modorganizer>("modorganizer-game_fallout4");
+	add_task<modorganizer>("modorganizer-game_fallout4vr");
+	add_task<modorganizer>("modorganizer-game_falloutnv");
+	add_task<modorganizer>("modorganizer-game_morrowind");
+	add_task<modorganizer>("modorganizer-game_skyrim");
+	add_task<modorganizer>("modorganizer-game_skyrimse");
+	add_task<modorganizer>("modorganizer-game_skyrimvr");
+	add_task<modorganizer>("modorganizer-game_ttw");
+	add_task<modorganizer>("modorganizer-game_enderal");
+
+	add_task<modorganizer>("modorganizer-tool_inieditor");
+	add_task<modorganizer>("modorganizer-tool_inibakery");
+	add_task<modorganizer>("modorganizer-preview_base");
+	add_task<modorganizer>("modorganizer-diagnose_basic");
+	add_task<modorganizer>("modorganizer-check_fnis");
+	add_task<modorganizer>("modorganizer-installer_bain");
+	add_task<modorganizer>("modorganizer-installer_manual");
+	add_task<modorganizer>("modorganizer-installer_bundle");
+	add_task<modorganizer>("modorganizer-installer_quick");
+	add_task<modorganizer>("modorganizer-installer_fomod");
+	add_task<modorganizer>("modorganizer-installer_ncc");
+	add_task<modorganizer>("modorganizer-bsa_extractor");
+	add_task<modorganizer>("modorganizer-plugin_python");
+
+	add_task<modorganizer>("modorganizer-tool_configurator");
+	add_task<modorganizer>("modorganizer-fnistool");
+	add_task<modorganizer>("modorganizer-script_extender_plugin_checker");
+	add_task<modorganizer>("modorganizer-form43_checker");
+	add_task<modorganizer>("modorganizer-preview_dds");
+	//add_task<modorganizer>("githubpp");
+	//add_task<modorganizer>("modorganizer-bsapacker");
+	//add_task<modorganizer>("modorganizer-preview_bsa");
+	//add_task<modorganizer>("modorganizer");
 }
 
 int run(int argc, char** argv)

src/op.cpp

@@ -222,8 +222,11 @@ void move_to_directory(
 void copy_file_to_dir_if_better(
 	const context& cx, const fs::path& file, const fs::path& dir, flags f)
 {
-	check(cx, file);
-	check(cx, dir);
+	if ((f & unsafe) == 0)
+	{
+		check(cx, file);
+		check(cx, dir);
+	}
 
 	if (file.string().find("*") != std::string::npos)
 		cx.bail_out(context::fs, file.string() + " contains a glob");

src/op.h

@@ -12,7 +12,8 @@ enum flags
 	noflags    = 0x00,
 	optional   = 0x01,
 	copy_files = 0x02,
-	copy_dirs  = 0x04
+	copy_dirs  = 0x04,
+	unsafe     = 0x08
 };
 
 MOB_ENUM_OPERATORS(flags);

src/process.cpp

@@ -8,6 +8,10 @@
 namespace mob
 {
 
+const DWORD pipe_timeout = 50;
+const DWORD process_wait_timeout = 50;
+
+
 HANDLE get_bit_bucket()
 {
 	SECURITY_ATTRIBUTES sa { .nLength = sizeof(sa), .bInheritHandle = TRUE };
@@ -131,7 +135,7 @@ std::string_view async_pipe::try_read()
 
 			case ERROR_BROKEN_PIPE:
 			{
-				// broken pipe probably means lootcli is finished
+				// broken pipe probably means the process is finished
 				break;
 			}
 
@@ -493,7 +497,8 @@ void process::join()
 
 	for (;;)
 	{
-		const auto r = WaitForSingleObject(impl_.handle.get(), 100);
+		const auto r = WaitForSingleObject(
+			impl_.handle.get(), process_wait_timeout);
 
 		if (r == WAIT_OBJECT_0)
 		{
@@ -515,14 +520,18 @@ void process::join()
 		cx_->trace(context::cmd, "process interrupted and finished");
 }
 
-void process::read_pipes()
+bool process::read_pipes()
 {
+	bool read_something = false;
+
 	// stdout
 	switch (stdout_flags_)
 	{
 		case forward_to_log:
 		{
 			std::string_view s = impl_.stdout_pipe.read();
+			if (!s.empty())
+				read_something = true;
 
 			for_each_line(s, [&](auto&& line)
 			{
@@ -544,6 +553,9 @@ void process::read_pipes()
 		case keep_in_string:
 		{
 			std::string_view s = impl_.stdout_pipe.read();
+			if (!s.empty())
+				read_something = true;
+
 			stdout_string_ += s;
 			break;
 		}
@@ -559,6 +571,8 @@ void process::read_pipes()
 		case forward_to_log:
 		{
 			std::string_view s = impl_.stderr_pipe.read();
+			if (!s.empty())
+				read_something = true;
 
 			for_each_line(s, [&](auto&& line)
 			{
@@ -580,6 +594,9 @@ void process::read_pipes()
 		case keep_in_string:
 		{
 			std::string_view s = impl_.stderr_pipe.read();
+			if (!s.empty())
+				read_something = true;
+
 			stderr_string_ += s;
 			break;
 		}
@@ -588,12 +605,18 @@ void process::read_pipes()
 		case inherit:
 			break;
 	}
+
+	return read_something;
 }
 
 void process::on_completed()
 {
 	// one last time
-	read_pipes();
+	for (;;)
+	{
+		if (!read_pipes())
+			break;
+	}
 
 	if (impl_.interrupt)
 		return;

src/process.h

@@ -20,7 +20,6 @@ public:
 
 private:
     static const std::size_t buffer_size = 50000;
-	static const DWORD pipe_timeout = 500;
 
     handle_ptr stdout_;
     handle_ptr event_;
@@ -200,7 +199,7 @@ private:
 	void pipe_into(const process& p);
 
 	void do_run(const std::string& what);
-	void read_pipes();
+	bool read_pipes();
 
 	void on_completed();
 	void on_timeout(bool& already_interrupted);

src/tasks/boost.cpp

@@ -110,6 +110,10 @@ void boost::build_and_install_from_source()
 		{"thread", "date_time", "filesystem", "locale"},
 		"static", "static", arch::x86);
 
+	do_b2(
+		{"thread", "date_time", "locale"},
+		"static", "shared", arch::x64);
+
 	do_b2(
 		{"thread", "python"},
 		"shared", "shared", arch::x64);

src/tasks/modorganizer.cpp

@@ -7,12 +7,24 @@ namespace mob
 static std::mutex g_super_mutex;
 static std::atomic<bool> g_super_initialized = false;
 
-modorganizer::modorganizer(std::string long_name)
-	: basic_task(long_name)
+std::string make_short_name(const std::string& name)
 {
-	auto s = short_name();
-	if (s != name())
-		add_name(s);
+	const auto dash = name.find("-");
+	if (dash == std::string::npos)
+		return name;
+
+	return name.substr(dash + 1);
+}
+
+
+modorganizer::modorganizer(std::string long_name)
+	: basic_task(make_short_name(long_name), long_name), repo_(long_name)
+{
+}
+
+bool modorganizer::is_super() const
+{
+	return true;
 }
 
 fs::path modorganizer::source_path()
@@ -22,7 +34,7 @@ fs::path modorganizer::source_path()
 
 fs::path modorganizer::this_source_path() const
 {
-	return super_path() / short_name();
+	return super_path() / name();
 }
 
 fs::path modorganizer::super_path()
@@ -34,36 +46,31 @@ void modorganizer::do_fetch()
 {
 	initialize_super(super_path());
 
-	if (fs::exists(this_source_path() / ".git"))
-	{
-		run_tool(git_clone()
-			.url(make_github_url(conf::mo_org(), name()))
-			.branch(conf::mo_branch())
-			.output(this_source_path()));
-	}
-	else
-	{
-		run_tool(process_runner(process()
-			.binary(tools::git())
-			.arg("-c", "core.autocrlf=false")
-			.arg("submodule")
-			.arg("--quiet")
-			.arg("add")
-			.arg("-b", conf::mo_branch())
-			.arg("--force")
-			.arg("--name", short_name())
-			.arg(make_github_url(conf::mo_org(), name()))
-			.arg(short_name())
-			.cwd(super_path())));
-	}
+	run_tool(git_clone()
+		.url(make_github_url(conf::mo_org(), repo_))
+		.branch(conf::mo_branch())
+		.output(this_source_path()));
 }
 
 void modorganizer::do_build_and_install()
 {
+	run_tool(process_runner(process()
+		.binary(tools::git())
+		.arg("-c", "core.autocrlf=false")
+		.arg("submodule")
+		.arg("--quiet")
+		.arg("add")
+		.arg("-b", conf::mo_branch())
+		.arg("--force")
+		.arg("--name", name())
+		.arg(make_github_url(conf::mo_org(), repo_))
+		.arg(name())
+		.cwd(super_path())));
+
 	if (!fs::exists(this_source_path() / "CMakeLists.txt"))
 	{
 		cx().trace(context::generic,
-			name() + " has no CMakeLists.txt, not running cmake");
+			repo_ + " has no CMakeLists.txt, not building");
 
 		return;
 	}
@@ -137,13 +144,4 @@ void modorganizer::initialize_super(const fs::path& super_root)
 		.cwd(super_root)));
 }
 
-std::string modorganizer::short_name() const
-{
-	const auto dash = name().find("-");
-	if (dash == std::string::npos)
-		return name();
-
-	return name().substr(dash + 1);
-}
-
 }	// namespace

src/tasks/pyqt.cpp

@@ -156,6 +156,16 @@ void pyqt::copy_files(const std::vector<std::string>& modules)
 	op::copy_file_to_dir_if_better(cx(),
 		sip::module_source_path() / "sip.pyi",
 		pyqt_plugin);
+
+	op::copy_file_to_dir_if_better(cx(),
+		paths::qt_bin() / "Qt5Core.dll",
+		python::build_path(),
+		op::unsafe);   // source file is outside prefix
+
+	op::copy_file_to_dir_if_better(cx(),
+		paths::qt_bin() / "Qt5Xml.dll",
+		python::build_path(),
+		op::unsafe);   // source file is outside prefix
 }
 
 url pyqt::source_url()

src/tasks/task.cpp

@@ -39,7 +39,7 @@ task* find_task(const std::string& name)
 	throw bailed("");
 }
 
-void run_tasks(const std::vector<task*> tasks)
+void run_tasks(const std::set<task*> tasks)
 {
 	for (auto* t : tasks)
 		t->fetch();
@@ -52,6 +52,15 @@ void run_tasks(const std::vector<task*> tasks)
 	}
 }
 
+void gather_super_tasks(std::set<task*>& tasks)
+{
+	for (auto& t : g_tasks)
+	{
+		if (t->is_super())
+			tasks.insert(t.get());
+	}
+}
+
 void run_task(const std::string& name)
 {
 	run_tasks({find_task(name)});
@@ -67,18 +76,23 @@ void run_tasks(const std::vector<std::string>& names)
 	else
 		gcx().debug(context::generic, "specified tasks: " + join(names, " "));
 
-	std::vector<task*> tasks;
+	std::set<task*> tasks;
 	for (auto&& name : names)
-		tasks.push_back(find_task(name));
+	{
+		if (name == "super")
+			gather_super_tasks(tasks);
+		else
+			tasks.insert(find_task(name));
+	}
 
 	run_tasks(tasks);
 }
 
 void run_all_tasks()
 {
-	std::vector<task*> tasks;
+	std::set<task*> tasks;
 	for (auto&& t : g_tasks)
-		tasks.push_back(t.get());
+		tasks.insert(t.get());
 
 	run_tasks(tasks);
 }
@@ -103,6 +117,11 @@ task::~task()
 	}
 }
 
+bool task::is_super() const
+{
+	return false;
+}
+
 const context& task::cx() const
 {
 	static const context bad("?");

src/tasks/task.h

@@ -36,6 +36,7 @@ public:
 	const std::vector<std::string>& names() const;
 
 	virtual fs::path get_source_path() const = 0;
+	virtual bool is_super() const;
 
 	void run();
 	void interrupt();

src/tasks/tasks.h

@@ -179,14 +179,17 @@ public:
 	modorganizer(std::string name);
 	static fs::path source_path();
 
+	bool is_super() const override;
+
 protected:
 	void do_fetch() override;
 	void do_build_and_install() override;
 
 private:
+	std::string repo_;
+
 	void initialize_super(const fs::path& super_root);
 
-	std::string short_name() const;
 	fs::path this_source_path() const;
 
 	static fs::path super_path();

src/tools/cmake.cpp

@@ -85,6 +85,7 @@ void cmake::do_run()
 		.arg("-DCMAKE_BUILD_TYPE=Release")
 		.arg("-DCMAKE_INSTALL_MESSAGE=NEVER", process::log_quiet)
 		.arg("--log-level", "WARNING", process::log_quiet)
+		.arg("--no-warn-unused-cli")
 		.arg(g.get_arch(arch_));
 
 	if (!prefix_.empty())