nasm

support for multiple urls in downloader
stop using -spe for 7z, do it manually
started boost from source and python, completed libffi and openssl

src/conf.cpp

@@ -11,7 +11,7 @@ static std::map<std::string, std::string> g_conf =
 	{"vs_year",      "2019"},
 	{"sevenzip",     "19.00"},
 	{"zlib",         "1.2.11"},
-	{"boost",        "1.72.0"},
+	{"boost",        "1.72.0-b1-rc1"},
 	{"boost_vs",     "14.2"},
 	{"python",       "3.8.1"},
 	{"fmt",          "6.1.2"},
@@ -19,6 +19,7 @@ static std::map<std::string, std::string> g_conf =
 	{"libbsarch",    "0.0.8"},
 	{"libloot",      "0.15.1"},
 	{"libloot_hash", "gf725dd7"},
+	{"openssl",      "1.1.1d"},
 
 	{"prefix",   R"(C:\dev\projects\mobuild-out)"},
 };
@@ -93,6 +94,9 @@ const std::string& versions::gtest()        { return get_conf("gtest"); }
 const std::string& versions::libbsarch()    { return get_conf("libbsarch"); }
 const std::string& versions::libloot()      { return get_conf("libloot"); }
 const std::string& versions::libloot_hash() { return get_conf("libloot_hash"); }
+const std::string& versions::openssl()      { return get_conf("openssl"); }
+
+bool prebuilt::boost() { return false; }
 
 fs::path paths::prefix()         { return get_conf("prefix"); }
 fs::path paths::cache()          { return prefix() / "downloads"; }
@@ -101,6 +105,7 @@ fs::path paths::install()        { return prefix() / "install"; }
 fs::path paths::install_bin( )   { return install() / "bin"; }
 fs::path paths::install_dlls()   { return install_bin() / "dlls"; }
 fs::path paths::install_loot()   { return install_bin() / "loot"; }
+fs::path paths::install_pdbs()   { return install_bin() / "pdbs"; }
 
 fs::path paths::patches()
 {
@@ -175,23 +180,26 @@ fs::path paths::temp_file()
 }
 
 
+fs::path find_third_party_directory()
+{
+	static fs::path path = find_in_root("third-party");
+	return path;
+}
+
 
 fs::path third_party::sevenz()
 {
-	static fs::path path = find_in_root("third-party/bin/7z.exe");
-	return path;
+	return "7z";
 }
 
 fs::path third_party::jom()
 {
-	static fs::path path = find_in_root("third-party/bin/jom.exe");
-	return path;
+	return "jom";
 }
 
 fs::path third_party::patch()
 {
-	static fs::path path = find_in_root("third-party/bin/patch.exe");
-	return path;
+	return "patch";
 }
 
 fs::path third_party::git()
@@ -204,10 +212,15 @@ fs::path third_party::cmake()
 	return "cmake";
 }
 
+fs::path third_party::perl()
+{
+	return "perl";
+}
+
 
 bool conf::verbose()
 {
-	return true;
+	return false;
 }
 
 bool conf::dry()

src/conf.h

@@ -17,6 +17,12 @@ struct versions
 	static const std::string& libbsarch();
 	static const std::string& libloot();
 	static const std::string& libloot_hash();
+	static const std::string& openssl();
+};
+
+struct prebuilt
+{
+	static bool boost();
 };
 
 struct paths
@@ -28,6 +34,7 @@ struct paths
 	static fs::path install();
 	static fs::path install_bin();
 	static fs::path install_dlls();
+	static fs::path install_pdbs();
 	static fs::path install_loot();
 
 	static fs::path program_files_x86();
@@ -43,8 +50,11 @@ struct third_party
 	static fs::path patch();
 	static fs::path git();
 	static fs::path cmake();
+	static fs::path perl();
 };
 
+fs::path find_third_party_directory();
+
 struct conf
 {
 	static bool verbose();

src/net.cpp

@@ -47,48 +47,35 @@ std::string url::file() const
 }
 
 
-curl_downloader::curl_downloader(fs::path where, url u) :
-	where_(std::move(where)), url_(std::move(u)), file_(nullptr),
-	interrupt_(false)
+curl_downloader::curl_downloader()
+	: interrupt_(false), ok_(false)
 {
-	path_ = where_ / url_.file();
 }
 
-void curl_downloader::start()
+fs::path curl_downloader::path_for_url(const fs::path& where, const url& u)
 {
+	return where / u.file();
+}
+
+void curl_downloader::start(const fs::path& where, const url& u)
+{
+	where_ = where;
+	url_ = u;
+	path_ = path_for_url(where, u);
+	ok_ = false;
+
 	debug("downloading " + url_.string() + " to " + path_.string());
 
-	op::create_directories(where_);
-
-	if (fs::exists(path_))
-	{
-		debug("download " + path_.string() + " already exists");
-		return;
-	}
-
 	if (conf::dry())
 		return;
 
-	thread_ = std::thread([&]
-	{
-		try
-		{
-			run();
-		}
-		catch(bailed e)
-		{
-			bailed_ = e;
-		}
-	});
+	thread_ = std::thread([&] { run(); });
 }
 
 void curl_downloader::join()
 {
 	if (thread_.joinable())
 		thread_.join();
-
-	if (bailed_)
-		throw *bailed_;
 }
 
 void curl_downloader::interrupt()
@@ -96,6 +83,11 @@ void curl_downloader::interrupt()
 	interrupt_ = true;
 }
 
+bool curl_downloader::ok() const
+{
+	return ok_;
+}
+
 fs::path curl_downloader::file() const
 {
 	return path_;
@@ -110,37 +102,35 @@ void curl_downloader::run()
 	curl_easy_setopt(c, CURLOPT_WRITEDATA, this);
 	curl_easy_setopt(c, CURLOPT_FOLLOWLOCATION, 1l);
 
+	file_deleter output_deleter(path_);
 
 	const auto r = curl_easy_perform(c);
+	file_.reset();
 
 	if (interrupt_)
-	{
-		if (file_)
-		{
-			std::fclose(file_);
-			op::delete_file(path_);
-		}
-
 		return;
-	}
 
 	if (r == 0)
 	{
 		long h = 0;
 		curl_easy_getinfo(c, CURLINFO_RESPONSE_CODE, &h);
 
-		if (h != 200)
-			bail_out(url_.string() + " http code: " + std::to_string(h));
+		if (h == 200)
+		{
+			ok_ = true;
+			output_deleter.cancel();
+		}
+		else
+		{
+			error(url_.string() + " http code: " + std::to_string(h));
+		}
 	}
 	else
 	{
-		bail_out(url_.string() + " curl: " + curl_easy_strerror(r));
+		error(url_.string() + " curl: " + curl_easy_strerror(r));
 	}
 
 	curl_easy_cleanup(c);
-
-	if (file_)
-		std::fclose(file_);
 }
 
 size_t curl_downloader::on_write_static(
@@ -161,9 +151,12 @@ size_t curl_downloader::on_write_static(
 void curl_downloader::on_write(char* ptr, std::size_t n) noexcept
 {
 	if (!file_)
-		file_ = _wfopen(path_.native().c_str(), L"wb");
+	{
+		op::create_directories(where_);
+		file_ .reset(_wfopen(path_.native().c_str(), L"wb"));
+	}
 
-	std::fwrite(ptr, n, 1, file_);
+	std::fwrite(ptr, n, 1, file_.get());
 }
 
 }	// namespace

src/net.h

@@ -24,22 +24,25 @@ private:
 class curl_downloader
 {
 public:
-	curl_downloader(fs::path where, url u);
+	curl_downloader();
 
-	void start();
+	void start(const fs::path& where, const url& u);
 	void join();
 	void interrupt();
 
+	bool ok() const;
 	fs::path file() const;
 
+	static fs::path path_for_url(const fs::path& where, const url& u);
+
 private:
 	fs::path where_;
 	url url_;
 	fs::path path_;
-	std::FILE* file_;
+	file_ptr file_;
 	std::thread thread_;
 	std::atomic<bool> interrupt_;
-	std::optional<bailed> bailed_;
+	bool ok_;
 
 	void run();
 

src/op.cpp

@@ -228,6 +228,40 @@ bool is_source_better(const fs::path& src, const fs::path& dest)
 	return false;
 }
 
+void op::rename(const fs::path& src, const fs::path& dest)
+{
+	check(src);
+	check(dest);
+
+	if (fs::exists(dest))
+	{
+		bail_out(
+			"can't rename " + src.string() + " to " + dest.string() + ", "
+			"already exists");
+	}
+
+	debug("renaming " + src.string() + " to " + dest.string());
+	do_rename(src, dest);
+}
+
+void op::move_to_directory(const fs::path& src, const fs::path& dest_dir)
+{
+	check(src);
+	check(dest_dir);
+
+	const auto target = dest_dir / src.filename();
+
+	if (fs::exists(target))
+	{
+		bail_out(
+			"can't move " + src.string() + " to " + dest_dir.string() + ", " +
+			src.filename().string() + " already exists");
+	}
+
+	debug("moving " + src.string() + " to " + target.string());
+	do_rename(src, target);
+}
+
 void op::copy_file_to_dir_if_better(const fs::path& file, const fs::path& dir)
 {
 	check(file);
@@ -245,7 +279,7 @@ void op::copy_file_to_dir_if_better(const fs::path& file, const fs::path& dir)
 	const auto target = dir / file.filename();
 	if (is_source_better(file, target))
 	{
-		info(file.string() + " -> " + dir.string());
+		debug(file.string() + " -> " + dir.string());
 
 		if (!conf::dry())
 			do_copy_file_to_dir(file, dir);
@@ -271,7 +305,11 @@ process op::run(const std::string& cmd, const fs::path& cwd)
 
 void op::do_touch(const fs::path& p)
 {
+	create_directories(p.parent_path());
+
 	std::ofstream out(p);
+	if (!out)
+		bail_out("failed to touch " + p.string());
 }
 
 void op::do_create_directories(const fs::path& p)
@@ -337,6 +375,15 @@ void op::do_remove_readonly(const fs::path& p)
 		bail_out("can't remove read-only flag on " + p.string(), ec);
 }
 
+void op::do_rename(const fs::path& src, const fs::path& dest)
+{
+	std::error_code ec;
+	fs::rename(src, dest, ec);
+
+	if (ec)
+		bail_out("can't rename " + src.string() + " to " + dest.string(), ec);
+}
+
 process op::do_run(const std::string& what, const fs::path& cwd)
 {
 	STARTUPINFOA si = { .cb=sizeof(si) };

src/op.h

@@ -40,7 +40,9 @@ public:
 	static void delete_directory(const fs::path& p);
 	static void delete_file(const fs::path& p);
 	static void remove_readonly(const fs::path& first);
-	static void copy_file_to_dir_if_better(const fs::path& file, const fs::path& dir);
+	static void rename(const fs::path& src, const fs::path& dest);
+	static void move_to_directory(const fs::path& src, const fs::path& dest_dir);
+	static void copy_file_to_dir_if_better(const fs::path& file, const fs::path& dest_dir);
 	static process run(const std::string& cmd, const fs::path& cwd={});
 
 private:
@@ -50,6 +52,7 @@ private:
 	static void do_delete_file(const fs::path& p);
 	static void do_copy_file_to_dir(const fs::path& f, const fs::path& d);
 	static void do_remove_readonly(const fs::path& p);
+	static void do_rename(const fs::path& src, const fs::path& dest);
 	static process do_run(const std::string& cmd, const fs::path& cwd);
 	static void check(const fs::path& p);
 };

src/tools.cpp

@@ -91,25 +91,6 @@ process do_cmake(
 	return op::run(cmd, build);
 }
 
-process do_nmake(
-	const fs::path& dir,
-	const std::string& what, const std::string& args)
-{
-	std::ostringstream oss;
-	oss << "\"" << third_party::jom().string() << "\"";
-
-	if (!conf::verbose())
-		oss << " /C /S";
-
-	if (!args.empty())
-		oss << " " << args;
-
-	if (!what.empty())
-		oss << " " << what;
-
-	return op::run(oss.str(), dir);
-}
-
 
 tool::tool(std::string name)
 	: name_(std::move(name)), interrupted_(false)
@@ -179,21 +160,57 @@ void process_runner::join(bool check_exit_code)
 	}
 }
 
+int process_runner::exit_code() const
+{
+	return process_.exit_code();
+}
+
 
 downloader::downloader(url u)
-	: tool("downloader"), dl_(paths::cache(), std::move(u))
+	: tool("downloader")
+{
+	urls_.push_back(std::move(u));
+}
+
+downloader::downloader(std::vector<url> urls)
+	: tool("downloader"), urls_(std::move(urls))
 {
 }
 
-fs::path downloader::result() const
+fs::path downloader::file() const
 {
-	return dl_.file();
+	return file_;
 }
 
 void downloader::do_run()
 {
-	dl_.start();
-	dl_.join();
+	// check if one the urls has already been downloaded
+	for (auto&& u : urls_)
+	{
+		const auto file = curl_downloader::path_for_url(paths::cache(), u);
+		if (fs::exists(file))
+		{
+			file_ = file;
+			debug("download " + file_.string() + " already exists");
+			return;
+		}
+	}
+
+	// try them in order
+	for (auto&& u : urls_)
+	{
+		dl_.start(paths::cache(), std::move(u));
+		dl_.join();
+
+		if (dl_.ok())
+		{
+			file_ = dl_.file();
+			return;
+		}
+	}
+
+	// all failed
+	bail_out("all urls failed");
 }
 
 void downloader::do_interrupt()
@@ -282,25 +299,44 @@ void decompresser::do_run()
 
 	const auto sevenz = "\"" + third_party::sevenz().string() + "\"";
 
-	//op::delete_directory(where_);
 	op::create_directories(where_);
+	directory_deleter delete_output(where_);
 
 	process p;
 
+	// the -spe from 7z is supposed to figure out if there's a folder in the
+	// archive with the same name as the target and extract its content to
+	// avoid duplicating the folder
+	//
+	// however, it fails miserably if there are files along with that folder,
+	// which is the case for openssl:
+	//
+	//  openssl-1.1.1d.tar/
+	//   +- openssl-1.1.1d/
+	//   +- pax_global_header
+	//
+	// that pax_global_header makes 7z fail with "unspecified error"
+	//
+	// so the handling of a duplicate directory is done manually in
+	// check_duplicate_directory() below
+
 	if (file_.string().ends_with(".tar.gz"))
 	{
 		p = op::run(
 			sevenz + " x -so \"" + file_.string() + "\" | " +
-			sevenz + " x -aoa -spe -si -ttar -o\"" + where_.string() + "\" " + redir_nul());
+			sevenz + " x -aoa -si -ttar -o\"" + where_.string() + "\" " + redir_nul());
 	}
 	else
 	{
 		p = op::run(
-			sevenz + " x -aoa -spe -bd -bb0 -o\"" + where_.string() + "\" "
+			sevenz + " x -aoa -bd -bb0 -o\"" + where_.string() + "\" "
 			"\"" + file_.string() + "\" " + redir_nul());
 	}
 
 	execute_and_join(std::move(p));
+	check_duplicate_directory();
+
+	delete_output.cancel();
 
 	if (!interrupted())
 		op::delete_file(interrupt_file());
@@ -308,7 +344,58 @@ void decompresser::do_run()
 
 fs::path decompresser::interrupt_file() const
 {
-	return where_ / "_builder_interrupted";
+	return where_ / "_mob_interrupted";
+}
+
+void decompresser::check_duplicate_directory()
+{
+	const auto dir_name = where_.filename().string();
+
+	// check for a folder with the same name
+	if (!fs::exists(where_ / dir_name))
+		return;
+
+	// the archive contained a directory with the same name as the output
+	// directory
+
+	// delete anything other than this directory; some archives have
+	// useless files along with it
+	for (auto e : fs::directory_iterator(where_))
+	{
+		// but don't delete the directory itself
+		if (e.path().filename() == dir_name)
+			continue;
+
+		// or the interrupt file
+		if (e.path().filename() == interrupt_file().filename())
+			continue;
+
+		if (!fs::is_regular_file(e.path()))
+		{
+			// don't know what to do with archives that have the
+			// same directory _and_ other directories
+			bail_out(
+				"check_duplicate_directory: " + e.path().string() + " is "
+				"yet another directory");
+		}
+
+		op::delete_file(e.path());
+	}
+
+	// now there should only be two things in this directory: another
+	// directory with the same name and the interrupt file
+
+	// give it a temp name in case there's yet another directory with the
+	// same name in it
+	const auto temp_dir_name = where_ / ("_mob_" + dir_name );
+	op::rename(where_ / dir_name, where_ / temp_dir_name);
+
+	// move the content of the directory up
+	for (auto e : fs::directory_iterator(where_ / temp_dir_name))
+		op::move_to_directory(e.path(), where_);
+
+	// delete the old directory, which should be empty now
+	op::delete_directory(where_ / temp_dir_name);
 }
 
 
@@ -411,25 +498,36 @@ void cmake_for_vs::do_run()
 }
 
 
-nmake::nmake(fs::path dir, std::string args)
-	: process_runner("nmake"), dir_(std::move(dir)), args_(std::move(args))
+jom::jom(fs::path dir, std::string target, std::string args, flags f) :
+	process_runner("jom " + target),
+	dir_(std::move(dir)), target_(std::move(target)),
+	args_(std::move(args)), flags_(f)
 {
 }
 
-void nmake::do_run()
+void jom::do_run()
 {
-	execute_and_join(do_nmake(dir_, "", args_));
-}
+	std::ostringstream oss;
+	oss << "\"" << third_party::jom().string() << "\"";
 
-nmake_install::nmake_install(fs::path dir, std::string args) :
-	process_runner("nmake_install"),
-	dir_(std::move(dir)), args_(std::move(args))
-{
-}
+	if (!conf::verbose())
+		oss << " /C /S";
 
-void nmake_install::do_run()
-{
-	execute_and_join(do_nmake(dir_, "install", args_));
+	oss << " /K ";
+
+	if (flags_ & single_job)
+		oss << " /J 1";
+
+	if (!args_.empty())
+		oss << " " << args_;
+
+	if (!target_.empty())
+		oss << " " << target_;
+
+	oss << redir_nul();
+
+	const bool check_exit_code = !(flags_ & accept_failure);
+	execute_and_join(op::run(oss.str(), dir_), check_exit_code);
 }
 
 }	// namespace

src/tools.h

@@ -16,7 +16,6 @@ public:
 
 	void run();
 	void interrupt();
-	void result() {}
 
 protected:
 	tool(std::string name);
@@ -36,7 +35,9 @@ class downloader : public tool
 {
 public:
 	downloader(url u);
-	fs::path result() const;
+	downloader(std::vector<url> urls);
+
+	fs::path file() const;
 
 protected:
 	void do_run() override;
@@ -44,6 +45,8 @@ protected:
 
 private:
 	curl_downloader dl_;
+	fs::path file_;
+	std::vector<url> urls_;
 };
 
 
@@ -51,7 +54,9 @@ class process_runner : public tool
 {
 public:
 	process_runner(std::string name, std::string cmd, fs::path cwd={});
+
 	void join(bool check_exit_code=true);
+	int exit_code() const;
 
 protected:
 	process_runner(std::string name);
@@ -101,6 +106,7 @@ private:
 	fs::path where_;
 
 	fs::path interrupt_file() const;
+	void check_duplicate_directory();
 };
 
 
@@ -150,31 +156,28 @@ private:
 };
 
 
-class nmake : public process_runner
+class jom : public process_runner
 {
 public:
-	nmake(fs::path dir, std::string args={});
-
-protected:
-	void do_run() override;
-
-private:
-	fs::path dir_;
-	std::string args_;
-};
-
-
-class nmake_install : public process_runner
-{
-public:
-	nmake_install(fs::path dir, std::string args={});
+	enum flags
+	{
+		default_flags  = 0x00,
+		single_job     = 0x01,
+		accept_failure = 0x02
+	};
+
+	jom(
+		fs::path dir, std::string target, std::string args,
+		flags f=default_flags);
 
 protected:
 	void do_run() override;
 
 private:
 	fs::path dir_;
+	std::string target_;
 	std::string args_;
+	flags flags_;
 };
 
 }	// namespace

src/utility.cpp

@@ -121,7 +121,23 @@ std::string env(const std::string& name)
 		name.c_str(), buffer.get(), static_cast<DWORD>(buffer_size));
 
 	return buffer.get();
+}
 
+void env(const std::string& name, const std::string& value)
+{
+	::SetEnvironmentVariableA(name.c_str(), value.c_str());
+}
+
+
+void prepend_to_path(const fs::path& dir)
+{
+	if (!fs::exists(dir))
+		bail_out("can't add " + dir.string() + " to path, doesn't exist");
+
+	if (!fs::is_directory(dir))
+		bail_out("can't add " + dir.string() + " to path, not a directory");
+
+	env("PATH", dir.string() + ";" + env("PATH"));
 }
 
 std::string read_text_file(const fs::path& p)
@@ -157,4 +173,50 @@ std::string replace_all(
 	return s;
 }
 
+
+file_deleter::file_deleter(fs::path p)
+	: p_(std::move(p)), delete_(true)
+{
+}
+
+file_deleter::~file_deleter()
+{
+	if (delete_)
+		delete_now();
+}
+
+void file_deleter::delete_now()
+{
+	if (fs::exists(p_))
+		op::delete_file(p_);
+}
+
+void file_deleter::cancel()
+{
+	delete_ = false;
+}
+
+
+directory_deleter::directory_deleter(fs::path p)
+	: p_(std::move(p)), delete_(true)
+{
+}
+
+directory_deleter::~directory_deleter()
+{
+	if (delete_)
+		delete_now();
+}
+
+void directory_deleter::delete_now()
+{
+	if (fs::exists(p_))
+		op::delete_directory(p_);
+}
+
+void directory_deleter::cancel()
+{
+	delete_ = false;
+}
+
 }	// namespace

src/utility.h

@@ -21,21 +21,65 @@ private:
 };
 
 
-struct handle_deleter
+struct handle_closer
 {
 	using pointer = HANDLE;
 
 	void operator()(HANDLE h)
 	{
-		if (h != INVALID_HANDLE_VALUE) {
+		if (h != INVALID_HANDLE_VALUE)
 			::CloseHandle(h);
-		}
 	}
 };
 
-using handle_ptr = std::unique_ptr<HANDLE, handle_deleter>;
+using handle_ptr = std::unique_ptr<HANDLE, handle_closer>;
 
 
+struct file_closer
+{
+	void operator()(std::FILE* f)
+	{
+		if (f)
+			std::fclose(f);
+	}
+};
+
+using file_ptr = std::unique_ptr<FILE, file_closer>;
+
+
+class file_deleter
+{
+public:
+	file_deleter(fs::path p);
+	file_deleter(const file_deleter&) = delete;
+	file_deleter& operator=(const file_deleter&) = delete;
+	~file_deleter();
+
+	void delete_now();
+	void cancel();
+
+private:
+	fs::path p_;
+	bool delete_;
+};
+
+
+class directory_deleter
+{
+public:
+	directory_deleter(fs::path p);
+	directory_deleter(const directory_deleter&) = delete;
+	directory_deleter& operator=(const directory_deleter&) = delete;
+	~directory_deleter();
+
+	void delete_now();
+	void cancel();
+
+private:
+	fs::path p_;
+	bool delete_;
+};
+
 
 enum class level
 {
@@ -82,6 +126,8 @@ void debug(Args&&... args)
 
 
 std::string env(const std::string& name);
+void env(const std::string& name, const std::string& value);
+void prepend_to_path(const fs::path& dir);
 
 std::string redir_nul();
 

vs/props.props

@@ -19,7 +19,7 @@
       <PrecompiledHeader>Use</PrecompiledHeader>
       <PrecompiledHeaderFile>pch.h</PrecompiledHeaderFile>
       <ObjectFileName>$(IntDir)fake\%(RelativeDir)</ObjectFileName>
-      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_SILENCE_CXX17_OLD_ALLOCATOR_MEMBERS_DEPRECATION_WARNING;CURL_STATICLIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>_CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;_SILENCE_CXX17_OLD_ALLOCATOR_MEMBERS_DEPRECATION_WARNING;CURL_STATICLIB;NOMINMAX;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <AdditionalIncludeDirectories>..\third-party\include;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
     </ClCompile>
     <Link>