working on --pr

added stdin for process
added json third party
curl_downloader to string

src/cmd/build.cpp

@@ -67,6 +67,15 @@ clipp::group build_command::do_group()
 			clipp::option("--no-pull").call([&]{ nopull_ = true; })
 		) % "whether to pull repos that are already cloned; global override",
 
+		(clipp::option("--pr")
+			& clipp::value("PR") >> pr_)
+			% "checks out the branch of the given PR, must be `task/pr`, such as "
+			  "`modorganizer/123`",
+
+		(clipp::option("--github-token")
+			& clipp::value("TOKEN") >> github_token_)
+			% "github api key for --pr",
+
 		(
 			clipp::option("--revert-ts").call([&]{ revert_ts_ = true; }) |
 			clipp::option("--no-revert-ts").call([&]{ revert_ts_ = false; })
@@ -154,6 +163,13 @@ int build_command::do_run()
 	try
 	{
 		create_prefix_ini();
+
+		//if (auto r=get_pr_branch(); r != 0)
+		//	return r;
+
+		apply_pr_diff();
+
+
 		run_all_tasks();
 
 		if (do_timings)
@@ -187,6 +203,136 @@ void build_command::create_prefix_ini()
 	}
 }
 
+std::pair<const modorganizer*, std::string> build_command::parse_pr(
+	const std::string& pr) const
+{
+	if (pr.empty())
+		return {};
+
+	const auto cs = split(pr, "/");
+	if (cs.size() != 2)
+	{
+		u8cerr << "--pr must be task/pr, such as modorganizer/123\n";
+		return {};
+	}
+
+	const std::string pattern = cs[0];
+	const std::string pr_number = cs[1];
+
+	const auto tasks = find_tasks(pattern);
+
+	if (tasks.empty())
+	{
+		u8cerr << "no task matches '" << pattern << "'\n";
+		return {};
+	}
+	else if (tasks.size() > 1)
+	{
+		u8cerr
+			<< "found " << tasks.size() << " matches for pattern "
+			<< "'" << pattern << "'\n"
+			<< "the pattern must only match one task\n";
+
+		return {};
+	}
+
+	const auto* task = dynamic_cast<modorganizer*>(tasks[0]);
+	if (!task)
+	{
+		u8cerr << "only modorganizer tasks are supported\n";
+		return {};
+	}
+
+	return {task, pr_number};
+}
+
+int build_command::get_pr_branch()
+{
+	if (pr_.empty())
+		return 0;
+
+	if (github_token_.empty())
+	{
+		u8cerr << "missing --github-token\n";
+		return 1;
+	}
+
+	auto&& [task, pr] = parse_pr(pr_);
+
+	if (!task)
+		return 1;
+
+	const url u(::fmt::format(
+		"https://api.github.com/repos/{}/{}/pulls/{}",
+		task->org(), task->repo(), pr));
+
+	curl_downloader dl;
+
+	dl
+		.url(u)
+		.header("Authorization", "token " + github_token_)
+		.start()
+		.join();
+
+	if (!dl.ok())
+	{
+		u8cerr << "getting pr failed\n";
+		return 1;
+	}
+
+	const auto output = dl.steal_output();
+	u8cout << output << "\n";
+
+
+	nlohmann::json j(output);
+
+	const std::string diff_url = j["diff_url"];
+
+	return 0;
+}
+
+int build_command::apply_pr_diff()
+{
+	if (pr_.empty())
+		return 0;
+
+	auto&& [task, pr] = parse_pr(pr_);
+
+	if (!task)
+		return 1;
+
+	const url u = ::fmt::format(
+		"https://github.com/{}/{}/pull/{}.diff",
+		task->org(), task->repo(), pr);
+
+
+	//curl_downloader dl;
+	//
+	//dl
+	//	.url(u)
+	//	.header("Authorization", "token " + github_token_)
+	//	.start()
+	//	.join();
+	//
+	//if (!dl.ok())
+	//{
+	//	u8cerr << "getting pr diff failed\n";
+	//	return 1;
+	//}
+	//
+	//const auto output = dl.steal_output();
+	//u8cout << output << "\n";
+
+	std::ifstream t("c:\\tmp\\1277.diff");
+	std::string output((std::istreambuf_iterator<char>(t)),
+		std::istreambuf_iterator<char>());
+
+	git::apply(task->this_source_path(), output);
+
+	return 0;
+}
+
+
 void build_command::dump_timings()
 {
 	using namespace std::chrono;

src/cmd/commands.cpp

@@ -24,13 +24,6 @@ void set_sigint_handler()
 }
 
 
-
-std::string version()
-{
-	return "mob whatever-is-on-master";
-}
-
-
 void help(const clipp::group& g, const std::string& more)
 {
 	auto usage_df = clipp::doc_formatting()
@@ -253,7 +246,7 @@ clipp::group version_command::do_group()
 
 int version_command::do_run()
 {
-	u8cout << version() << "\n";
+	u8cout << mob_version() << "\n";
 	return 0;
 }
 

src/cmd/commands.h

@@ -6,6 +6,8 @@ namespace mob
 {
 
 class task;
+class modorganizer;
+class url;
 
 // base class for all commands
 //
@@ -218,6 +220,8 @@ private:
 	bool ignore_uncommitted_ = false;
 	bool keep_msbuild_ = false;
 	std::optional<bool> revert_ts_;
+	std::string pr_;
+	std::string github_token_;
 
 
 	// creates a bare bones ini file in the prefix so mob can be invoked in any
@@ -225,6 +229,12 @@ private:
 	//
 	void create_prefix_ini();
 
+	std::pair<const modorganizer*, std::string> parse_pr(
+		const std::string& pr) const;
+
+	int get_pr_branch();
+	int apply_pr_diff();
+
 	// for instrumentation
 	//
 	void dump_timings();

src/core/process.cpp

@@ -32,10 +32,20 @@ bool async_pipe::closed() const
 	return closed_;
 }
 
-handle_ptr async_pipe::create()
+handle_ptr async_pipe::create_for_stdout()
+{
+	return create(true);
+}
+
+handle_ptr async_pipe::create_for_stdin()
+{
+	return create(false);
+}
+
+handle_ptr async_pipe::create(bool for_stdout)
 {
 	// creating pipe
-	handle_ptr out(create_pipe());
+	handle_ptr out(for_stdout ? create_named_pipe() : create_anonymous_pipe());
 	if (out.get() == INVALID_HANDLE_VALUE)
 		return {};
 
@@ -75,7 +85,19 @@ std::string_view async_pipe::read(bool finish)
 	return s;
 }
 
-HANDLE async_pipe::create_pipe()
+std::size_t async_pipe::write(std::string_view s)
+{
+	const DWORD n = static_cast<DWORD>(s.size());
+	DWORD written = 0;
+	const auto r = ::WriteFile(stdout_.get(), s.data(), n, &written, nullptr);
+
+	if (written >= s.size())
+		stdout_ = {};
+
+	return written;
+}
+
+HANDLE async_pipe::create_named_pipe()
 {
 	const auto pipe_id = g_next_pipe_id.fetch_add(1) + 1;
 
@@ -124,7 +146,6 @@ HANDLE async_pipe::create_pipe()
 		stdout_.reset(output_read);
 	}
 
-
 	// creating handle to pipe which is passed to CreateProcess()
 	HANDLE output_write = ::CreateFileW(
 		pipe_name.c_str(), FILE_WRITE_DATA|SYNCHRONIZE, 0,
@@ -140,6 +161,34 @@ HANDLE async_pipe::create_pipe()
 	return output_write;
 }
 
+HANDLE async_pipe::create_anonymous_pipe()
+{
+	SECURITY_ATTRIBUTES saAttr = {};
+	saAttr.nLength = sizeof(SECURITY_ATTRIBUTES);
+	saAttr.bInheritHandle = TRUE;
+
+	// Create a pipe for the child process's STDIN.
+	HANDLE read_pipe, write_pipe;
+	if (!CreatePipe(&read_pipe, &write_pipe, &saAttr, 0))
+	{
+		const auto e = GetLastError();
+		cx_.bail_out(context::cmd,
+			"CreatePipe failed, {}", error_message(e));
+	}
+
+	// Ensure the write handle to the pipe for STDIN is not inherited.
+	if (!SetHandleInformation(write_pipe, HANDLE_FLAG_INHERIT, 0))
+	{
+		const auto e = GetLastError();
+		cx_.bail_out(context::cmd,
+			"SetHandleInformation failed, {}", error_message(e));
+	}
+
+	stdout_.reset(write_pipe);
+
+	return read_pipe;
+}
+
 std::string_view async_pipe::try_read()
 {
 	DWORD bytes_read = 0;
@@ -248,6 +297,8 @@ process::impl& process::impl::operator=(const impl& i)
 	interrupt = i.interrupt.load();
 	stdout_pipe = {};
 	stderr_pipe = {};
+	stdin_pipe = {};
+	stdin_handle = {};
 
 	return *this;
 }
@@ -255,7 +306,8 @@ process::impl& process::impl::operator=(const impl& i)
 
 process::process() :
 	cx_(&gcx()), unicode_(false), chcp_(-1), flags_(process::noflags),
-	stdout_(context::level::trace), stderr_(context::level::error), code_(0)
+	stdout_(context::level::trace), stderr_(context::level::error),
+	stdin_offset_(0), code_(0)
 {
 	success_.insert(0);
 }
@@ -363,6 +415,12 @@ process& process::stderr_encoding(encodings e)
 	return *this;
 }
 
+process& process::stdin_string(std::string s)
+{
+	stdin_ = s;
+	return *this;
+}
+
 process& process::cmd_unicode(bool b)
 {
 	unicode_ = b;
@@ -484,7 +542,7 @@ void process::do_run(const std::string& what)
 	STARTUPINFOW si = { .cb=sizeof(si) };
 	PROCESS_INFORMATION pi = {};
 
-	handle_ptr stdout_pipe, stderr_pipe, stdin_pipe;
+	handle_ptr stdout_pipe, stderr_pipe;
 
 	impl_.stdout_pipe.reset(new async_pipe(*cx_));
 	impl_.stderr_pipe.reset(new async_pipe(*cx_));
@@ -494,7 +552,7 @@ void process::do_run(const std::string& what)
 		case forward_to_log:
 		case keep_in_string:
 		{
-			stdout_pipe = impl_.stdout_pipe->create();
+			stdout_pipe = impl_.stdout_pipe->create_for_stdout();
 			si.hStdOutput = stdout_pipe.get();
 			break;
 		}
@@ -517,7 +575,7 @@ void process::do_run(const std::string& what)
 		case forward_to_log:
 		case keep_in_string:
 		{
-			stderr_pipe = impl_.stderr_pipe->create();
+			stderr_pipe = impl_.stderr_pipe->create_for_stdout();
 			si.hStdError = stderr_pipe.get();
 			break;
 		}
@@ -535,9 +593,17 @@ void process::do_run(const std::string& what)
 		}
 	}
 
-	stdin_pipe.reset(get_bit_bucket());
-	si.hStdInput = stdin_pipe.get();
+	if (stdin_)
+	{
+		impl_.stdin_pipe.reset(new async_pipe(*cx_));
+		impl_.stdin_handle = impl_.stdin_pipe->create_for_stdin();
+	}
+	else
+	{
+		impl_.stdin_handle.reset(get_bit_bucket());
+	}
 
+	si.hStdInput = impl_.stdin_handle.get();
 	si.dwFlags = STARTF_USESTDHANDLES;
 
 	const std::wstring cmd = utf8_to_utf16(this_env::get("COMSPEC"));
@@ -765,6 +831,12 @@ void process::on_timeout(bool& already_interrupted)
 {
 	read_pipes(false);
 
+	if (stdin_ && stdin_offset_ < stdin_->size())
+	{
+		stdin_offset_ += impl_.stdin_pipe->write({
+			stdin_->data() + stdin_offset_, stdin_->size() - stdin_offset_});
+	}
+
 	if (impl_.interrupt && !already_interrupted)
 	{
 		const auto pid = GetProcessId(impl_.handle.get());

src/core/process.h

@@ -15,8 +15,12 @@ class async_pipe
 public:
 	async_pipe(const context& cx);
 
-	handle_ptr create();
+	handle_ptr create_for_stdout();
 	std::string_view read(bool finish);
+
+	handle_ptr create_for_stdin();
+	std::size_t write(std::string_view s);
+
 	bool closed() const;
 
 private:
@@ -30,7 +34,10 @@ private:
 	bool pending_;
 	bool closed_;
 
-	HANDLE create_pipe();
+	handle_ptr create(bool for_stdout);
+	HANDLE create_named_pipe();
+	HANDLE create_anonymous_pipe();
+
 	std::string_view try_read();
 	std::string_view check_pending();
 };
@@ -257,6 +264,8 @@ public:
 	process& stderr_filter(filter_fun f);
 	process& stderr_encoding(encodings e);
 
+	process& stdin_string(std::string s);
+
 	process& chcp(int cp);
 	process& cmd_unicode(bool b);
 
@@ -317,6 +326,8 @@ private:
 		std::atomic<bool> interrupt{false};
 		std::unique_ptr<async_pipe> stdout_pipe;
 		std::unique_ptr<async_pipe> stderr_pipe;
+		std::unique_ptr<async_pipe> stdin_pipe;
+		handle_ptr stdin_handle;
 
 		impl() = default;
 		impl(const impl&);
@@ -347,6 +358,8 @@ private:
 	std::set<int> success_;
 	stream stdout_;
 	stream stderr_;
+	std::optional<std::string> stdin_;
+	std::size_t stdin_offset_;
 	mob::env env_;
 	std::string raw_;
 	std::string cmd_;

src/net.cpp

@@ -83,24 +83,49 @@ curl_downloader::curl_downloader(const context* cx)
 {
 }
 
-void curl_downloader::start(const url& u, const fs::path& path)
+void curl_downloader::start(const mob::url& u, const fs::path& path)
+{
+	url(u);
+	file(path);
+	start();
+}
+
+curl_downloader& curl_downloader::url(const mob::url& u)
 {
 	url_ = u;
-	path_ = path;
-	ok_ = false;
+	return *this;
+}
 
+curl_downloader& curl_downloader::file(const fs::path& file)
+{
+	path_ = file;
+	return *this;
+}
+
+curl_downloader& curl_downloader::header(std::string name, std::string value)
+{
+	headers_.emplace_back(std::move(name), std::move(value));
+	return *this;
+}
+
+curl_downloader& curl_downloader::start()
+{
+	ok_ = false;
 	cx_.debug(context::net, "downloading {} to {}", url_, path_);
 
 	if (conf::dry())
-		return;
+		return *this;
 
 	thread_ = start_thread([&]{ run(); });
+	return *this;
 }
 
-void curl_downloader::join()
+curl_downloader& curl_downloader::join()
 {
 	if (thread_.joinable())
 		thread_.join();
+
+	return *this;
 }
 
 void curl_downloader::interrupt()
@@ -114,6 +139,18 @@ bool curl_downloader::ok() const
 	return ok_;
 }
 
+const std::string& curl_downloader::output()
+{
+	return output_;
+}
+
+std::string curl_downloader::steal_output()
+{
+	std::string s = std::move(output_);
+	output_.clear();
+	return s;
+}
+
 void curl_downloader::run()
 {
 	cx_.trace(context::net, "curl: initializing {}", url_);
@@ -122,6 +159,8 @@ void curl_downloader::run()
 	guard g([&]{ curl_easy_cleanup(c); });
 
 	char error_buffer[CURL_ERROR_SIZE + 1] = {};
+	const std::string ua =
+		"ModOrganizer's " + mob_version() + " " + curl_version();
 
 	curl_easy_setopt(c, CURLOPT_URL, url_.c_str());
 	curl_easy_setopt(c, CURLOPT_WRITEFUNCTION, on_write_static);
@@ -133,6 +172,7 @@ void curl_downloader::run()
 	curl_easy_setopt(c, CURLOPT_NOPROGRESS, 0l);
 	curl_easy_setopt(c, CURLOPT_FOLLOWLOCATION, 1l);
 	curl_easy_setopt(c, CURLOPT_ERRORBUFFER, error_buffer);
+	curl_easy_setopt(c, CURLOPT_USERAGENT, ua.c_str());
 
 	if (context::enabled(context::level::dump))
 	{
@@ -143,7 +183,9 @@ void curl_downloader::run()
 
 
 	// deletes the file in dtor unless cancel() is called
-	file_deleter output_deleter(cx_, path_);
+	std::unique_ptr<file_deleter> output_deleter;
+	if (!path_.empty())
+		output_deleter.reset(new file_deleter(cx_, path_));
 
 	cx_.trace(context::net, "curl: performing {}", url_);
 	const auto r = curl_easy_perform(c);
@@ -175,7 +217,9 @@ void curl_downloader::run()
 				url_, bytes_);
 
 			ok_ = true;
-			output_deleter.cancel();
+
+			if (output_deleter)
+				output_deleter->cancel();
 		}
 		else
 		{
@@ -214,34 +258,55 @@ size_t curl_downloader::on_write_static(
 
 void curl_downloader::on_write(char* ptr, std::size_t n) noexcept
 {
-	if (!file_)
+	if (!create_file())
 	{
-		// file is lazily created on first write
-
-		op::create_directories(cx_, path_.parent_path());
-
-		cx_.trace(context::net, "opening {}", path_);
-
-		HANDLE h = ::CreateFileW(
-			path_.native().c_str(), GENERIC_WRITE, FILE_SHARE_READ,
-			nullptr, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, 0);
-
-		if (h == INVALID_HANDLE_VALUE)
-		{
-			const auto e = GetLastError();
-
-			cx_.error(context::net,
-				"failed to open {}, {}", path_, error_message(e));
-
-			interrupt_ = true;
-			return;
-		}
-
-		file_.reset(h);
+		interrupt_ = true;
+		return;
 	}
 
-	bytes_ += n;
+	bool b = false;
+	if (file_)
+		b = write_file(ptr, n);
+	else
+		b = write_string(ptr, n);
 
+	if (!b)
+		interrupt_ = true;
+
+	bytes_ += n;
+}
+
+bool curl_downloader::create_file()
+{
+	if (file_ || path_.empty())
+		return true;
+
+	// file is lazily created on first write
+
+	op::create_directories(cx_, path_.parent_path());
+
+	cx_.trace(context::net, "opening {}", path_);
+
+	HANDLE h = ::CreateFileW(
+		path_.native().c_str(), GENERIC_WRITE, FILE_SHARE_READ,
+		nullptr, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, 0);
+
+	if (h == INVALID_HANDLE_VALUE)
+	{
+		const auto e = GetLastError();
+
+		cx_.error(context::net,
+			"failed to open {}, {}", path_, error_message(e));
+
+		return false;
+	}
+
+	file_.reset(h);
+	return true;
+}
+
+bool curl_downloader::write_file(char* ptr, size_t n)
+{
 	DWORD written = 0;
 	if (!::WriteFile(file_.get(), ptr, static_cast<DWORD>(n), &written, nullptr))
 	{
@@ -250,8 +315,16 @@ void curl_downloader::on_write(char* ptr, std::size_t n) noexcept
 		cx_.error(context::net,
 			"failed to write to {}, {}", path_, error_message(e));
 
-		interrupt_ = true;
+		return false;
 	}
+
+	return true;
+}
+
+bool curl_downloader::write_string(char* ptr, size_t n)
+{
+	output_.append(ptr, n);
+	return true;
 }
 
 int curl_downloader::on_progress_static(

src/net.h

@@ -45,35 +45,66 @@ private:
 class curl_downloader
 {
 public:
+	using headers = std::vector<std::pair<std::string, std::string>>;
+
 	curl_downloader(const context* cx=nullptr);
 
-	// starts a thread, downloads url into given file
+	// convenience: starts a thread, downloads url into given file
 	//
-	void start(const url& u, const fs::path& file);
+	void start(const mob::url& u, const fs::path& file);
+
+
+	// sets the url to download from
+	//
+	curl_downloader& url(const mob::url& u);
+
+	// sets the output file
+	//
+	curl_downloader& file(const fs::path& file);
+
+	// adds a header
+	//
+	curl_downloader& header(std::string name, std::string value);
+
+	// starts the download in a thread
+	//
+	curl_downloader& start();
+
 
 	// joins download thread
 	//
-	void join();
+	curl_downloader& join();
 
 	// async interrupt
 	//
 	void interrupt();
 
+
 	// whether the file was downloaded correctly; only valid after join()
 	//
 	bool ok() const;
 
+	// if file() wasn't called, returns the content that was retrieved
+	//
+	const std::string& output();
+	std::string steal_output();
+
 private:
 	const context& cx_;
-	url url_;
+	mob::url url_;
 	fs::path path_;
 	handle_ptr file_;
 	std::thread thread_;
 	std::size_t bytes_;
 	std::atomic<bool> interrupt_;
 	bool ok_;
+	std::string output_;
+	headers headers_;
 
 	void run();
+	bool create_file();
+	bool write_file(char* ptr, size_t size);
+	bool write_string(char* ptr, size_t size);
 
 	static size_t on_write_static(
 		char* ptr, size_t size, size_t nmemb, void* user) noexcept;

src/pch.h

@@ -71,6 +71,7 @@
 #include <curl/curl.h>
 #include <clipp.h>
 #include <fmt/format.h>
+#include <nlohmann/json.hpp>
 
 #pragma warning(pop)
 

src/tasks/modorganizer.cpp

@@ -80,6 +80,16 @@ url modorganizer::git_url() const
 	return task_conf().make_git_url(task_conf().mo_org(), repo_);
 }
 
+std::string modorganizer::org() const
+{
+	return task_conf().mo_org();
+}
+
+std::string modorganizer::repo() const
+{
+	return repo_;
+}
+
 void modorganizer::do_clean(clean c)
 {
 	instrument<times::clean>([&]

src/tasks/tasks.h

@@ -306,7 +306,12 @@ public:
 
 	bool is_super() const override;
 	bool is_gamebryo_plugin() const;
+
 	url git_url() const;
+	std::string org() const;
+	std::string repo() const;
+	fs::path this_source_path() const;
+	fs::path this_solution_path() const;
 
 protected:
 	void do_clean(clean c) override;
@@ -321,8 +326,6 @@ private:
 	msbuild create_this_msbuild_tool(msbuild::ops o=msbuild::build);
 	void initialize_super(const fs::path& super_root);
 
-	fs::path this_source_path() const;
-	fs::path this_solution_path() const;
 };
 
 

src/tools/git.cpp

@@ -111,6 +111,24 @@ void git::init_repo(const fs::path& p)
 	g.init();
 }
 
+void git::apply(const fs::path& p, const std::string& diff)
+{
+	git g(no_op);
+	g.root(p);
+	g.apply(diff);
+}
+
+void git::apply(const std::string& diff)
+{
+	process_ = make_process()
+		.stdin_string(diff)
+		.arg("apply")
+		.arg("-")
+		.cwd(root_);
+
+	execute_and_join();
+}
+
 fs::path git::binary()
 {
 	return conf::tool_by_name("git");

src/tools/tools.h

@@ -183,6 +183,8 @@ public:
 	static bool branch_exists(const mob::url& u, const std::string& name);
 	static void init_repo(const fs::path& p);
 
+	static void apply(const fs::path& p, const std::string& diff);
+
 
 	git& url(const mob::url& u);
 	git& branch(const std::string& name);
@@ -237,6 +239,7 @@ private:
 	void add_remote(const std::string& name, const std::string& url);
 	void set_remote_push(const std::string& remote, const std::string& url);
 	void set_assume_unchanged(const fs::path& relative_file, bool on);
+	void apply(const std::string& diff);
 	bool is_repo();
 	bool branch_exists();
 	bool has_uncommitted_changes();

src/utility.cpp

@@ -9,6 +9,11 @@
 namespace mob
 {
 
+std::string mob_version()
+{
+	return "mob 5.0";
+}
+
 url make_prebuilt_url(const std::string& filename)
 {
 	return

src/utility.h

@@ -183,4 +183,8 @@ url make_prebuilt_url(const std::string& filename);
 url make_appveyor_artifact_url(
 	arch a, const std::string& project, const std::string& filename);
 
+// returns "mob x.y"
+//
+std::string mob_version();
+
 }	// namespace