fixed __FILE__ for unicode

fixed utf8 for msbuild and cmake
process now uses encoded_buffer
unicode command line
fixed paths from command line and ini being interpreted as ACP

src/conf.cpp

@@ -247,6 +247,13 @@ bool parse_value<bool>(const std::string& s, bool& out)
 	return !iss.bad();
 }
 
+template <>
+bool parse_value<fs::path>(const std::string& s, fs::path& out)
+{
+	out = utf8_to_utf16(s);
+	return true;
+}
+
 template <class Map>
 bool set_option_impl(
 	Map& map, const std::string& key, const std::string& value)
@@ -913,6 +920,7 @@ void init_options(const fs::path& ini, const std::vector<std::string>& opts)
 	find_vcvars();
 	validate_qt();
 
+	make_canonical_path("prefix",           fs::current_path(), "");
 	make_canonical_path("cache",            paths::prefix(), "downloads");
 	make_canonical_path("build",            paths::prefix(), "build");
 	make_canonical_path("install",          paths::prefix(), "install");

src/env.cpp

@@ -32,7 +32,7 @@ env get_vcvars_env(arch a)
 	const fs::path tmp = make_temp_file();
 
 	// "vcvarsall.bat" amd64 && set > temp_file
-	std::string cmd =
+	const std::string cmd =
 		"\"" + path_to_utf8(tools::vs::vcvars()) + "\" " + arch_s +
 		" && set > \"" + path_to_utf8(tmp) + "\"";
 

src/main.cpp

@@ -25,7 +25,7 @@ void show_help(const clipp::group& g)
 		.doc_column(30));
 }
 
-std::optional<int> handle_command_line(int argc, char** argv)
+std::optional<int> handle_command_line(const std::vector<std::string>& args)
 {
 	struct
 	{
@@ -105,7 +105,7 @@ std::optional<int> handle_command_line(int argc, char** argv)
 	);
 
 
-	const auto pr = clipp::parse(argc, argv, g);
+	const auto pr = clipp::parse(args, g);
 
 	if (!pr)
 	{
@@ -270,13 +270,13 @@ void add_tasks()
 		.add_task<modorganizer>("modorganizer");
 }
 
-int run(int argc, char** argv)
+int run(const std::vector<std::string>& args)
 {
 	add_tasks();
 
 	try
 	{
-		if (auto r=handle_command_line(argc, argv))
+		if (auto r=handle_command_line(args))
 			return *r;
 	}
 	catch(bailed&)
@@ -310,13 +310,17 @@ int run(int argc, char** argv)
 } // namespace
 
 
-int main(int argc, char** argv)
+int wmain(int argc, wchar_t** argv)
 {
 	_setmode(_fileno(stdout), _O_U16TEXT);
 
 	try
 	{
-		int r = mob::run(argc, argv);
+		std::vector<std::string> args;
+		for (int i=1; i<argc; ++i)
+			args.push_back(mob::utf16_to_utf8(argv[i]));
+
+		int r = mob::run(args);
 
 		if (r == 0)
 		{

src/op.cpp

@@ -391,30 +391,9 @@ std::string read_text_file(
 	if (bytes.empty())
 		return bytes;
 
-	std::string utf8;
-
-	switch (e)
-	{
-		case encodings::utf16:
-		{
-			const auto* wbuf = reinterpret_cast<const wchar_t*>(bytes.data());
-			const std::size_t n = bytes.size() / sizeof(wchar_t);
-			const std::wstring ws(wbuf, wbuf + n);
-
-			utf8 = utf16_to_utf8(ws);
-			break;
-		}
-
-		case encodings::utf8:
-		case encodings::dont_know:
-		default:
-		{
-			utf8 = std::move(bytes);
-			break;
-		}
-	}
-
+	std::string utf8 = bytes_to_utf8(e, bytes);
 	utf8 = replace_all(utf8, "\r\n", "\n");
+
 	return utf8;
 }
 

src/pch.cpp

@@ -1,2 +1,4 @@
 #include "pch.h"
+
+#pragma warning(disable: 4566)  // character cannot be represented in CP
 #include <fmt/format.cc>

src/pch.h

@@ -43,6 +43,7 @@
 #pragma warning(disable: 4242)  // possible loss of data
 #pragma warning(disable: 4244)  // possible loss of data
 #pragma warning(disable: 4275)  // non dll-interface base
+#pragma warning(disable: 4566)  // character cannot be represented in CP
 
 
 #include <string>

src/process.cpp

@@ -55,6 +55,9 @@ handle_ptr async_pipe::create()
 
 std::string_view async_pipe::read()
 {
+	if (closed_)
+		return {};
+
 	if (pending_)
 		return check_pending();
 	else
@@ -229,7 +232,7 @@ process::impl& process::impl::operator=(const impl& i)
 
 
 process::process()
-	: cx_(&gcx()), unicode_(false), flags_(process::noflags), code_(0)
+	: cx_(&gcx()), unicode_(false), chcp_(-1), flags_(process::noflags), code_(0)
 {
 }
 
@@ -312,6 +315,12 @@ process& process::stdout_filter(filter_fun f)
 	return *this;
 }
 
+process& process::stdout_encoding(encodings e)
+{
+	stdout_.encoding = e;
+	return *this;
+}
+
 process& process::stderr_flags(stream_flags s)
 {
 	stderr_.flags = s;
@@ -330,9 +339,28 @@ process& process::stderr_filter(filter_fun f)
 	return *this;
 }
 
+process& process::stderr_encoding(encodings e)
+{
+	stderr_.encoding = e;
+	return *this;
+}
+
 process& process::cmd_unicode(bool b)
 {
 	unicode_ = b;
+
+	if (b)
+	{
+		stdout_.encoding = encodings::utf16;
+		stderr_.encoding = encodings::utf16;
+	}
+
+	return *this;
+}
+
+process& process::chcp(int i)
+{
+	chcp_ = i;
 	return *this;
 }
 
@@ -407,6 +435,9 @@ void process::do_run(const std::string& what)
 		op::delete_file(*cx_, error_log_file_, op::optional);
 	}
 
+	stdout_.buffer = encoded_buffer(stdout_.encoding);
+	stderr_.buffer = encoded_buffer(stderr_.encoding);
+
 	STARTUPINFOW si = { .cb=sizeof(si) };
 	PROCESS_INFORMATION pi = {};
 
@@ -488,7 +519,7 @@ void process::do_run(const std::string& what)
 	{
 		const auto e = GetLastError();
 		cx_->bail_out(context::cmd,
-			"failed to start '{}', {}", cmd, error_message(e));
+			"failed to start '{}', {}", args, error_message(e));
 	}
 
 	cx_->trace(context::cmd, "pid {}", pi.dwProcessId);
@@ -504,7 +535,12 @@ std::wstring process::make_cmd_args(const std::string& what) const
 	if (unicode_)
 		s += L"/U ";
 
-	s += L"/C \"" + utf8_to_utf16(what) + L"\"";
+	s += L"/C \"";
+
+	if (chcp_ != -1)
+		s += L"chcp " + std::to_wstring(chcp_) + L" && ";
+
+	s += utf8_to_utf16(what) + L"\"";
 
 	return s;
 }
@@ -551,21 +587,22 @@ void process::join()
 		cx_->trace(context::cmd, "process interrupted and finished");
 }
 
-void process::read_pipes()
+void process::read_pipes(bool finish)
 {
-	read_pipe(stdout_, impl_.stdout_pipe, context::std_out);
-	read_pipe(stderr_, impl_.stderr_pipe, context::std_err);
+	read_pipe(finish, stdout_, impl_.stdout_pipe, context::std_out);
+	read_pipe(finish, stderr_, impl_.stderr_pipe, context::std_err);
 }
 
-void process::read_pipe(stream& s, async_pipe& pipe, context::reason r)
+void process::read_pipe(
+	bool finish, stream& s, async_pipe& pipe, context::reason r)
 {
 	switch (s.flags)
 	{
 		case forward_to_log:
 		{
-			const std::string_view buffer = pipe.read();
+			s.buffer.add(pipe.read());
 
-			for_each_line(buffer, [&](auto&& line)
+			s.buffer.next_utf8_lines(finish, [&](auto&& line)
 			{
 				filter f = {line, r, s.level, false};
 
@@ -579,15 +616,12 @@ void process::read_pipe(stream& s, async_pipe& pipe, context::reason r)
 				cx_->log(f.r, f.lv, "{}", f.line);
 			});
 
-			s.string.append(buffer.begin(), buffer.end());
-
 			break;
 		}
 
 		case keep_in_string:
 		{
-			const std::string_view buffer = pipe.read();
-			s.string.append(buffer.begin(), buffer.end());
+			s.buffer.add(pipe.read());
 			break;
 		}
 
@@ -602,12 +636,15 @@ void process::on_completed()
 	// one last time
 	for (;;)
 	{
-		read_pipes();
+		read_pipes(false);
 
 		if (impl_.stdout_pipe.closed() && impl_.stderr_pipe.closed())
 			break;
 	}
 
+	read_pipes(true);
+
+
 	if (impl_.interrupt)
 		return;
 
@@ -643,7 +680,7 @@ void process::on_completed()
 
 void process::on_timeout(bool& already_interrupted)
 {
-	read_pipes();
+	read_pipes(false);
 
 	if (impl_.interrupt && !already_interrupted)
 	{
@@ -715,12 +752,14 @@ void process::dump_stderr() noexcept
 {
 	try
 	{
-		if (!stderr_.string.empty())
+		const std::string s = stderr_.buffer.utf8_string();
+
+		if (!s.empty())
 		{
 			cx_->error(context::cmd,
 				"{} failed, content of stderr:", make_name());
 
-			for_each_line(stderr_.string, [&](auto&& line)
+			for_each_line(s, [&](auto&& line)
 			{
 				cx_->error(context::cmd, "        {}", line);
 			});
@@ -744,12 +783,12 @@ int process::exit_code() const
 
 std::string process::stdout_string()
 {
-	return stdout_.string;
+	return stdout_.buffer.utf8_string();
 }
 
 std::string process::stderr_string()
 {
-	return stderr_.string;
+	return stderr_.buffer.utf8_string();
 }
 
 void process::add_arg(const std::string& k, const std::string& v, arg_flags f)
@@ -806,4 +845,100 @@ std::string process::arg_to_string(const url& u, bool force_quote)
 		return u.string();
 }
 
+
+encoded_buffer::encoded_buffer(encodings e, std::string bytes)
+	: e_(e), bytes_(std::move(bytes)), last_(0)
+{
+}
+
+void encoded_buffer::add(std::string_view bytes)
+{
+	bytes_.append(bytes.begin(), bytes.end());
+}
+
+std::string encoded_buffer::utf8_string() const
+{
+	return bytes_to_utf8(e_, bytes_);
+}
+
+template <class CharT>
+std::basic_string<CharT> next_line(
+	bool finished, std::string_view bytes, std::size_t& byte_offset)
+{
+	std::size_t size = bytes.size();
+	if ((size & 1) == 1)
+		--size;
+
+	const CharT* start = reinterpret_cast<const CharT*>(bytes.data() + byte_offset);
+	const CharT* end = reinterpret_cast<const CharT*>(bytes.data() + size);
+	const CharT* p = start;
+
+	std::basic_string<CharT> line;
+
+	while (p != end)
+	{
+		if (*p == CharT('\n') || *p == CharT('\r'))
+		{
+			line.assign(start, static_cast<std::size_t>(p - start));
+
+			while (p != end && (*p == CharT('\n') || *p == CharT('\r')))
+				++p;
+
+			if (!line.empty())
+				break;
+
+			start = p;
+		}
+		else
+		{
+			++p;
+		}
+	}
+
+	if (line.empty() && finished)
+	{
+		line = {
+			reinterpret_cast<const wchar_t*>(bytes.data() + byte_offset),
+			reinterpret_cast<const wchar_t*>(bytes.data() + size)
+		};
+
+		byte_offset = bytes.size();
+	}
+	else
+	{
+		byte_offset = static_cast<std::size_t>(
+			reinterpret_cast<const char*>(p) - bytes.data());
+
+		MOB_ASSERT(byte_offset <= bytes.size());
+	}
+
+	return line;
+}
+
+std::string encoded_buffer::next_utf8_line(bool finished)
+{
+	switch (e_)
+	{
+		case encodings::utf16:
+		{
+			const std::wstring utf16 = next_line<wchar_t>(finished, bytes_, last_);
+			return utf16_to_utf8(utf16);
+		}
+
+		case encodings::acp:
+		case encodings::oem:
+		{
+			const std::string cp = next_line<char>(finished, bytes_, last_);
+			return bytes_to_utf8(e_, cp);
+		}
+
+		case encodings::utf8:
+		case encodings::dont_know:
+		default:
+		{
+			return next_line<char>(finished, bytes_, last_);
+		}
+	}
+}
+
 }	// namespace

src/process.h

@@ -35,6 +35,37 @@ private:
 };
 
 
+class encoded_buffer
+{
+public:
+	encoded_buffer(encodings e=encodings::dont_know, std::string bytes={});
+
+	void add(std::string_view bytes);
+
+	std::string utf8_string() const;
+
+	template <class F>
+	void next_utf8_lines(bool finished, F&& f)
+	{
+		for (;;)
+		{
+			std::string line = next_utf8_line(finished);
+			if (line.empty())
+				break;
+
+			f(line);
+		}
+	}
+
+private:
+	encodings e_;
+	std::string bytes_;
+	std::size_t last_;
+
+	std::string next_utf8_line(bool finished);
+};
+
+
 class process
 {
 public:
@@ -108,11 +139,14 @@ public:
 	process& stdout_flags(stream_flags s);
 	process& stdout_level(context::level lv);
 	process& stdout_filter(filter_fun f);
+	process& stdout_encoding(encodings e);
 
 	process& stderr_flags(stream_flags s);
 	process& stderr_level(context::level lv);
 	process& stderr_filter(filter_fun f);
+	process& stderr_encoding(encodings e);
 
+	process& chcp(int cp);
 	process& cmd_unicode(bool b);
 
 	process& external_error_log(const fs::path& p);
@@ -179,7 +213,8 @@ private:
 		stream_flags flags = forward_to_log;
 		context::level level = context::level::trace;
 		filter_fun filter;
-		std::string string;
+		encodings encoding = encodings::dont_know;
+		encoded_buffer buffer;
 	};
 
 	const context* cx_;
@@ -187,6 +222,7 @@ private:
 	fs::path bin_;
 	fs::path cwd_;
 	bool unicode_;
+	int chcp_;
 	flags_t flags_;
 	stream stdout_;
 	stream stderr_;
@@ -204,8 +240,8 @@ private:
 	void pipe_into(const process& p);
 
 	void do_run(const std::string& what);
-	void read_pipes();
-	void read_pipe(stream& s, async_pipe& pipe, context::reason r);
+	void read_pipes(bool finish);
+	void read_pipe(bool finish, stream& s, async_pipe& pipe, context::reason r);
 
 	void on_completed();
 	void on_timeout(bool& already_interrupted);

src/tools/cmake.cpp

@@ -81,6 +81,8 @@ void cmake::do_run()
 	output_ = root_ / (g.output_dir(arch_));
 
 	process_
+		.stdout_encoding(encodings::utf8)
+		.stderr_encoding(encodings::utf8)
 		.arg("-G", "\"" + g.name + "\"")
 		.arg("-DCMAKE_BUILD_TYPE=Release")
 		.arg("-DCMAKE_INSTALL_MESSAGE=NEVER", process::log_quiet)

src/tools/msbuild.cpp

@@ -96,6 +96,9 @@ void msbuild::do_run()
 
 	process_
 		.binary(tools::msbuild::binary())
+		.chcp(65001)
+		.stdout_encoding(encodings::utf8)
+		.stderr_encoding(encodings::utf8)
 		.arg("-nologo");
 
 	if ((flags_ & single_job) == 0)

src/utility.cpp

@@ -21,19 +21,19 @@ void u8stream::do_output(const std::string& s)
 
 void mob_assertion_failed(
 	const char* message,
-	const char* exp, const char* file, int line, const char* func)
+	const char* exp, const wchar_t* file, int line, const char* func)
 {
 	if (message)
 	{
 		gcx().error(context::generic,
 			"assertion failed: {}:{} {}: {} ({})",
-			file, line, func, message, exp);
+			std::wstring(file), line, func, message, exp);
 	}
 	else
 	{
 		gcx().error(context::generic,
 			"assertion failed: {}:{} {}: '{}'",
-			file, line, func, exp);
+			std::wstring(file), line, func, exp);
 	}
 
 	if (IsDebuggerPresent())
@@ -457,9 +457,9 @@ std::string utf16_to_utf8(std::wstring_view ws)
 	return *s;
 }
 
-std::wstring cp_to_utf16(std::string_view s)
+std::wstring cp_to_utf16(UINT from, std::string_view s)
 {
-	auto ws = to_utf16(CP_ACP, s);
+	auto ws = to_utf16(from, s);
 	if (!ws)
 	{
 		std::cerr << "can't convert from acp to utf16\n";
@@ -469,20 +469,39 @@ std::wstring cp_to_utf16(std::string_view s)
 	return *ws;
 }
 
-std::string cp_to_utf8(std::string_view s)
+std::string bytes_to_utf8(encodings e, std::string_view s)
 {
-	auto ws = cp_to_utf16(s);
-	auto s8 = to_utf8(ws);
-
-	if (!s8)
+	switch (e)
 	{
-		std::cerr << "can't convert from acp to utf8\n";
-		return "???";
-	}
+		case encodings::utf16:
+		{
+			const auto* ws = reinterpret_cast<const wchar_t*>(s.data());
+			const auto chars = s.size() / sizeof(wchar_t);
+			return utf16_to_utf8({ws, chars});
+		}
 
-	return *s8;
+		case encodings::acp:
+		{
+			const std::wstring utf16 = cp_to_utf16(CP_ACP, s);
+			return utf16_to_utf8(utf16);
+		}
+
+		case encodings::oem:
+		{
+			const std::wstring utf16 = cp_to_utf16(CP_OEMCP, s);
+			return utf16_to_utf8(utf16);
+		}
+
+		case encodings::utf8:
+		case encodings::dont_know:
+		default:
+		{
+			return {s.begin(), s.end()};
+		}
+	}
 }
 
+
 std::string path_to_utf8(fs::path p)
 {
 	return utf16_to_utf8(p.native());

src/utility.h

@@ -7,17 +7,21 @@ namespace mob
 	inline E operator|(E e1, E e2) { return (E)((int)e1 | (int)e2); } \
 	inline E operator|=(E& e1, E e2) { e1 = e1 | e2; return e1; }
 
+#define MOB_WIDEN2(x) L ## x
+#define MOB_WIDEN(x) MOB_WIDEN2(x)
+#define MOB_FILE_UTF16 MOB_WIDEN(__FILE__)
+
 #define MOB_ASSERT(x, ...) \
-	mob_assert(x, __VA_ARGS__, #x, __FILE__, __LINE__, __FUNCSIG__);
+	mob_assert(x, __VA_ARGS__, #x, MOB_FILE_UTF16, __LINE__, __FUNCSIG__);
 
 void mob_assertion_failed(
 	const char* message,
-	const char* exp, const char* file, int line, const char* func);
+	const char* exp, const wchar_t* file, int line, const char* func);
 
 template <class X>
 inline void mob_assert(
 	X&& x, const char* message,
-	const char* exp, const char* file, int line, const char* func)
+	const char* exp, const wchar_t* file, int line, const char* func)
 {
 	if (!(x))
 		mob_assertion_failed(message, exp, file, line, func);
@@ -25,7 +29,7 @@ inline void mob_assert(
 
 template <class X>
 inline void mob_assert(
-	X&& x, const char* exp, const char* file, int line, const char* func)
+	X&& x, const char* exp, const wchar_t* file, int line, const char* func)
 {
 	if (!(x))
 		mob_assertion_failed(nullptr, exp, file, line, func);
@@ -36,7 +40,9 @@ enum class encodings
 {
 	dont_know = 0,
 	utf8,
-	utf16
+	utf16,
+	acp,
+	oem
 };
 
 
@@ -222,6 +228,7 @@ std::string trim_copy(const std::string& s, const std::string& what=" \t\r\n");
 
 std::wstring utf8_to_utf16(std::string_view s);
 std::string utf16_to_utf8(std::wstring_view ws);
+std::string bytes_to_utf8(encodings e, std::string_view s);
 
 template <class T>
 std::string path_to_utf8(T&&) = delete;