optimizations:

- less string copies
 - encoding conversions use a basic_string directly instead of buffers and guess an initial buffer size instead of always calling twice
 - don't get environment variables all the time
 - cache dry as a bool
This commit is contained in:
isanae
2020-05-19 20:19:56 -04:00
parent 4c85474f7c
commit fa8b62be9d
13 changed files with 376 additions and 263 deletions
+14 -12
View File
@@ -12,6 +12,7 @@ namespace mob
conf::task_map conf::map_;
int conf::output_log_level_ = 3;
int conf::file_log_level_ = 5;
bool conf::dry_ = false;
std::string master_ini_filename()
{
@@ -123,10 +124,8 @@ void conf::set_for_task(
bool conf::prebuilt_by_name(const std::string& task)
{
std::istringstream iss(get_global("prebuilt", task));
bool b;
iss >> std::boolalpha >> b;
return b;
const std::string s = get_global("prebuilt", task);
return (s == "true" || s == "yes" || s == "1");
}
fs::path conf::path_by_name(const std::string& name)
@@ -151,10 +150,8 @@ std::string conf::global_by_name(const std::string& name)
bool conf::bool_global_by_name(const std::string& name)
{
std::istringstream iss(global_by_name(name));
bool b;
iss >> std::boolalpha >> b;
return b;
const std::string s = global_by_name(name);
return (s == "true" || s == "yes" || s == "1");
}
std::string conf::option_by_name(
@@ -166,10 +163,8 @@ std::string conf::option_by_name(
bool conf::bool_option_by_name(
const std::vector<std::string>& task_names, const std::string& name)
{
std::istringstream iss(option_by_name(task_names, name));
bool b;
iss >> std::boolalpha >> b;
return b;
const std::string s = option_by_name(task_names, name);
return (s == "true" || s == "yes" || s == "1");
}
void conf::set_output_log_level(const std::string& s)
@@ -211,6 +206,12 @@ void conf::set_file_log_level(const std::string& s)
}
}
void conf::set_dry(const std::string& s)
{
dry_ = (s == "true" || s == "yes" || s == "1");
}
std::vector<std::string> conf::format_options()
{
std::size_t longest_task = 0;
@@ -808,6 +809,7 @@ void set_special_options()
{
conf::set_output_log_level(conf::get_global("global", "output_log_level"));
conf::set_file_log_level(conf::get_global("global", "file_log_level"));
conf::set_dry(conf::get_global("global", "dry"));
}
std::vector<fs::path> find_inis(
+4 -1
View File
@@ -48,8 +48,10 @@ public:
static int file_log_level() { return file_log_level_; }
static void set_file_log_level(const std::string& s);
static bool dry() { return dry_; }
static void set_dry(const std::string& s);
static fs::path log_file() { return global_by_name("log_file"); }
static bool dry() { return bool_global_by_name("dry"); }
static bool redownload() { return bool_global_by_name("redownload"); }
static bool reextract() { return bool_global_by_name("reextract"); }
static bool rebuild() { return bool_global_by_name("rebuild"); }
@@ -66,6 +68,7 @@ private:
// special cases to avoid string manipulations
static int output_log_level_;
static int file_log_level_;
static bool dry_;
};
+91 -68
View File
@@ -57,7 +57,7 @@ console_color level_color(context::level lv)
}
}
std::string reason_string(context::reason r)
const char* reason_string(context::reason r)
{
switch (r)
{
@@ -77,41 +77,6 @@ std::string reason_string(context::reason r)
}
}
std::string task_name(const std::string& name)
{
const std::size_t longest = 15;
const std::size_t total = 1 + longest + 2; // '[x] '
if (!name.empty())
return pad_right("[" + name.substr(0, longest) + "]", total);
else
return std::string(total, ' ');
}
std::string tool_name(const tool* t)
{
const std::size_t longest = 7;
const std::size_t total = 1 + longest + 2; // '[x] '
if (t && !t->name().empty())
return pad_right("[" + t->name().substr(0, longest) + "]", total);
else
return std::string(total, ' ');
}
std::string prefix(context::reason r)
{
const std::size_t longest = 7;
const std::size_t total = 1 + longest + 2; // '[x] '
const std::string rs = reason_string(r).substr(0, longest);
if (!rs.empty())
return pad_right("[" + rs + "] ", total);
else
return std::string(total, ' ');
}
std::string error_message(DWORD id)
{
wchar_t* message = nullptr;
@@ -142,9 +107,8 @@ std::string error_message(DWORD id)
return utf16_to_utf8(s);
}
std::string timestamp()
std::string_view timestamp()
{
const std::size_t max_length = 7; // 0000.00
static thread_local char buffer[50];
using namespace std::chrono;
@@ -160,7 +124,7 @@ std::string timestamp()
const auto n = static_cast<std::size_t>(r.ptr - buffer);
if (r.ec == std::errc())
return pad_left(std::string(buffer, n), max_length, ' ');
return {buffer, n};
else
return "?";
}
@@ -237,17 +201,14 @@ void context::set_log_file(const fs::path& p)
}
void context::do_log_impl(
bool bail, reason r, level lv, const std::string& utf8) const
bool bail, reason r, level lv, std::string_view utf8) const
{
if (!bail && !enabled(lv))
return;
const auto ls = make_log_string(r, lv, utf8);
std::string_view ls = make_log_string(r, lv, utf8);
if (bail)
{
emit_log(lv, ls + " (bailing out)");
throw bailed(ls);
emit_log(lv, std::string(ls) + " (bailing out)");
throw bailed(std::string(ls));
}
else
{
@@ -255,19 +216,14 @@ void context::do_log_impl(
}
}
void context::emit_log(level lv, const std::string& utf8) const
void context::emit_log(level lv, std::string_view utf8) const
{
std::scoped_lock lock(g_mutex);
if (lv == level::error)
g_errors.push_back(utf8);
else if (lv == level::warning)
g_warnings.push_back(utf8);
if (log_enabled(lv, conf::output_log_level()))
{
auto c = level_color(lv);
u8cout << utf8 << "\n";
u8cout.write_ln(utf8);
}
if (g_log_file && log_enabled(lv, conf::file_log_level()))
@@ -275,41 +231,109 @@ void context::emit_log(level lv, const std::string& utf8) const
DWORD written = 0;
::WriteFile(
g_log_file.get(), utf8.c_str(), static_cast<DWORD>(utf8.size()),
g_log_file.get(), utf8.data(), static_cast<DWORD>(utf8.size()),
&written, nullptr);
::WriteFile(g_log_file.get(), "\r\n", 2, &written, nullptr);
}
if (lv == level::error)
g_errors.emplace_back(utf8);
else if (lv == level::warning)
g_warnings.emplace_back(utf8);
}
std::string context::make_log_string(reason r, level, std::string_view s) const
void append_brackets(std::string& s, std::string_view what, std::size_t total)
{
std::ostringstream oss;
if (what.empty())
{
s.append(total, ' ');
}
else
{
s.append(1, '[');
s.append(what);
s.append(1, ']');
oss
<< task_name(task_)
<< tool_name(tool_)
<< prefix(r);
const std::size_t written = 1 + what.size() + 1; // "[x]"
if (written < total)
s.append(total - written, ' ');
}
}
void append(std::string& s, std::string_view what, std::size_t total)
{
if (what.empty())
{
s.append(total, ' ');
}
else
{
s.append(what);
const std::size_t written = what.size(); // "x"
if (written < total)
s.append(total - written, ' ');
}
}
std::string_view context::make_log_string(reason r, level, std::string_view s) const
{
const std::size_t total_timestamp = 8; // '0000.00 '
const std::size_t longest_task_name = 15;
const std::size_t total_task_name = 1 + longest_task_name + 2; // '[x] '
const std::size_t longest_tool_name = 7;
const std::size_t total_tool_name = 1 + longest_tool_name + 2; // '[x] '
const std::size_t longest_prefix = 7;
const std::size_t total_prefix = 1 + longest_prefix + 2; // '[x] '
static thread_local std::string ls;
ls.reserve(
total_timestamp + // timestamp
total_task_name +
total_tool_name +
total_prefix +
s.size() +
50); // possible additional stuff
ls.clear();
append(ls, timestamp(), total_timestamp);
ls.append(1, ' ');
append_brackets(ls, task_, total_task_name);
if (tool_)
append_brackets(ls, tool_->name(), total_tool_name);
else
ls.append(total_tool_name, ' ');
append_brackets(ls, reason_string(r), total_prefix);
ls.append(s);
switch (r)
{
case context::redownload:
oss << s << " (happened because of --redownload)";
ls.append(" (happened because of --redownload)");
break;
case context::rebuild:
oss << s << " (happened because of --rebuild)";
ls.append(" (happened because of --rebuild)");
break;
case context::reextract:
oss << s << " (happened because of --reextract)";
ls.append(" (happened because of --reextract)");
break;
case context::interruption:
if (s.empty())
oss << "interrupted";
else
oss << s;
ls.append("interrupted");
break;
@@ -322,11 +346,10 @@ std::string context::make_log_string(reason r, level, std::string_view s) const
case context::generic:
case context::conf:
default:
oss << s;
break;
}
return std::string(timestamp()) + " " + oss.str();
return ls;
}
void dump_logs()
+19 -3
View File
@@ -128,6 +128,11 @@ public:
do_log(false, r, lv, f, std::forward<Args>(args)...);
}
void log_string(reason r, level lv, std::string_view s) const
{
do_log_string(false, r, lv, s);
}
template <class... Args>
void dump(reason r, const char* f, Args&&... args) const
{
@@ -174,9 +179,20 @@ private:
std::string task_;
const tool* tool_;
void do_log_string(bool bail, reason r, level lv, std::string_view s) const
{
if (!bail && !enabled(lv))
return;
do_log_impl(bail, r, lv, s);
}
template <class... Args>
void do_log(bool bail, reason r, level lv, const char* f, Args&&... args) const
{
if (!bail && !enabled(lv))
return;
try
{
const std::string utf8 = ::fmt::format(
@@ -202,9 +218,9 @@ private:
}
}
std::string make_log_string(reason r, level lv, std::string_view s) const;
void do_log_impl(bool bail, reason r, level lv, const std::string& utf8) const;
void emit_log(level lv, const std::string& utf8) const;
std::string_view make_log_string(reason r, level lv, std::string_view s) const;
void do_log_impl(bool bail, reason r, level lv, std::string_view utf8) const;
void emit_log(level lv, std::string_view utf8) const;
};
+46 -29
View File
@@ -209,10 +209,45 @@ void* env::get_unicode_pointers() const
static std::mutex g_sys_env_mutex;
static env g_sys_env;
static bool g_sys_env_inited;
env this_env::get()
{
std::scoped_lock lock(g_sys_env_mutex);
if (g_sys_env_inited)
return g_sys_env;
auto free = [](wchar_t* p) { FreeEnvironmentStringsW(p); };
auto env_block = std::unique_ptr<wchar_t, decltype(free)>{
GetEnvironmentStringsW(), free};
for (const wchar_t* name = env_block.get(); *name != L'\0'; )
{
const wchar_t* equal = std::wcschr(name, '=');
std::wstring key(name, static_cast<std::size_t>(equal - name));
const wchar_t* pValue = equal + 1;
std::wstring value(pValue);
if (!key.empty())
g_sys_env.set(utf16_to_utf8(key), utf16_to_utf8(value));
name = pValue + value.length() + 1;
}
g_sys_env_inited = true;
return g_sys_env;
}
void this_env::set(const std::string& k, const std::string& v, env::flags f)
{
const std::wstring wk = utf8_to_utf16(k);
const std::wstring wv = utf8_to_utf16(v);
std::wstring wv = utf8_to_utf16(v);
switch (f)
{
@@ -225,17 +260,25 @@ void this_env::set(const std::string& k, const std::string& v, env::flags f)
case env::append:
{
const std::wstring current = get_impl(k).value_or(L"");
::SetEnvironmentVariableW(wk.c_str(), (current + wv).c_str());
wv = current + wv;
::SetEnvironmentVariableW(wk.c_str(), wv.c_str());
break;
}
case env::prepend:
{
const std::wstring current = get_impl(k).value_or(L"");
::SetEnvironmentVariableW(wk.c_str(), (wv + current).c_str());
wv = wv + current;
::SetEnvironmentVariableW(wk.c_str(), wv.c_str());
break;
}
}
{
std::scoped_lock lock(g_sys_env_mutex);
if (g_sys_env_inited)
g_sys_env.set(k, utf16_to_utf8(wv));
}
}
void this_env::prepend_to_path(const fs::path& p)
@@ -286,30 +329,4 @@ std::optional<std::wstring> this_env::get_impl(const std::string& k)
return std::wstring(buffer.get(), buffer.get() + written);
}
env this_env::get()
{
env e;
auto free = [](wchar_t* p) { FreeEnvironmentStringsW(p); };
auto env_block = std::unique_ptr<wchar_t, decltype(free)>{
GetEnvironmentStringsW(), free};
for (const wchar_t* name = env_block.get(); *name != L'\0'; )
{
const wchar_t* equal = std::wcschr(name, '=');
std::wstring key(name, static_cast<std::size_t>(equal - name));
const wchar_t* pValue = equal + 1;
std::wstring value(pValue);
if (!key.empty())
e.set(utf16_to_utf8(key), utf16_to_utf8(value));
name = pValue + value.length() + 1;
}
return e;
}
} // namespace
+4 -1
View File
@@ -35,7 +35,10 @@ void create_directories(const context& cx, const fs::path& p)
check(cx, p);
if (!conf::dry())
do_create_directories(cx, p);
{
if (!fs::exists(p))
do_create_directories(cx, p);
}
}
void delete_directory(const context& cx, const fs::path& p, flags f)
+3 -87
View File
@@ -641,7 +641,7 @@ void process::read_pipe(
{
s.buffer.add(pipe.read());
s.buffer.next_utf8_lines(finish, [&](auto&& line)
s.buffer.next_utf8_lines(finish, [&](std::string&& line)
{
filter f = {line, r, s.level, false};
@@ -652,8 +652,8 @@ void process::read_pipe(
return;
}
logs_[f.lv].push_back(std::string(f.line));
cx_->log(f.r, f.lv, "{}", f.line);
cx_->log_string(f.r, f.lv, f.line);
logs_[f.lv].emplace_back(std::move(line));
});
break;
@@ -949,88 +949,4 @@ std::string encoded_buffer::utf8_string() const
return bytes_to_utf8(e_, bytes_);
}
template <class CharT>
std::basic_string<CharT> next_line(
bool finished, std::string_view bytes, std::size_t& byte_offset)
{
std::size_t size = bytes.size();
if constexpr (sizeof(CharT) == 2)
{
if ((size & 1) == 1)
--size;
}
const CharT* start = reinterpret_cast<const CharT*>(bytes.data() + byte_offset);
const CharT* end = reinterpret_cast<const CharT*>(bytes.data() + size);
const CharT* p = start;
std::basic_string<CharT> line;
while (p != end)
{
if (*p == CharT('\n') || *p == CharT('\r'))
{
line.assign(start, static_cast<std::size_t>(p - start));
while (p != end && (*p == CharT('\n') || *p == CharT('\r')))
++p;
if (!line.empty())
break;
start = p;
}
else
{
++p;
}
}
if (line.empty() && finished)
{
line = {
reinterpret_cast<const wchar_t*>(bytes.data() + byte_offset),
reinterpret_cast<const wchar_t*>(bytes.data() + size)
};
byte_offset = bytes.size();
}
else
{
byte_offset = static_cast<std::size_t>(
reinterpret_cast<const char*>(p) - bytes.data());
MOB_ASSERT(byte_offset <= bytes.size());
}
return line;
}
std::string encoded_buffer::next_utf8_line(bool finished)
{
switch (e_)
{
case encodings::utf16:
{
const std::wstring utf16 = next_line<wchar_t>(finished, bytes_, last_);
return utf16_to_utf8(utf16);
}
case encodings::acp:
case encodings::oem:
{
const std::string cp = next_line<char>(finished, bytes_, last_);
return bytes_to_utf8(e_, cp);
}
case encodings::utf8:
case encodings::dont_know:
default:
{
return next_line<char>(finished, bytes_, last_);
}
}
}
} // namespace
+98 -6
View File
@@ -49,11 +49,47 @@ public:
{
for (;;)
{
std::string line = next_utf8_line(finished);
if (line.empty())
break;
switch (e_)
{
case encodings::utf16:
{
std::wstring_view utf16 =
next_line<wchar_t>(finished, bytes_, last_);
f(line);
if (utf16.empty())
return;
f(utf16_to_utf8(utf16));
break;
}
case encodings::acp:
case encodings::oem:
{
std::string_view cp =
next_line<char>(finished, bytes_, last_);
if (cp.empty())
return;
f(bytes_to_utf8(e_, cp));
break;
}
case encodings::utf8:
case encodings::dont_know:
default:
{
std::string_view utf8 =
next_line<char>(finished, bytes_, last_);
if (utf8.empty())
return;
f(std::string(utf8));
break;
}
}
}
}
@@ -62,7 +98,63 @@ private:
std::string bytes_;
std::size_t last_;
std::string next_utf8_line(bool finished);
template <class CharT>
std::basic_string_view<CharT> next_line(
bool finished, std::string_view bytes, std::size_t& byte_offset)
{
std::size_t size = bytes.size();
if constexpr (sizeof(CharT) == 2)
{
if ((size & 1) == 1)
--size;
}
const CharT* start = reinterpret_cast<const CharT*>(bytes.data() + byte_offset);
const CharT* end = reinterpret_cast<const CharT*>(bytes.data() + size);
const CharT* p = start;
std::basic_string_view<CharT> line;
while (p != end)
{
if (*p == CharT('\n') || *p == CharT('\r'))
{
line = {start, static_cast<std::size_t>(p - start)};
while (p != end && (*p == CharT('\n') || *p == CharT('\r')))
++p;
if (!line.empty())
break;
start = p;
}
else
{
++p;
}
}
if (line.empty() && finished)
{
line = {
reinterpret_cast<const CharT*>(bytes.data() + byte_offset),
size - byte_offset
};
byte_offset = bytes.size();
}
else
{
byte_offset = static_cast<std::size_t>(
reinterpret_cast<const char*>(p) - bytes.data());
MOB_ASSERT(byte_offset <= bytes.size());
}
return line;
}
};
@@ -98,7 +190,7 @@ public:
struct filter
{
const std::string_view line;
std::string_view line;
context::reason r;
context::level lv;
bool ignore;
+2 -10
View File
@@ -10,11 +10,6 @@ basic_process_runner::basic_process_runner(std::string name)
{
}
std::string basic_process_runner::do_name() const
{
return process_.name();
}
void basic_process_runner::do_interrupt()
{
process_.interrupt();
@@ -22,6 +17,7 @@ void basic_process_runner::do_interrupt()
int basic_process_runner::execute_and_join()
{
set_name(process_.name());
process_.set_context(&cx());
process_.run();
join();
@@ -60,11 +56,6 @@ int process_runner::result() const
return exit_code();
}
std::string process_runner::do_name() const
{
return real_process().name();
}
void process_runner::do_interrupt()
{
real_process().interrupt();
@@ -72,6 +63,7 @@ void process_runner::do_interrupt()
int process_runner::execute_and_join()
{
set_name(real_process().name());
real_process().set_context(&cx());
real_process().run();
+5 -4
View File
@@ -26,12 +26,13 @@ tool& tool::operator=(tool&& t)
return *this;
}
std::string tool::name() const
void tool::set_name(const std::string& s)
{
std::string s = do_name();
if (!s.empty())
return s;
name_ = s;
}
const std::string& tool::name() const
{
return name_;
}
+4 -5
View File
@@ -17,7 +17,7 @@ public:
virtual ~tool() = default;
std::string name() const;
const std::string& name() const;
void run(context& cx);
void interrupt();
@@ -33,7 +33,8 @@ protected:
virtual void do_run() = 0;
virtual void do_interrupt() = 0;
virtual std::string do_name() const { return {}; }
void set_name(const std::string& s);
private:
context* cx_;
@@ -89,14 +90,13 @@ private:
class basic_process_runner : public tool
{
public:
std::string do_name() const override;
void join();
int exit_code() const;
protected:
process process_;
basic_process_runner(std::string name={});
basic_process_runner(std::string name);
void do_interrupt() override;
int execute_and_join();
@@ -109,7 +109,6 @@ public:
process_runner(process&& p);
process_runner(process& p);
std::string do_name() const override;
void join();
int exit_code() const;
+84 -37
View File
@@ -244,6 +244,26 @@ void u8stream::do_output(const std::string& s)
}
}
void u8stream::write_ln(std::string_view utf8)
{
std::scoped_lock lock(g_output_mutex);
if (err_)
{
if (stderr_console)
std::wcerr << utf8_to_utf16(utf8) << L"\n";
else
std::cerr << utf8 << "\n";
}
else
{
if (stdout_console)
std::wcout << utf8_to_utf16(utf8) << L"\n";
else
std::cout << utf8 << "\n";
}
}
void mob_assertion_failed(
const char* message,
@@ -640,55 +660,82 @@ console_color::~console_color()
std::optional<std::wstring> to_widechar(UINT from, std::string_view s)
{
std::wstring ws;
if (s.empty())
return std::wstring();
return ws;
const int wsize = MultiByteToWideChar(
from, 0, s.data(), static_cast<int>(s.size()), nullptr, 0);
ws.resize(s.size() + 1);
if (wsize == 0)
return {};
for (int t=0; t<3; ++t)
{
const int written = MultiByteToWideChar(
from, 0, s.data(), static_cast<int>(s.size()),
ws.data(), static_cast<int>(ws.size()));
auto buffer = std::make_unique<wchar_t[]>(
static_cast<std::size_t>(wsize + 1));
if (written <= 0)
{
const auto e = GetLastError();
const int written = MultiByteToWideChar(
from, 0, s.data(), static_cast<int>(s.size()),
buffer.get(), wsize);
if (e == ERROR_INSUFFICIENT_BUFFER)
{
ws.resize(ws.size() * 2);
continue;
}
else
{
return {};
}
}
else
{
MOB_ASSERT(static_cast<std::size_t>(written) <= s.size());
ws.resize(static_cast<std::size_t>(written));
break;
}
}
if (written == 0)
return {};
MOB_ASSERT(written == wsize);
return std::wstring(buffer.get(), buffer.get() + written);
return ws;
}
std::optional<std::string> to_multibyte(UINT to, std::wstring_view ws)
{
std::string s;
if (ws.empty())
return std::string();
return s;
const int size = WideCharToMultiByte(
to, 0, ws.data(), static_cast<int>(ws.size()), nullptr, 0,
nullptr, nullptr);
s.resize(static_cast<std::size_t>(ws.size() * 1.5));
if (size == 0)
return {};
for (int t=0; t<3; ++t)
{
const int written = WideCharToMultiByte(
to, 0, ws.data(), static_cast<int>(ws.size()),
s.data(), static_cast<int>(s.size()), nullptr, nullptr);
auto buffer = std::make_unique<char[]>(
static_cast<std::size_t>(size + 1));
if (written <= 0)
{
const auto e = GetLastError();
const int written = WideCharToMultiByte(
to, 0, ws.data(), static_cast<int>(ws.size()),
buffer.get(), size, nullptr, nullptr);
if (e == ERROR_INSUFFICIENT_BUFFER)
{
s.resize(ws.size() * 2);
continue;
}
else
{
return {};
}
}
else
{
MOB_ASSERT(static_cast<std::size_t>(written) <= s.size());
s.resize(static_cast<std::size_t>(written));
break;
}
}
if (written == 0)
return {};
MOB_ASSERT(written == size);
return std::string(buffer.get(), buffer.get() + written);
return s;
}
@@ -701,7 +748,7 @@ std::wstring utf8_to_utf16(std::string_view s)
return L"???";
}
return *ws;
return std::move(*ws);
}
std::string utf16_to_utf8(std::wstring_view ws)
@@ -713,7 +760,7 @@ std::string utf16_to_utf8(std::wstring_view ws)
return "???";
}
return *s;
return std::move(*s);
}
std::wstring cp_to_utf16(UINT from, std::string_view s)
@@ -725,7 +772,7 @@ std::wstring cp_to_utf16(UINT from, std::string_view s)
return L"???";
}
return *ws;
return std::move(*ws);
}
std::string utf16_to_cp(UINT to, std::wstring_view ws)
@@ -738,7 +785,7 @@ std::string utf16_to_cp(UINT to, std::wstring_view ws)
return "???";
}
return *s;
return std::move(*s);
}
+2
View File
@@ -273,6 +273,8 @@ public:
return *this;
}
void write_ln(std::string_view utf8);
private:
bool err_;