From 932495f36d1360d039ae636073d3a5251fcd2cea Mon Sep 17 00:00:00 2001 From: isanae <14251494+isanae@users.noreply.github.com> Date: Wed, 6 May 2020 07:43:15 -0400 Subject: [PATCH] fixed a few leaking exceptions force filenames for stylesheets, some are just version numbers changed curl downloader to use CreateFile(), was getting access denied errors fixed curl sometimes getting stuck sip has to start downloading after python because it uses python.exe fixed op::check() to be case insensitive better message when important variables are missing bootstrap: uses vcvars, multi process --- bootstrap.bat | 34 +++++++++++++++- src/conf.cpp | 33 ++++++++++----- src/main.cpp | 21 +++++++++- src/net.cpp | 86 ++++++++++++++++++++++++++++++++++++--- src/net.h | 9 +++- src/op.cpp | 22 ++++++++-- src/process.cpp | 3 ++ src/tasks/sip.cpp | 20 +++++---- src/tasks/stylesheets.cpp | 8 ++-- src/tasks/task.cpp | 3 +- src/tools/downloader.cpp | 64 +++++++++++++++++++---------- src/tools/tools.cpp | 2 +- src/tools/tools.h | 2 + src/utility.cpp | 22 ++++++++-- vs/mob.vcxproj | 2 +- 15 files changed, 267 insertions(+), 64 deletions(-) diff --git a/bootstrap.bat b/bootstrap.bat index 3d35eb8..4ef73c8 100644 --- a/bootstrap.bat +++ b/bootstrap.bat @@ -1 +1,33 @@ -@msbuild vs/builder.sln -m -p:Configuration=Release -noLogo -clp:ErrorsOnly;Verbosity=minimal \ No newline at end of file +@echo off +Setlocal EnableDelayedExpansion + +set "vswhere_cmd=third-party\bin\vswhere.exe -nologo -prerelease -latest -property installationPath" + +for /F "tokens=* USEBACKQ" %%F in (`%vswhere_cmd%`) do ( + set ret=%errorlevel% + if %errorlevel% neq 0 ( + echo %%F + echo vswhere returned %ret% + exit /b 1 + ) + + set installation_path=%%F + + if "%installation_path%" == "" ( + echo empty installation path + exit /b 1 + ) +) + +set "opts=" +set "opts=%opts% vs/mob.sln" +set "opts=%opts% -m " +set "opts=%opts% -p:Configuration=Release" +set "opts=%opts% -noLogo " +set "opts=%opts% -p:UseMultiToolTask=true" +set "opts=%opts% -p:EnforceProcessCountAcrossBuilds=true" +set "opts=%opts% -clp:ErrorsOnly;Verbosity=minimal" + +set "vcvars=%installation_path%\VC\Auxiliary\Build\vcvarsall.bat" +cmd /c ""%vcvars%" amd64 > NUL && msbuild %opts%" +echo run `mob` to start building diff --git a/src/conf.cpp b/src/conf.cpp index 2dd4464..b49beef 100644 --- a/src/conf.cpp +++ b/src/conf.cpp @@ -277,13 +277,15 @@ void dump_available_options() bool try_parts(fs::path& check, const std::vector& parts) { - for (std::size_t i=0; ion_write(ptr, size * nmemb); + + if (self->interrupt_) + { + debug("downloader: interrupting"); + return (size * nmemb) + 1; // force failure + } + return size * nmemb; } @@ -179,11 +205,59 @@ void curl_downloader::on_write(char* ptr, std::size_t n) noexcept op::create_directories(cx_, path_.parent_path()); cx_.trace(context::net, "opening " + path_.string()); - file_.reset(_wfopen(path_.native().c_str(), L"wb")); + + HANDLE h = ::CreateFileA( + path_.string().c_str(), GENERIC_WRITE, FILE_SHARE_READ, + nullptr, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, 0); + + if (h == INVALID_HANDLE_VALUE) + { + const auto e = GetLastError(); + cx_.error(context::net, "failed to open " + path_.string(), e); + interrupt_ = true; + return; + } + + file_.reset(h); } bytes_ += n; - std::fwrite(ptr, n, 1, file_.get()); + + DWORD written = 0; + if (!::WriteFile(file_.get(), ptr, static_cast(n), &written, nullptr)) + { + const auto e = GetLastError(); + cx_.error(context::net, "failed to write to " + path_.string(), e); + interrupt_ = true; + } +} + +int curl_downloader::on_progress_static( + void* user, double, double, double, double) noexcept +{ + auto* self = static_cast(user); + + if (self->interrupt_) + { + debug("downloader: interrupting"); + return 1; + } + + return 0; +} + +int curl_downloader::on_xfer_static( + void* user, curl_off_t, curl_off_t, curl_off_t, curl_off_t) noexcept +{ + auto* self = static_cast(user); + + if (self->interrupt_) + { + debug("downloader: interrupting"); + return 1; + } + + return 0; } int curl_downloader::on_debug_static( diff --git a/src/net.h b/src/net.h index caa381b..c9d8ef5 100644 --- a/src/net.h +++ b/src/net.h @@ -48,7 +48,7 @@ private: const context& cx_; url url_; fs::path path_; - file_ptr file_; + handle_ptr file_; std::thread thread_; std::size_t bytes_; std::atomic interrupt_; @@ -61,6 +61,13 @@ private: void on_write(char* ptr, std::size_t n) noexcept; + static int on_progress_static( + void* user, double dltotal, double dlnow, + double ultotal, double ulnow) noexcept; + + static int on_xfer_static( + void* user, curl_off_t dltotal, curl_off_t dlnow, + curl_off_t ultotal, curl_off_t ulnow) noexcept; static int on_debug_static( CURL* handle, curl_infotype type, diff --git a/src/op.cpp b/src/op.cpp index 680b810..3739684 100644 --- a/src/op.cpp +++ b/src/op.cpp @@ -484,13 +484,29 @@ void check(const context& cx, const fs::path& p) if (p.empty()) cx.bail_out(context::fs, "path is empty"); - if (p.native().starts_with(paths::prefix().native())) + auto is_inside = [](auto&& p, auto&& dir) + { + const std::string s = p.string(); + const std::string prefix = dir.string(); + + if (s.size() < prefix.size()) + return false; + + const std::string scut = s.substr(0, prefix.size()); + + if (_stricmp(scut.c_str(), prefix.c_str()) != 0) + return false; + + return true; + }; + + if (is_inside(p, paths::prefix())) return; - if (p.native().starts_with(paths::temp_dir().native())) + if (is_inside(p, paths::temp_dir())) return; - if (p.native().starts_with(paths::licenses().native())) + if (is_inside(p, paths::licenses())) return; cx.bail_out(context::fs, "path " + p.string() + " is outside prefix"); diff --git a/src/process.cpp b/src/process.cpp index 454a46b..6e7d08e 100644 --- a/src/process.cpp +++ b/src/process.cpp @@ -607,7 +607,10 @@ void process::on_completed() // success if (code_ == 0) + { + cx_->trace(context::cmd, "process exit code is 0"); return; + } if (flags_ & allow_failure) { diff --git a/src/tasks/sip.cpp b/src/tasks/sip.cpp index 74394f4..f0f4515 100644 --- a/src/tasks/sip.cpp +++ b/src/tasks/sip.cpp @@ -46,12 +46,23 @@ void sip::do_clean_for_rebuild() } void sip::do_fetch() +{ + // downloading uses python.exe and so has to wait until it's built +} + +void sip::do_build_and_install() { download(); run_tool(extractor() .file(download_file()) .output(source_path())); + + generate(); + + op::copy_file_to_dir_if_better(cx(), + source_path() / "sip.h", + python::include_path()); } void sip::download() @@ -84,15 +95,6 @@ void sip::download() .arg("sip==" + versions::sip()))); } -void sip::do_build_and_install() -{ - generate(); - - op::copy_file_to_dir_if_better(cx(), - source_path() / "sip.h", - python::include_path()); -} - void sip::generate() { const auto header = source_path() / "sip.h"; diff --git a/src/tasks/stylesheets.cpp b/src/tasks/stylesheets.cpp index 3010367..60a4925 100644 --- a/src/tasks/stylesheets.cpp +++ b/src/tasks/stylesheets.cpp @@ -22,9 +22,11 @@ void stylesheets::do_fetch() for (auto&& r : releases()) { - const auto file = run_tool(downloader( - "https://github.com/" + r.repo + "/" + r.name + "/releases/" - "download/v" + r.version + "/" + r.file + ".7z")); + const auto file = run_tool(downloader() + .url( + "https://github.com/" + r.repo + "/" + r.name + "/releases/" + "download/v" + r.version + "/" + r.file + ".7z") + .file(paths::cache() / (r.name + ".7z"))); run_tool(extractor() .file(file) diff --git a/src/tasks/task.cpp b/src/tasks/task.cpp index 75b3490..15a65ad 100644 --- a/src/tasks/task.cpp +++ b/src/tasks/task.cpp @@ -232,7 +232,8 @@ void task::fetch() if (!get_source_path().empty()) { - cx().info(context::generic, "patching"); + cx().debug(context::generic, "patching"); + run_tool(patcher() .task(name()) .root(get_source_path())); diff --git a/src/tools/downloader.cpp b/src/tools/downloader.cpp index 574fe67..6075b0c 100644 --- a/src/tools/downloader.cpp +++ b/src/tools/downloader.cpp @@ -17,9 +17,13 @@ downloader::downloader(mob::url u) downloader& downloader::url(const mob::url& u) { - cx_->trace(context::net, "adding url " + u.string()); urls_.push_back(u); + return *this; +} +downloader& downloader::file(const fs::path& p) +{ + file_ = p; return *this; } @@ -34,28 +38,23 @@ void downloader::do_run() cx_->trace(context::net, "looking for already downloaded files"); - for (auto&& u : urls_) + if (!file_.empty()) { - const auto file = path_for_url(u); - - if (fs::exists(file)) + if (try_picking(file_)) + return; + } + else + { + for (auto&& u : urls_) { - if (conf::redownload()) + const auto file = path_for_url(u); + + if (try_picking(file)) { - cx_->trace(context::redownload, "deleting " + file.string()); - op::delete_file(*cx_, file, op::optional); - } - else - { - cx_->trace(context::bypass, "picking " + file_.string()); file_ = file; return; } } - else - { - cx_->trace(context::net, "no " + file.string()); - } } @@ -67,19 +66,19 @@ void downloader::do_run() // try them in order for (auto&& u : urls_) { - const fs::path file = path_for_url(u); + if (file_.empty()) + file_ = path_for_url(u); cx_->trace(context::net, - "trying " + u.string() + " into " + file.string()); + "trying " + u.string() + " into " + file_.string()); - dl_->start(u, file); + dl_->start(u, file_); cx_->trace(context::net, "waiting for download"); dl_->join(); if (dl_->ok()) { - cx_->trace(context::net, "file " + file.string() + " downloaded"); - file_ = file; + cx_->trace(context::net, "file " + file_.string() + " downloaded"); return; } @@ -102,6 +101,29 @@ void downloader::do_interrupt() dl_->interrupt(); } +bool downloader::try_picking(const fs::path& file) +{ + if (fs::exists(file)) + { + if (conf::redownload()) + { + cx_->trace(context::redownload, "deleting " + file.string()); + op::delete_file(*cx_, file, op::optional); + } + else + { + cx_->trace(context::bypass, "picking " + file_.string()); + return true; + } + } + else + { + cx_->trace(context::net, "no " + file.string()); + } + + return false; +} + fs::path downloader::path_for_url(const mob::url& u) const { std::string filename; diff --git a/src/tools/tools.cpp b/src/tools/tools.cpp index ddd9636..c8745c4 100644 --- a/src/tools/tools.cpp +++ b/src/tools/tools.cpp @@ -48,7 +48,7 @@ void tool::interrupt() { if (!interrupted_) { - cx_->info(context::interruption, "interrupting " + name_); + cx_->debug(context::interruption, "interrupting " + name_); interrupted_ = true; do_interrupt(); } diff --git a/src/tools/tools.h b/src/tools/tools.h index f51ba6a..fe90f81 100644 --- a/src/tools/tools.h +++ b/src/tools/tools.h @@ -49,6 +49,7 @@ public: downloader(mob::url u); downloader& url(const mob::url& u); + downloader& file(const fs::path& p); fs::path result() const; @@ -62,6 +63,7 @@ private: std::vector urls_; fs::path path_for_url(const mob::url& u) const; + bool try_picking(const fs::path& file); }; diff --git a/src/utility.cpp b/src/utility.cpp index bf7188c..a446525 100644 --- a/src/utility.cpp +++ b/src/utility.cpp @@ -84,8 +84,15 @@ file_deleter::file_deleter(const context& cx, fs::path p) file_deleter::~file_deleter() { - if (delete_) - delete_now(); + try + { + if (delete_) + delete_now(); + } + catch(...) + { + // eat it + } } void file_deleter::delete_now() @@ -109,8 +116,15 @@ directory_deleter::directory_deleter(const context& cx, fs::path p) directory_deleter::~directory_deleter() { - if (delete_) - delete_now(); + try + { + if (delete_) + delete_now(); + } + catch(...) + { + // eat it + } } void directory_deleter::delete_now() diff --git a/vs/mob.vcxproj b/vs/mob.vcxproj index e38dc6d..c90ed1b 100644 --- a/vs/mob.vcxproj +++ b/vs/mob.vcxproj @@ -73,7 +73,7 @@ Console true true - false + true