encounter/buildcache
0
0
Fork 0
mirror of https://github.com/encounter/buildcache.git synced 2026-07-09 18:18:50 -07:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #32 from mbitsnbites/feature/add_cache_entry_size_limits

Browse Source 
Feature/add cache entry size limits
This commit is contained in:
mbitsnbites
2019-05-21 09:46:25 +02:00
committed by GitHub
parent 2ab0ae2d5c 4111be95fd
commit 6b873c2861
9 changed files with 151 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+2
View File
@@ -152,6 +152,8 @@ The following options control the behavior of BuildCache:
| `BUILDCACHE_LUA_PATH` | `lua_paths` | Extra path(s) to Lua wrappers | None |
| `BUILDCACHE_DEBUG` | `debug` | Debug level | None |
| `BUILDCACHE_MAX_CACHE_SIZE` | `max_cache_size` | Cache size limit in bytes | 5368709120 |
| `BUILDCACHE_MAX_LOCAL_ENTRY_SIZE` | `max_local_entry_size` | Local cache entry size limit in bytes (uncompressed) | 134217728 |
| `BUILDCACHE_MAX_REMOTE_ENTRY_SIZE` | `max_remote_entry_size` | Remote cache entry size limit in bytes (uncompressed) | 134217728 |
| `BUILDCACHE_HARD_LINKS` | `hard_links` | Allow the use of hard links when caching | true |
| `BUILDCACHE_COMPRESS` | `compress` | Allow the use of compression when caching (overrides hard links) | true |
| `BUILDCACHE_PERF` | `perf` | Enable performance logging | false |
src/base/file_utils.cpp
+19
View File
@@ -644,6 +644,25 @@ file_info_t get_file_info(const std::string& path) {
throw std::runtime_error("Unable to get file information.");
}
std::string human_readable_size(const int64_t byte_size) {
static const char* SUFFIX[6] = {"bytes", "KiB", "MiB", "GiB", "TiB", "PiB"};
static const int MAX_SUFFIX_IDX = (sizeof(SUFFIX) / sizeof(SUFFIX[0])) - 1;
double scaled_size = static_cast<double>(byte_size);
int suffix_idx = 0;
for (; scaled_size >= 1024.0 && suffix_idx < MAX_SUFFIX_IDX; ++suffix_idx) {
scaled_size /= 1024.0;
}
char buf[20];
if (suffix_idx >= 1) {
std::snprintf(buf, sizeof(buf), "%.1f %s", scaled_size, SUFFIX[suffix_idx]);
} else {
std::snprintf(buf, sizeof(buf), "%d %s", static_cast<int>(byte_size), SUFFIX[suffix_idx]);
}
return std::string(buf);
}
std::vector<file_info_t> walk_directory(const std::string& path) {
std::vector<file_info_t> files;
src/base/file_utils.hpp
+5
View File
@@ -158,6 +158,11 @@ std::string find_executable(const std::string& path, const std::string& exclude
/// @returns a file information object.
file_info_t get_file_info(const std::string& path);
/// @brief Convert a size to a human readable string.
/// @param byte_size The size (number of bytes).
/// @returns a string containing a human readable version of the size, e.g. "4.7 MiB".
std::string human_readable_size(const int64_t byte_size);
/// @brief Walk a directory and its subdirectories.
/// @param path The path to the directory.
/// @returns a vector of file information objects.
src/cache/cache.cpp
Vendored
+35 -8
View File
@@ -21,12 +21,26 @@
#include <base/debug_utils.hpp>
#include <base/file_utils.hpp>
#include <config/configuration.hpp>
#include <sys/perf_utils.hpp>
#include <sys/sys_utils.hpp>
#include <iostream>
namespace bcache {
namespace {
// Return the total size (uncompressed bytes) for a cache entry.
int64_t get_total_entry_size(const cache_entry_t& entry,
const std::map<std::string, std::string>& file_paths) {
int64_t total_size =
static_cast<int64_t>(entry.std_out().size()) + static_cast<int64_t>(entry.std_err().size());
for (const auto& item : file_paths) {
total_size += file::get_file_info(item.second).size();
}
return total_size;
}
} // namespace
bool cache_t::lookup(const hasher_t::hash_t hash,
const std::map<std::string, std::string>& file_paths,
const bool allow_hard_links,
@@ -50,18 +64,31 @@ void cache_t::add(const hasher_t::hash_t hash,
const bool allow_hard_links) {
PERF_START(ADD_TO_CACHE);
// We need the size of the cache entry for checking against the configured limits.
const auto size = get_total_entry_size(entry, file_paths);
// Add the entry to the local cache.
m_local_cache.add(hash, entry, file_paths, allow_hard_links);
const auto max_local_size = config::max_local_entry_size();
if (size < max_local_size || max_local_size <= 0) {
m_local_cache.add(hash, entry, file_paths, allow_hard_links);
} else {
debug::log(debug::INFO) << "Cache entry too large for the local cache: " << size << " bytes";
}
// Add the entry to the remote cache.
if (m_remote_cache.is_connected()) {
// Note: We always compress entries for the remote cache.
const cache_entry_t remote_entry(entry.file_ids(),
cache_entry_t::comp_mode_t::ALL,
entry.std_out(),
entry.std_err(),
entry.return_code());
m_remote_cache.add(hash, remote_entry, file_paths);
const auto max_remote_size = config::max_remote_entry_size();
if (size < max_remote_size || max_remote_size <= 0) {
// Note: We always compress entries for the remote cache.
const cache_entry_t remote_entry(entry.file_ids(),
cache_entry_t::comp_mode_t::ALL,
entry.std_out(),
entry.std_err(),
entry.return_code());
m_remote_cache.add(hash, remote_entry, file_paths);
} else {
debug::log(debug::INFO) << "Cache entry too large for the remote cache: " << size << " bytes";
}
}
PERF_STOP(ADD_TO_CACHE);
src/cache/local_cache.cpp
Vendored
+5 -6
View File
@@ -180,17 +180,16 @@ void local_cache_t::show_stats() {
num_entries++;
total_size += dir.size();
}
const auto total_size_mib = static_cast<double>(total_size) / (1024.0 * 1024.0);
const auto max_size_mib = static_cast<double>(config::max_cache_size()) / (1024.0 * 1024.0);
const auto full_percentage = 100.0 * total_size_mib / max_size_mib;
const auto full_percentage =
100.0 * static_cast<double>(total_size) / static_cast<double>(config::max_cache_size());
// Print stats.
std::ios old_fmt(nullptr);
old_fmt.copyfmt(std::cout);
std::cout << std::setiosflags(std::ios::fixed) << std::setprecision(1);
std::cout << " Entries in cache: " << num_entries << "\n";
std::cout << " Cache size: " << total_size_mib << " MiB (" << full_percentage
<< "%)\n";
std::cout << " Entries in cache: " << num_entries << "\n";
std::cout << " Cache size: " << file::human_readable_size(total_size) << " ("
<< full_percentage << "%)\n";
std::cout.copyfmt(old_fmt);
}
src/config/configuration.cpp
+53 -1
View File
@@ -32,7 +32,9 @@ namespace {
// Various constants.
const std::string ROOT_FOLDER_NAME = ".buildcache";
const std::string CONFIGURATION_FILE_NAME = "config.json";
const int64_t DEFAULT_MAX_CACHE_SIZE = 5368709120L; // 5 GB
const int64_t DEFAULT_MAX_CACHE_SIZE = 5368709120L; // 5 GiB
const int64_t DEFAULT_MAX_LOCAL_ENTRY_SIZE = 134217728L; // 128 MiB
const int64_t DEFAULT_MAX_REMOTE_ENTRY_SIZE = 134217728L; // 128 MiB
// Configuration options.
std::string s_dir;
@@ -41,6 +43,8 @@ string_list_t s_lua_paths;
std::string s_prefix;
std::string s_remote;
int64_t s_max_cache_size = DEFAULT_MAX_CACHE_SIZE;
int64_t s_max_local_entry_size = DEFAULT_MAX_LOCAL_ENTRY_SIZE;
int64_t s_max_remote_entry_size = DEFAULT_MAX_REMOTE_ENTRY_SIZE;
int32_t s_debug = -1;
bool s_hard_links = true;
bool s_compress = false;
@@ -169,6 +173,22 @@ void load_from_file(const std::string& file_name) {
}
}
// Get "max_local_entry_size".
{
const auto* node = cJSON_GetObjectItemCaseSensitive(root, "max_local_entry_size");
if (cJSON_IsNumber(node)) {
s_max_local_entry_size = static_cast<int64_t>(node->valuedouble);
}
}
// Get "max_remote_entry_size".
{
const auto* node = cJSON_GetObjectItemCaseSensitive(root, "max_remote_entry_size");
if (cJSON_IsNumber(node)) {
s_max_remote_entry_size = static_cast<int64_t>(node->valuedouble);
}
}
// Get "debug".
{
const auto* node = cJSON_GetObjectItemCaseSensitive(root, "debug");
@@ -279,6 +299,30 @@ void init() {
}
}
// Get the max local cache entry size from the environment.
{
const env_var_t max_local_entry_size_env("BUILDCACHE_MAX_LOCAL_ENTRY_SIZE");
if (max_local_entry_size_env) {
try {
s_max_local_entry_size = max_local_entry_size_env.as_int64();
} catch (...) {
// Ignore...
}
}
}
// Get the max remote cache entry size from the environment.
{
const env_var_t max_remote_entry_size_env("BUILDCACHE_MAX_REMOTE_ENTRY_SIZE");
if (max_remote_entry_size_env) {
try {
s_max_remote_entry_size = max_remote_entry_size_env.as_int64();
} catch (...) {
// Ignore...
}
}
}
// Get the debug level from the environment.
{
const env_var_t debug_env("BUILDCACHE_DEBUG");
@@ -353,6 +397,14 @@ int64_t max_cache_size() {
return s_max_cache_size;
}
int64_t max_local_entry_size() {
return s_max_local_entry_size;
}
int64_t max_remote_entry_size() {
return s_max_remote_entry_size;
}
int32_t debug() {
return s_debug;
}
src/config/configuration.hpp
+6
View File
@@ -49,6 +49,12 @@ const std::string& remote();
/// @returns the maximum cache size (in bytes).
int64_t max_cache_size();
/// @returns the maximum local cache entry size (in bytes).
int64_t max_local_entry_size();
/// @returns the maximum remote cache entry size (in bytes).
int64_t max_remote_entry_size();
/// @returns the debug level (-1 for no debugging).
int32_t debug();
src/main.cpp
+23 -13
View File
@@ -135,24 +135,34 @@ std::unique_ptr<bcache::program_wrapper_t> find_suitable_wrapper(
#endif
std::cout << "\nConfiguration:\n";
std::cout << " Configuration file: " << bcache::config::config_file() << "\n\n";
std::cout << " BUILDCACHE_DIR: " << bcache::config::dir() << "\n";
std::cout << " BUILDCACHE_LUA_PATH: "
std::cout << " Configuration file: " << bcache::config::config_file() << "\n\n";
std::cout << " BUILDCACHE_DIR: " << bcache::config::dir() << "\n";
std::cout << " BUILDCACHE_LUA_PATH: "
<< bcache::config::lua_paths().join(PATH_SEP, false) << "\n";
std::cout << " BUILDCACHE_PREFIX: " << bcache::config::prefix() << "\n";
std::cout << " BUILDCACHE_REMOTE: "
std::cout << " BUILDCACHE_PREFIX: " << bcache::config::prefix() << "\n";
std::cout << " BUILDCACHE_REMOTE: "
<< (bcache::config::remote().empty() ? "(disabled)" : bcache::config::remote())
<< "\n";
std::cout << " BUILDCACHE_MAX_CACHE_SIZE: " << bcache::config::max_cache_size() << "\n";
std::cout << " BUILDCACHE_DEBUG: " << bcache::config::debug() << "\n";
std::cout << " BUILDCACHE_HARD_LINKS: "
std::cout << " BUILDCACHE_MAX_CACHE_SIZE: " << bcache::config::max_cache_size()
<< " (" << bcache::file::human_readable_size(bcache::config::max_cache_size())
<< ")\n";
std::cout << " BUILDCACHE_MAX_LOCAL_ENTRY_SIZE: " << bcache::config::max_local_entry_size()
<< " (" << bcache::file::human_readable_size(bcache::config::max_local_entry_size())
<< ")\n";
std::cout << " BUILDCACHE_MAX_REMOTE_ENTRY_SIZE: "
<< bcache::config::max_remote_entry_size() << " ("
<< bcache::file::human_readable_size(bcache::config::max_remote_entry_size())
<< ")\n";
std::cout << " BUILDCACHE_DEBUG: " << bcache::config::debug() << "\n";
std::cout << " BUILDCACHE_HARD_LINKS: "
<< (bcache::config::hard_links() ? "true" : "false") << "\n";
std::cout << " BUILDCACHE_COMPRESS: "
std::cout << " BUILDCACHE_COMPRESS: "
<< (bcache::config::compress() ? "true" : "false") << "\n";
std::cout << " BUILDCACHE_PERF: " << (bcache::config::perf() ? "true" : "false")
<< "\n";
std::cout << " BUILDCACHE_DISABLE: " << (bcache::config::disable() ? "true" : "false")
<< "\n";
std::cout << " BUILDCACHE_PERF: "
<< (bcache::config::perf() ? "true" : "false") << "\n";
std::cout << " BUILDCACHE_DISABLE: "
<< (bcache::config::disable() ? "true" : "false") << "\n";
}
} catch (const std::exception& e) {
std::cerr << "*** Unexpected error: " << e.what() << "\n";
src/wrappers/program_wrapper.cpp
+3 -3
View File
@@ -132,14 +132,14 @@ bool program_wrapper_t::handle_command(int& return_code) {
// Note: We do not want to create cache entries for failed program runs. We could, but that
// would run the risk of caching intermittent faults for instance.
if (result.return_code == 0) {
// Add the entry to the local cache.
const cache_entry_t local_entry(
// Add the entry to the cache.
const cache_entry_t entry(
file_ids,
config::compress() ? cache_entry_t::comp_mode_t::ALL : cache_entry_t::comp_mode_t::NONE,
result.std_out,
result.std_err,
result.return_code);
m_cache.add(hash, local_entry, build_files, allow_hard_links);
m_cache.add(hash, entry, build_files, allow_hard_links);
}
// Everything's ok!