diff --git a/README.md b/README.md index a13ef45..482aafb 100644 --- a/README.md +++ b/README.md @@ -152,6 +152,8 @@ The following options control the behavior of BuildCache: | `BUILDCACHE_LUA_PATH` | `lua_paths` | Extra path(s) to Lua wrappers | None | | `BUILDCACHE_DEBUG` | `debug` | Debug level | None | | `BUILDCACHE_MAX_CACHE_SIZE` | `max_cache_size` | Cache size limit in bytes | 5368709120 | +| `BUILDCACHE_MAX_LOCAL_ENTRY_SIZE` | `max_local_entry_size` | Local cache entry size limit in bytes (uncompressed) | 134217728 | +| `BUILDCACHE_MAX_REMOTE_ENTRY_SIZE` | `max_remote_entry_size` | Remote cache entry size limit in bytes (uncompressed) | 134217728 | | `BUILDCACHE_HARD_LINKS` | `hard_links` | Allow the use of hard links when caching | true | | `BUILDCACHE_COMPRESS` | `compress` | Allow the use of compression when caching (overrides hard links) | true | | `BUILDCACHE_PERF` | `perf` | Enable performance logging | false | diff --git a/src/base/file_utils.cpp b/src/base/file_utils.cpp index a2a5511..74e4ac3 100644 --- a/src/base/file_utils.cpp +++ b/src/base/file_utils.cpp @@ -644,6 +644,25 @@ file_info_t get_file_info(const std::string& path) { throw std::runtime_error("Unable to get file information."); } +std::string human_readable_size(const int64_t byte_size) { + static const char* SUFFIX[6] = {"bytes", "KiB", "MiB", "GiB", "TiB", "PiB"}; + static const int MAX_SUFFIX_IDX = (sizeof(SUFFIX) / sizeof(SUFFIX[0])) - 1; + + double scaled_size = static_cast(byte_size); + int suffix_idx = 0; + for (; scaled_size >= 1024.0 && suffix_idx < MAX_SUFFIX_IDX; ++suffix_idx) { + scaled_size /= 1024.0; + } + + char buf[20]; + if (suffix_idx >= 1) { + std::snprintf(buf, sizeof(buf), "%.1f %s", scaled_size, SUFFIX[suffix_idx]); + } else { + std::snprintf(buf, sizeof(buf), "%d %s", static_cast(byte_size), SUFFIX[suffix_idx]); + } + return std::string(buf); +} + std::vector walk_directory(const std::string& path) { std::vector files; diff --git a/src/base/file_utils.hpp b/src/base/file_utils.hpp index 0a0f859..594800a 100644 --- a/src/base/file_utils.hpp +++ b/src/base/file_utils.hpp @@ -158,6 +158,11 @@ std::string find_executable(const std::string& path, const std::string& exclude /// @returns a file information object. file_info_t get_file_info(const std::string& path); +/// @brief Convert a size to a human readable string. +/// @param byte_size The size (number of bytes). +/// @returns a string containing a human readable version of the size, e.g. "4.7 MiB". +std::string human_readable_size(const int64_t byte_size); + /// @brief Walk a directory and its subdirectories. /// @param path The path to the directory. /// @returns a vector of file information objects. diff --git a/src/cache/cache.cpp b/src/cache/cache.cpp index b35b224..b227961 100644 --- a/src/cache/cache.cpp +++ b/src/cache/cache.cpp @@ -21,12 +21,26 @@ #include #include +#include #include #include #include namespace bcache { +namespace { +// Return the total size (uncompressed bytes) for a cache entry. +int64_t get_total_entry_size(const cache_entry_t& entry, + const std::map& file_paths) { + int64_t total_size = + static_cast(entry.std_out().size()) + static_cast(entry.std_err().size()); + for (const auto& item : file_paths) { + total_size += file::get_file_info(item.second).size(); + } + return total_size; +} +} // namespace + bool cache_t::lookup(const hasher_t::hash_t hash, const std::map& file_paths, const bool allow_hard_links, @@ -50,18 +64,31 @@ void cache_t::add(const hasher_t::hash_t hash, const bool allow_hard_links) { PERF_START(ADD_TO_CACHE); + // We need the size of the cache entry for checking against the configured limits. + const auto size = get_total_entry_size(entry, file_paths); + // Add the entry to the local cache. - m_local_cache.add(hash, entry, file_paths, allow_hard_links); + const auto max_local_size = config::max_local_entry_size(); + if (size < max_local_size || max_local_size <= 0) { + m_local_cache.add(hash, entry, file_paths, allow_hard_links); + } else { + debug::log(debug::INFO) << "Cache entry too large for the local cache: " << size << " bytes"; + } // Add the entry to the remote cache. if (m_remote_cache.is_connected()) { - // Note: We always compress entries for the remote cache. - const cache_entry_t remote_entry(entry.file_ids(), - cache_entry_t::comp_mode_t::ALL, - entry.std_out(), - entry.std_err(), - entry.return_code()); - m_remote_cache.add(hash, remote_entry, file_paths); + const auto max_remote_size = config::max_remote_entry_size(); + if (size < max_remote_size || max_remote_size <= 0) { + // Note: We always compress entries for the remote cache. + const cache_entry_t remote_entry(entry.file_ids(), + cache_entry_t::comp_mode_t::ALL, + entry.std_out(), + entry.std_err(), + entry.return_code()); + m_remote_cache.add(hash, remote_entry, file_paths); + } else { + debug::log(debug::INFO) << "Cache entry too large for the remote cache: " << size << " bytes"; + } } PERF_STOP(ADD_TO_CACHE); diff --git a/src/cache/local_cache.cpp b/src/cache/local_cache.cpp index 94577c3..ee0200b 100644 --- a/src/cache/local_cache.cpp +++ b/src/cache/local_cache.cpp @@ -180,17 +180,16 @@ void local_cache_t::show_stats() { num_entries++; total_size += dir.size(); } - const auto total_size_mib = static_cast(total_size) / (1024.0 * 1024.0); - const auto max_size_mib = static_cast(config::max_cache_size()) / (1024.0 * 1024.0); - const auto full_percentage = 100.0 * total_size_mib / max_size_mib; + const auto full_percentage = + 100.0 * static_cast(total_size) / static_cast(config::max_cache_size()); // Print stats. std::ios old_fmt(nullptr); old_fmt.copyfmt(std::cout); std::cout << std::setiosflags(std::ios::fixed) << std::setprecision(1); - std::cout << " Entries in cache: " << num_entries << "\n"; - std::cout << " Cache size: " << total_size_mib << " MiB (" << full_percentage - << "%)\n"; + std::cout << " Entries in cache: " << num_entries << "\n"; + std::cout << " Cache size: " << file::human_readable_size(total_size) << " (" + << full_percentage << "%)\n"; std::cout.copyfmt(old_fmt); } diff --git a/src/config/configuration.cpp b/src/config/configuration.cpp index a2a5689..5eb4109 100644 --- a/src/config/configuration.cpp +++ b/src/config/configuration.cpp @@ -32,7 +32,9 @@ namespace { // Various constants. const std::string ROOT_FOLDER_NAME = ".buildcache"; const std::string CONFIGURATION_FILE_NAME = "config.json"; -const int64_t DEFAULT_MAX_CACHE_SIZE = 5368709120L; // 5 GB +const int64_t DEFAULT_MAX_CACHE_SIZE = 5368709120L; // 5 GiB +const int64_t DEFAULT_MAX_LOCAL_ENTRY_SIZE = 134217728L; // 128 MiB +const int64_t DEFAULT_MAX_REMOTE_ENTRY_SIZE = 134217728L; // 128 MiB // Configuration options. std::string s_dir; @@ -41,6 +43,8 @@ string_list_t s_lua_paths; std::string s_prefix; std::string s_remote; int64_t s_max_cache_size = DEFAULT_MAX_CACHE_SIZE; +int64_t s_max_local_entry_size = DEFAULT_MAX_LOCAL_ENTRY_SIZE; +int64_t s_max_remote_entry_size = DEFAULT_MAX_REMOTE_ENTRY_SIZE; int32_t s_debug = -1; bool s_hard_links = true; bool s_compress = false; @@ -169,6 +173,22 @@ void load_from_file(const std::string& file_name) { } } + // Get "max_local_entry_size". + { + const auto* node = cJSON_GetObjectItemCaseSensitive(root, "max_local_entry_size"); + if (cJSON_IsNumber(node)) { + s_max_local_entry_size = static_cast(node->valuedouble); + } + } + + // Get "max_remote_entry_size". + { + const auto* node = cJSON_GetObjectItemCaseSensitive(root, "max_remote_entry_size"); + if (cJSON_IsNumber(node)) { + s_max_remote_entry_size = static_cast(node->valuedouble); + } + } + // Get "debug". { const auto* node = cJSON_GetObjectItemCaseSensitive(root, "debug"); @@ -279,6 +299,30 @@ void init() { } } + // Get the max local cache entry size from the environment. + { + const env_var_t max_local_entry_size_env("BUILDCACHE_MAX_LOCAL_ENTRY_SIZE"); + if (max_local_entry_size_env) { + try { + s_max_local_entry_size = max_local_entry_size_env.as_int64(); + } catch (...) { + // Ignore... + } + } + } + + // Get the max remote cache entry size from the environment. + { + const env_var_t max_remote_entry_size_env("BUILDCACHE_MAX_REMOTE_ENTRY_SIZE"); + if (max_remote_entry_size_env) { + try { + s_max_remote_entry_size = max_remote_entry_size_env.as_int64(); + } catch (...) { + // Ignore... + } + } + } + // Get the debug level from the environment. { const env_var_t debug_env("BUILDCACHE_DEBUG"); @@ -353,6 +397,14 @@ int64_t max_cache_size() { return s_max_cache_size; } +int64_t max_local_entry_size() { + return s_max_local_entry_size; +} + +int64_t max_remote_entry_size() { + return s_max_remote_entry_size; +} + int32_t debug() { return s_debug; } diff --git a/src/config/configuration.hpp b/src/config/configuration.hpp index 3e920b4..bc19009 100644 --- a/src/config/configuration.hpp +++ b/src/config/configuration.hpp @@ -49,6 +49,12 @@ const std::string& remote(); /// @returns the maximum cache size (in bytes). int64_t max_cache_size(); +/// @returns the maximum local cache entry size (in bytes). +int64_t max_local_entry_size(); + +/// @returns the maximum remote cache entry size (in bytes). +int64_t max_remote_entry_size(); + /// @returns the debug level (-1 for no debugging). int32_t debug(); diff --git a/src/main.cpp b/src/main.cpp index 5041b33..9d96d1e 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -135,24 +135,34 @@ std::unique_ptr find_suitable_wrapper( #endif std::cout << "\nConfiguration:\n"; - std::cout << " Configuration file: " << bcache::config::config_file() << "\n\n"; - std::cout << " BUILDCACHE_DIR: " << bcache::config::dir() << "\n"; - std::cout << " BUILDCACHE_LUA_PATH: " + std::cout << " Configuration file: " << bcache::config::config_file() << "\n\n"; + + std::cout << " BUILDCACHE_DIR: " << bcache::config::dir() << "\n"; + std::cout << " BUILDCACHE_LUA_PATH: " << bcache::config::lua_paths().join(PATH_SEP, false) << "\n"; - std::cout << " BUILDCACHE_PREFIX: " << bcache::config::prefix() << "\n"; - std::cout << " BUILDCACHE_REMOTE: " + std::cout << " BUILDCACHE_PREFIX: " << bcache::config::prefix() << "\n"; + std::cout << " BUILDCACHE_REMOTE: " << (bcache::config::remote().empty() ? "(disabled)" : bcache::config::remote()) << "\n"; - std::cout << " BUILDCACHE_MAX_CACHE_SIZE: " << bcache::config::max_cache_size() << "\n"; - std::cout << " BUILDCACHE_DEBUG: " << bcache::config::debug() << "\n"; - std::cout << " BUILDCACHE_HARD_LINKS: " + std::cout << " BUILDCACHE_MAX_CACHE_SIZE: " << bcache::config::max_cache_size() + << " (" << bcache::file::human_readable_size(bcache::config::max_cache_size()) + << ")\n"; + std::cout << " BUILDCACHE_MAX_LOCAL_ENTRY_SIZE: " << bcache::config::max_local_entry_size() + << " (" << bcache::file::human_readable_size(bcache::config::max_local_entry_size()) + << ")\n"; + std::cout << " BUILDCACHE_MAX_REMOTE_ENTRY_SIZE: " + << bcache::config::max_remote_entry_size() << " (" + << bcache::file::human_readable_size(bcache::config::max_remote_entry_size()) + << ")\n"; + std::cout << " BUILDCACHE_DEBUG: " << bcache::config::debug() << "\n"; + std::cout << " BUILDCACHE_HARD_LINKS: " << (bcache::config::hard_links() ? "true" : "false") << "\n"; - std::cout << " BUILDCACHE_COMPRESS: " + std::cout << " BUILDCACHE_COMPRESS: " << (bcache::config::compress() ? "true" : "false") << "\n"; - std::cout << " BUILDCACHE_PERF: " << (bcache::config::perf() ? "true" : "false") - << "\n"; - std::cout << " BUILDCACHE_DISABLE: " << (bcache::config::disable() ? "true" : "false") - << "\n"; + std::cout << " BUILDCACHE_PERF: " + << (bcache::config::perf() ? "true" : "false") << "\n"; + std::cout << " BUILDCACHE_DISABLE: " + << (bcache::config::disable() ? "true" : "false") << "\n"; } } catch (const std::exception& e) { std::cerr << "*** Unexpected error: " << e.what() << "\n"; diff --git a/src/wrappers/program_wrapper.cpp b/src/wrappers/program_wrapper.cpp index b483f53..37dcbf1 100644 --- a/src/wrappers/program_wrapper.cpp +++ b/src/wrappers/program_wrapper.cpp @@ -132,14 +132,14 @@ bool program_wrapper_t::handle_command(int& return_code) { // Note: We do not want to create cache entries for failed program runs. We could, but that // would run the risk of caching intermittent faults for instance. if (result.return_code == 0) { - // Add the entry to the local cache. - const cache_entry_t local_entry( + // Add the entry to the cache. + const cache_entry_t entry( file_ids, config::compress() ? cache_entry_t::comp_mode_t::ALL : cache_entry_t::comp_mode_t::NONE, result.std_out, result.std_err, result.return_code); - m_cache.add(hash, local_entry, build_files, allow_hard_links); + m_cache.add(hash, entry, build_files, allow_hard_links); } // Everything's ok!