encounter/buildcache
0
0
Fork 0
mirror of https://github.com/encounter/buildcache.git synced 2026-07-09 18:18:50 -07:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #24 from mbitsnbites/feature/create_cache_class

Browse Source 
Move the cache logic to a new class cache_t
This commit is contained in:
mbitsnbites
2019-02-12 12:10:07 +01:00
committed by GitHub
parent 37c3f1f4f9 a995b720a8
commit cc66ffa6aa
5 changed files with 223 additions and 158 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/cache/CMakeLists.txt
Vendored
+2
View File
@@ -18,6 +18,8 @@
#---------------------------------------------------------------------------------------------------
set(CACHE_SRCS
cache.cpp
cache.hpp
cache_entry.cpp
cache_entry.hpp
local_cache.cpp
src/cache/cache.cpp
Vendored
+140
View File
@@ -0,0 +1,140 @@
//--------------------------------------------------------------------------------------------------
// Copyright (c) 2019 Marcus Geelnard
//
// This software is provided 'as-is', without any express or implied warranty. In no event will the
// authors be held liable for any damages arising from the use of this software.
//
// Permission is granted to anyone to use this software for any purpose, including commercial
// applications, and to alter it and redistribute it freely, subject to the following restrictions:
//
// 1. The origin of this software must not be misrepresented; you must not claim that you wrote
// the original software. If you use this software in a product, an acknowledgment in the
// product documentation would be appreciated but is not required.
//
// 2. Altered source versions must be plainly marked as such, and must not be misrepresented as
// being the original software.
//
// 3. This notice may not be removed or altered from any source distribution.
//--------------------------------------------------------------------------------------------------
#include <cache/cache.hpp>
#include <base/debug_utils.hpp>
#include <base/file_utils.hpp>
#include <sys/perf_utils.hpp>
#include <iostream>
namespace bcache {
bool cache_t::lookup(const hasher_t::hash_t hash,
const std::map<std::string, std::string>& file_paths,
const bool allow_hard_links,
int& return_code) {
// First try the local cache.
if (lookup_in_local_cache(hash, file_paths, allow_hard_links, return_code)) {
return true;
}
// Then try the remote cache.
if (lookup_in_remote_cache(hash, file_paths, return_code)) {
return true;
}
return false;
}
void cache_t::add(const hasher_t::hash_t hash,
const cache_entry_t& entry,
const std::map<std::string, std::string>& file_paths,
const bool allow_hard_links) {
PERF_START(ADD_TO_CACHE);
// Add the entry to the local cache.
m_local_cache.add(hash, entry, file_paths, allow_hard_links);
// Add the entry to the remote cache.
if (m_remote_cache.is_connected()) {
// Note: We always compress entries for the remote cache.
const cache_entry_t remote_entry(entry.file_ids(),
cache_entry_t::comp_mode_t::ALL,
entry.std_out(),
entry.std_err(),
entry.return_code());
m_remote_cache.add(hash, entry, file_paths);
}
PERF_STOP(ADD_TO_CACHE);
}
bool cache_t::lookup_in_local_cache(const hasher_t::hash_t hash,
const std::map<std::string, std::string>& file_paths,
const bool allow_hard_links,
int& return_code) {
PERF_START(CACHE_LOOKUP);
// Note: The lookup will give us a lock file that is locked until we go out of scope.
auto lookup_result = m_local_cache.lookup(hash);
const auto& cached_entry = lookup_result.first;
PERF_STOP(CACHE_LOOKUP);
if (!cached_entry) {
return false;
}
// Copy all files from the cache to their respective target paths.
// Note: If there is a mismatch in the expected (target) files and the actual (cached)
// files, this will throw an exception (i.e. fall back to full program execution).
PERF_START(RETRIEVE_CACHED_FILES);
for (const auto& file_id : cached_entry.file_ids()) {
const auto& target_path = file_paths.at(file_id);
debug::log(debug::INFO) << "Cache hit (" << hash.as_string() << "): " << file_id << " => "
<< target_path;
const auto is_compressed = (cached_entry.compression_mode() == cache_entry_t::comp_mode_t::ALL);
m_local_cache.get_file(hash, file_id, target_path, is_compressed, allow_hard_links);
}
PERF_STOP(RETRIEVE_CACHED_FILES);
// Return/print the cached program results.
std::cout << cached_entry.std_out();
std::cerr << cached_entry.std_err();
return_code = cached_entry.return_code();
return true;
}
bool cache_t::lookup_in_remote_cache(const hasher_t::hash_t hash,
const std::map<std::string, std::string>& file_paths,
int& return_code) {
// Start by trying to connect to the remote cache.
if (!m_remote_cache.connect()) {
return false;
}
PERF_START(CACHE_LOOKUP);
const auto cached_entry = m_remote_cache.lookup(hash);
PERF_STOP(CACHE_LOOKUP);
if (!cached_entry) {
return false;
}
// Copy all files from the cache to their respective target paths.
// Note: If there is a mismatch in the expected (target) files and the actual (cached)
// files, this will throw an exception (i.e. fall back to full program execution).
PERF_START(RETRIEVE_CACHED_FILES);
for (const auto& file_id : cached_entry.file_ids()) {
const auto& target_path = file_paths.at(file_id);
debug::log(debug::INFO) << "Remote cache hit (" << hash.as_string() << "): " << file_id
<< " => " << target_path;
const auto is_compressed = (cached_entry.compression_mode() == cache_entry_t::comp_mode_t::ALL);
m_remote_cache.get_file(hash, file_id, target_path, is_compressed);
}
PERF_STOP(RETRIEVE_CACHED_FILES);
// Return/print the cached program results.
std::cout << cached_entry.std_out();
std::cerr << cached_entry.std_err();
return_code = cached_entry.return_code();
return true;
}
} // namespace bcache
src/cache/cache.hpp
Vendored
+73
View File
@@ -0,0 +1,73 @@
//--------------------------------------------------------------------------------------------------
// Copyright (c) 2019 Marcus Geelnard
//
// This software is provided 'as-is', without any express or implied warranty. In no event will the
// authors be held liable for any damages arising from the use of this software.
//
// Permission is granted to anyone to use this software for any purpose, including commercial
// applications, and to alter it and redistribute it freely, subject to the following restrictions:
//
// 1. The origin of this software must not be misrepresented; you must not claim that you wrote
// the original software. If you use this software in a product, an acknowledgment in the
// product documentation would be appreciated but is not required.
//
// 2. Altered source versions must be plainly marked as such, and must not be misrepresented as
// being the original software.
//
// 3. This notice may not be removed or altered from any source distribution.
//--------------------------------------------------------------------------------------------------
#ifndef BUILDCACHE_CACHE_HPP_
#define BUILDCACHE_CACHE_HPP_
#include <base/hasher.hpp>
#include <cache/cache_entry.hpp>
#include <cache/local_cache.hpp>
#include <cache/remote_cache.hpp>
#include <map>
#include <string>
namespace bcache {
/// @brief An interface to the different caches.
class cache_t {
public:
/// @brief Perform a cache lookup.
/// @param hash The hash of the cache entry.
/// @param file_paths Paths to the actual files in the local file system (map from file ID to
/// path).
/// @param allow_hard_links True if we are allowed to use hard links.
/// @param[out] return_code The return code of the program.
/// @returns true if we had a cache hit, otherwise false.
bool lookup(const hasher_t::hash_t hash,
const std::map<std::string, std::string>& file_paths,
const bool allow_hard_links,
int& return_code);
/// @brief Add a new entry to the cache(s).
/// @param hash The hash of the cache entry.
/// @param entry The cache entry description.
/// @param file_paths Paths to the actual files in the local file system (map from file ID to
/// path).
/// @param allow_hard_links True if we are allowed to use hard links.
void add(const hasher_t::hash_t hash,
const cache_entry_t& entry,
const std::map<std::string, std::string>& file_paths,
const bool allow_hard_links);
private:
bool lookup_in_local_cache(const hasher_t::hash_t hash,
const std::map<std::string, std::string>& file_paths,
const bool allow_hard_links,
int& return_code);
bool lookup_in_remote_cache(const hasher_t::hash_t hash,
const std::map<std::string, std::string>& file_paths,
int& return_code);
local_cache_t m_local_cache;
remote_cache_t m_remote_cache;
};
} // namespace bcache
#endif // BUILDCACHE_CACHE_HPP_
src/wrappers/program_wrapper.cpp
+6 -119
View File
@@ -101,22 +101,17 @@ bool program_wrapper_t::handle_command(int& return_code) {
// Check if we can use hard links.
const auto allow_hard_links = config::hard_links() && capabilites.hard_links();
// Look up the entry in the local cache.
if (lookup_in_local_cache(hash, allow_hard_links, return_code)) {
return true;
}
// Look up the entry in the remote cache.
if (lookup_in_remote_cache(hash, return_code)) {
return true;
}
// Get the list of files that are expected to be generated by the command. This is in fact a
// map of file ID:s to their corresponding file path.
PERF_START(GET_BUILD_FILES);
const auto build_files = get_build_files();
PERF_STOP(GET_BUILD_FILES);
// Look up the entry in the cache(s).
if (m_cache.lookup(hash, build_files, allow_hard_links, return_code)) {
return true;
}
// Extract only the file ID:s.
std::vector<std::string> file_ids;
{
@@ -144,16 +139,7 @@ bool program_wrapper_t::handle_command(int& return_code) {
result.std_out,
result.std_err,
result.return_code);
add_to_local_cache(hash, local_entry, build_files, allow_hard_links);
// Add the entry to the remote cache.
// Note: We always compress entries for the remote cache.
const cache_entry_t remote_entry(file_ids,
cache_entry_t::comp_mode_t::ALL,
result.std_out,
result.std_err,
result.return_code);
add_to_remote_cache(hash, remote_entry, build_files);
m_cache.add(hash, local_entry, build_files, allow_hard_links);
}
// Everything's ok!
@@ -171,105 +157,6 @@ bool program_wrapper_t::handle_command(int& return_code) {
return false;
}
bool program_wrapper_t::lookup_in_local_cache(const hasher_t::hash_t hash,
const bool allow_hard_links,
int& return_code) {
PERF_START(CACHE_LOOKUP);
// Note: The lookup will give us a lock file that is locked until we go out of scope.
auto lookup_result = m_cache.lookup(hash);
const auto& cached_entry = lookup_result.first;
PERF_STOP(CACHE_LOOKUP);
if (!cached_entry) {
return false;
}
// Get the list of files that are expected to be generated by the command.
PERF_START(GET_BUILD_FILES);
const auto target_files = get_build_files();
PERF_STOP(GET_BUILD_FILES);
// Copy all files from the cache to their respective target paths.
// Note: If there is a mismatch in the expected (target) files and the actual (cached)
// files, this will throw an exception (i.e. fall back to full program execution).
PERF_START(RETRIEVE_CACHED_FILES);
for (const auto& file_id : cached_entry.file_ids()) {
const auto& target_path = target_files.at(file_id);
debug::log(debug::INFO) << "Cache hit (" << hash.as_string() << "): " << file_id << " => "
<< target_path;
const auto is_compressed = (cached_entry.compression_mode() == cache_entry_t::comp_mode_t::ALL);
m_cache.get_file(hash, file_id, target_path, is_compressed, allow_hard_links);
}
PERF_STOP(RETRIEVE_CACHED_FILES);
// Return/print the cached program results.
std::cout << cached_entry.std_out();
std::cerr << cached_entry.std_err();
return_code = cached_entry.return_code();
return true;
}
void program_wrapper_t::add_to_local_cache(const hasher_t::hash_t hash,
const cache_entry_t& entry,
const std::map<std::string, std::string>& file_paths,
const bool allow_hard_links) {
PERF_START(ADD_TO_CACHE);
m_cache.add(hash, entry, file_paths, allow_hard_links);
PERF_STOP(ADD_TO_CACHE);
}
bool program_wrapper_t::lookup_in_remote_cache(const hasher_t::hash_t hash, int& return_code) {
// Start by trying to connect to the remote cache.
if (!m_remote_cache.connect()) {
return false;
}
PERF_START(CACHE_LOOKUP);
const auto cached_entry = m_remote_cache.lookup(hash);
PERF_STOP(CACHE_LOOKUP);
if (!cached_entry) {
return false;
}
// Get the list of files that are expected to be generated by the command.
PERF_START(GET_BUILD_FILES);
const auto target_files = get_build_files();
PERF_STOP(GET_BUILD_FILES);
// Copy all files from the cache to their respective target paths.
// Note: If there is a mismatch in the expected (target) files and the actual (cached)
// files, this will throw an exception (i.e. fall back to full program execution).
PERF_START(RETRIEVE_CACHED_FILES);
for (const auto& file_id : cached_entry.file_ids()) {
const auto& target_path = target_files.at(file_id);
debug::log(debug::INFO) << "Remote cache hit (" << hash.as_string() << "): " << file_id
<< " => " << target_path;
const auto is_compressed = (cached_entry.compression_mode() == cache_entry_t::comp_mode_t::ALL);
m_remote_cache.get_file(hash, file_id, target_path, is_compressed);
}
PERF_STOP(RETRIEVE_CACHED_FILES);
// Return/print the cached program results.
std::cout << cached_entry.std_out();
std::cerr << cached_entry.std_err();
return_code = cached_entry.return_code();
return true;
}
void program_wrapper_t::add_to_remote_cache(const hasher_t::hash_t hash,
const cache_entry_t& entry,
const std::map<std::string, std::string>& file_paths) {
if (!m_remote_cache.is_connected()) {
return;
}
PERF_START(ADD_TO_CACHE);
m_remote_cache.add(hash, entry, file_paths);
PERF_STOP(ADD_TO_CACHE);
}
//--------------------------------------------------------------------------------------------------
// Default wrapper interface implementation. Wrappers are expected to override the parts that are
src/wrappers/program_wrapper.hpp
+2 -39
View File
@@ -22,8 +22,7 @@
#include <base/file_utils.hpp>
#include <base/string_list.hpp>
#include <cache/local_cache.hpp>
#include <cache/remote_cache.hpp>
#include <cache/cache.hpp>
#include <string>
@@ -115,43 +114,7 @@ protected:
const string_list_t& m_args;
private:
/// @brief Perform a cache lookup in the local cache.
/// @param hash The hash of the cache entry.
/// @param allow_hard_links True if we are allowed to use hard links.
/// @param[out] return_code The return code of the program.
/// @returns true if we had a cache hit, otherwise false.
bool lookup_in_local_cache(const hasher_t::hash_t hash,
const bool allow_hard_links,
int& return_code);
/// @brief Add a new entry to the local cache.
/// @param hash The hash of the cache entry.
/// @param entry The cache entry description.
/// @param file_paths Paths to the actual files in the local file system (map from file ID to
/// path).
/// @param allow_hard_links True if we are allowed to use hard links.
void add_to_local_cache(const hasher_t::hash_t hash,
const cache_entry_t& entry,
const std::map<std::string, std::string>& file_paths,
const bool allow_hard_links);
/// @brief Perform a cache lookup in the remote cache.
/// @param hash The hash of the cache entry.
/// @param[out] return_code The return code of the program.
/// @returns true if we had a cache hit, otherwise false.
bool lookup_in_remote_cache(const hasher_t::hash_t hash, int& return_code);
/// @brief Add a new entry to the remote cache.
/// @param hash The hash of the cache entry.
/// @param entry The cache entry description.
/// @param file_paths Paths to the actual files in the local file system (map from file ID to
/// path).
void add_to_remote_cache(const hasher_t::hash_t hash,
const cache_entry_t& entry,
const std::map<std::string, std::string>& file_paths);
local_cache_t m_cache;
remote_cache_t m_remote_cache;
cache_t m_cache;
};
} // namespace bcache