izzy/Torch
0
0
Fork 0
mirror of https://github.com/izzy2lost/Torch.git synced 2026-07-06 00:18:35 -07:00
Code Issues Packages Projects Releases Wiki Activity

Implemented full file change detection

Browse Source
This commit is contained in:
KiritoDv
2024-03-29 03:03:55 -06:00
committed by Lywx
parent d8df2a4826
commit c7c512e7a0
4 changed files with 54 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Companion.cpp
+47 -87
View File
@@ -199,19 +199,6 @@ void Companion::ExtractNode(YAML::Node& node, std::string& name, SWrapper* binar
input.close();
result = impl->parse_modding(data, node);
} else if(this->gNodeIsCacheable && impl->IsCacheable()) {
if(this->gCacheData.contains(this->gCurrentCacheHash)) {
result = impl->CreateDataPointer();
result->get()->FromCacheBuffer(this->GetCache(name));
} else {
result = impl->parse(this->gRomData, node);
if(result.has_value()) {
auto cache = result.value()->ToCacheBuffer();
if(cache.has_value()) {
this->StoreCache(name, cache.value());
}
}
}
} else {
result = impl->parse(this->gRomData, node);
}
@@ -405,92 +392,60 @@ void Companion::ParseCurrentFileConfig(YAML::Node node) {
}
this->gEnablePadGen = GetSafeNode<bool>(node, "autopads", true);
this->gNodeIsCacheable = GetSafeNode<bool>(node, "cacheable", true);
this->gNodeForceProcessing = GetSafeNode<bool>(node, "force", false);
}
void Companion::ParseCache() {
const std::string out = "torch.cache.yml";
void Companion::ParseHash() {
const std::string out = "torch.hash.yml";
if(fs::exists(out)) {
this->gCacheNode = YAML::LoadFile(out);
this->gHashNode = YAML::LoadFile(out);
} else {
this->gCacheNode = YAML::Node();
this->gHashNode = YAML::Node();
}
}
void Companion::PrepareCache(const std::string& path) {
std::string ExportTypeToString(ExportType type) {
switch (type) {
case ExportType::Binary: return "Binary";
case ExportType::Header: return "Header";
case ExportType::Code: return "Code";
case ExportType::Modding: return "Modding";
default:
throw std::runtime_error("Invalid ExportType");
}
}
bool Companion::NodeHasChanges(const std::string& path) {
if(this->gConfig.exporterType == ExportType::Modding) {
return true;
}
std::ifstream yaml(path);
bool hasChanges = false;
const std::vector<uint8_t> data = std::vector<uint8_t>(std::istreambuf_iterator( yaml ), {});
this->gCurrentCacheHash = CalculateHash(data);
this->gCurrentHash = CalculateHash(data);
if(this->gCacheNode[path]) {
const auto cache = GetSafeNode<std::string>(this->gCacheNode, path);
if(cache == this->gCurrentCacheHash) {
const auto tdir = "tcache/" + cache;
if(!fs::exists(tdir)) {
fs::create_directories(tdir);
} else {
for(auto& entry : fs::directory_iterator("tcache/" + cache)) {
const auto ext = entry.path().extension().string();
if(this->gHashNode[path]) {
auto entry = GetSafeNode<YAML::Node>(this->gHashNode, path);
const auto hash = GetSafeNode<std::string>(entry, "hash");
auto modes = GetSafeNode<YAML::Node>(entry, "extracted");
auto extracted = GetSafeNode<bool>(modes, ExportTypeToString(this->gConfig.exporterType));
if(ext == ".data") {
std::ifstream file(entry.path(), std::ios::binary);
std::vector<uint8_t> buffer = std::vector<uint8_t>(std::istreambuf_iterator(file), {});
file.close();
std::string cpath = entry.path().filename().string();
std::replace(cpath.begin(), cpath.end(), '+', '/');
this->gCacheData[cache][cpath] = buffer;
} else if(ext == ".yaml") {
std::ifstream file(entry.path(), std::ios::binary);
YAML::Node dependencies = YAML::LoadFile(entry.path().string());
for(auto dep = dependencies.begin(); dep != dependencies.end(); ++dep) {
this->gAssetDependencies[this->gCurrentFile][dep->first.as<std::string>()] = std::make_pair(dep->second.as<YAML::Node>(), false);
}
}
}
}
} else {
hasChanges = true;
this->gCacheNode[path] = this->gCurrentCacheHash;
fs::remove("cache/" + cache);
if(hash == this->gCurrentHash && extracted) {
SPDLOG_INFO("Skipping {} as it has not changed", path);
return false;
}
} else {
this->gCacheNode[path] = this->gCurrentCacheHash;
}
if(hasChanges) {
std::ofstream file("torch.cache.yml", std::ios::binary);
file << this->gCacheNode;
file.close();
}
}
void Companion::StoreCache(const std::string path, const std::vector<uint8_t>& value) {
std::string outpath = path;
std::replace(outpath.begin(), outpath.end(), '/', '+');
YAML::Node dependencies = YAML::Node();
// Dump asset dependencies
for (auto [fst, snd] : this->gAssetDependencies[this->gCurrentFile]) {
dependencies[fst] = snd.first;
this->gHashNode[path] = YAML::Node();
this->gHashNode[path]["hash"] = this->gCurrentHash;
this->gHashNode[path]["extracted"] = YAML::Node();
for(size_t m = 0; m < static_cast<size_t>(ExportType::Modding); m++) {
this->gHashNode[path]["extracted"][ExportTypeToString(static_cast<ExportType>(m))] = false;
}
std::ofstream deps("tcache/" + this->gCurrentCacheHash + "/" + outpath + "-deps.yaml", std::ios::binary);
deps << dependencies;
deps.close();
std::ofstream file("tcache/" + this->gCurrentCacheHash + "/" + outpath + ".data", std::ios::binary);
file.write(reinterpret_cast<const char*>(value.data()), value.size());
file.close();
}
std::vector<uint8_t>& Companion::GetCache(const std::string path) {
std::string outpath = path;
std::replace(outpath.begin(), outpath.end(), '+', '/');
return this->gCacheData[this->gCurrentCacheHash][outpath];
return true;
}
void Companion::Process() {
@@ -633,7 +588,7 @@ void Companion::Process() {
}
}
this->ParseCache();
this->ParseHash();
SPDLOG_INFO("------------------------------------------------");
spdlog::set_pattern(line);
@@ -669,8 +624,6 @@ void Companion::Process() {
this->gCurrentDirectory = relative(entry.path(), path).replace_extension("");
this->gCurrentFile = yamlPath;
this->PrepareCache(yamlPath);
// Set compressed file offsets and compression type
if (auto segments = root[":config"]["segments"]) {
if (segments.IsSequence() && segments.size() > 0) {
@@ -757,8 +710,8 @@ void Companion::Process() {
this->ParseCurrentFileConfig(root[":config"]);
}
if(this->gNodeIsCacheable) {
this->PrepareCache(yamlPath);
if(!this->NodeHasChanges(yamlPath) && !this->gNodeForceProcessing) {
continue;
}
spdlog::set_pattern(regular);
@@ -953,6 +906,7 @@ void Companion::Process() {
}
file.close();
this->gHashNode[this->gCurrentFile]["extracted"][ExportTypeToString(this->gConfig.exporterType)] = true;
}
}
@@ -962,6 +916,12 @@ void Companion::Process() {
vWriter.Close();
wrapper->Close();
}
// Write entries hash
std::ofstream file("torch.hash.yml", std::ios::binary);
file << this->gHashNode;
file.close();
auto end = duration_cast<milliseconds>(system_clock::now().time_since_epoch());
auto level = spdlog::get_level();
spdlog::set_level(spdlog::level::info);
src/Companion.h
+5 -10
View File
@@ -86,7 +86,7 @@ public:
void Init(ExportType type);
void PrepareCache(const std::string& string);
bool NodeHasChanges(const std::string& string);
void Process();
@@ -125,11 +125,11 @@ private:
TorchConfig gConfig;
YAML::Node gModdingConfig;
fs::path gCurrentDirectory;
std::string gCurrentCacheHash;
std::string gCurrentHash;
std::vector<uint8_t> gRomData;
std::filesystem::path gRomPath;
bool gNodeIsCacheable;
YAML::Node gCacheNode;
bool gNodeForceProcessing = false;
YAML::Node gHashNode;
std::shared_ptr<N64::Cartridge> gCartridge;
std::unordered_map<std::string, std::unordered_map<int32_t, std::string>> gEnums;
@@ -146,19 +146,14 @@ private:
std::variant<std::vector<std::string>, std::string> gWriteOrder;
std::unordered_map<std::string, std::shared_ptr<BaseFactory>> gFactories;
std::unordered_map<std::string, std::string> gModdedAssetPaths;
std::unordered_map<std::string, std::unordered_map<std::string, std::vector<uint8_t>>> gCacheData;
std::unordered_map<std::string, std::map<std::string, std::vector<WriteEntry>>> gWriteMap;
std::unordered_map<std::string, std::map<std::string, std::pair<YAML::Node, bool>>> gAssetDependencies;
std::unordered_map<std::string, std::unordered_map<uint32_t, std::tuple<std::string, YAML::Node>>> gAddrMap;
void ParseEnums(std::string& file);
void ParseCache();
void ParseHash();
void ParseModdingConfig();
void ParseCurrentFileConfig(YAML::Node node);
void RegisterFactory(const std::string& type, const std::shared_ptr<BaseFactory>& factory);
void StoreCache(std::string path, const std::vector<uint8_t>& value);
std::vector<uint8_t>& GetCache(std::string path);
void ExtractNode(YAML::Node& node, std::string& name, SWrapper* binary);
};
src/factories/BaseFactory.h
+2 -14
View File
@@ -37,7 +37,7 @@ enum class ExportType {
Header,
Code,
Binary,
Modding,
Modding
};
template<typename T>
@@ -71,16 +71,7 @@ T GetSafeNode(YAML::Node& node, const std::string& key, const T& def) {
return node[key].as<T>();
}
class IParsedData {
public:
virtual std::optional<std::vector<uint8_t>> ToCacheBuffer() {
return std::nullopt;
}
virtual void FromCacheBuffer(std::vector<uint8_t>& buffer) {
}
};
class IParsedData{};
template<typename T>
class GenericData : public IParsedData {
@@ -113,9 +104,6 @@ public:
virtual uint32_t GetAlignment() {
return 4;
}
virtual bool IsCacheable() {
return false;
}
virtual std::optional<std::shared_ptr<IParsedData>> CreateDataPointer() {
return std::nullopt;
}
src/factories/DisplayListFactory.h
-16
View File
@@ -10,16 +10,6 @@ public:
DListData() {}
DListData(std::vector<uint32_t> gfxs) : mGfxs(gfxs) {}
std::optional<std::vector<uint8_t>> ToCacheBuffer() override {
std::vector<uint8_t> buffer;
buffer.insert(buffer.end(), reinterpret_cast<uint8_t *>(mGfxs.data()), reinterpret_cast<uint8_t *>(mGfxs.data()) + mGfxs.size() * sizeof(uint32_t));
return buffer;
}
void FromCacheBuffer(std::vector<uint8_t>& buffer) override {
mGfxs = std::vector(reinterpret_cast<uint32_t *>(buffer.data()), reinterpret_cast<uint32_t *>(buffer.data()) + buffer.size() / sizeof(uint32_t));
}
};
class DListHeaderExporter : public BaseExporter {
@@ -47,10 +37,4 @@ public:
uint32_t GetAlignment() override {
return 8;
}
bool IsCacheable() override {
return true;
}
virtual std::optional<std::shared_ptr<IParsedData>> CreateDataPointer() {
return std::make_shared<DListData>();
}
};