Added WIP Cache System

src/Companion.cpp

@@ -187,26 +187,35 @@ void Companion::ExtractNode(YAML::Node& node, std::string& name, SWrapper* binar
     }
 
     std::optional<std::shared_ptr<IParsedData>> result;
-    if(this->gConfig.modding) {
-        if(impl->SupportModdedAssets() && this->gModdedAssetPaths.contains(name)) {
+    if(this->gConfig.modding && impl->SupportModdedAssets() && this->gModdedAssetPaths.contains(name)) {
+        auto path = fs::path(this->gConfig.moddingPath) / this->gModdedAssetPaths[name];
+        if(!fs::exists(path)) {
+            SPDLOG_ERROR("Modded asset {} not found", this->gModdedAssetPaths[name]);
+            return;
+        }
 
-            auto path = fs::path(this->gConfig.moddingPath) / this->gModdedAssetPaths[name];
-            if(!fs::exists(path)) {
-                SPDLOG_ERROR("Modded asset {} not found", this->gModdedAssetPaths[name]);
-                return;
-            }
+        std::ifstream input(path, std::ios::binary);
+        std::vector<uint8_t> data = std::vector<uint8_t>( std::istreambuf_iterator( input ), {});
+        input.close();
 
-            std::ifstream input(path, std::ios::binary);
-            std::vector<uint8_t> data = std::vector<uint8_t>( std::istreambuf_iterator( input ), {});
-            input.close();
-
-            result = factory->get()->parse_modding(data, node);
+        result = impl->parse_modding(data, node);
+    } else if(this->gNodeIsCacheable && impl->IsCacheable()) {
+        if(this->gCacheData.contains(name)) {
+            result = std::make_shared<IParsedData>();
+            result->get()->FromCacheBuffer(this->gCacheData[name]);
         } else {
-            result = factory->get()->parse(this->gRomData, node);
+            result = impl->parse(this->gRomData, node);
+            if(result.has_value()) {
+                auto cache = result.value()->ToCacheBuffer();
+                if(cache.has_value()) {
+                    this->StoreCache(cache.value());
+                }
+            }
         }
     } else {
-        result = factory->get()->parse(this->gRomData, node);
+        result = impl->parse(this->gRomData, node);
     }
+
     if(!result.has_value()){
         SPDLOG_ERROR("Failed to process {}", name);
         return;
@@ -396,6 +405,46 @@ void Companion::ParseCurrentFileConfig(YAML::Node node) {
     }
 
     this->gEnablePadGen = GetSafeNode<bool>(node, "autopads", true);
+    this->gNodeIsCacheable = GetSafeNode<bool>(node, "cacheable", true);
+}
+
+void Companion::PrepareCache(const std::string& path) {
+    std::ifstream yaml(path);
+    const std::vector<uint8_t> data = std::vector<uint8_t>(std::istreambuf_iterator( yaml ), {});
+    this->gCurrentCacheHash = CalculateHash(data);
+
+    const std::string out = "torch.cache.yml";
+    YAML::Node root;
+
+    if(fs::exists(out)) {
+        root = YAML::LoadFile(out);
+    } else {
+        root = YAML::Node();
+    }
+
+    if(root[this->gCurrentCacheHash]) {
+        const auto cache = GetSafeNode<std::string>(root, this->gCurrentCacheHash);
+        if(cache == this->gCurrentCacheHash) {
+            std::ifstream file("cache/" + cache, std::ios::binary);
+            this->gCacheData[cache] = std::vector<uint8_t>(std::istreambuf_iterator( file ), {});
+            file.close();
+        } else {
+            root[path] = this->gCurrentCacheHash;
+            fs::remove("cache/" + cache);
+        }
+    } else {
+        root[path] = this->gCurrentCacheHash;
+    }
+
+    std::ofstream file(out, std::ios::binary);
+    file << root;
+    file.close();
+}
+
+void Companion::StoreCache(const std::vector<uint8_t>& value) {
+    std::ofstream file("cache/" + this->gCurrentCacheHash, std::ios::binary);
+    file.write(reinterpret_cast<const char*>(value.data()), value.size());
+    file.close();
 }
 
 void Companion::Process() {
@@ -572,6 +621,8 @@ void Companion::Process() {
         this->gCurrentDirectory = relative(entry.path(), path).replace_extension("");
         this->gCurrentFile = yamlPath;
 
+        this->PrepareCache(yamlPath);
+
         // Set compressed file offsets and compression type
         if (auto segments = root[":config"]["segments"]) {
             if (segments.IsSequence() && segments.size() > 0) {
@@ -658,6 +709,10 @@ void Companion::Process() {
             this->ParseCurrentFileConfig(root[":config"]);
         }
 
+        if(this->gNodeIsCacheable) {
+            this->PrepareCache(yamlPath);
+        }
+
         spdlog::set_pattern(regular);
         SPDLOG_INFO("------------------------------------------------");
         spdlog::set_pattern(line);
@@ -671,7 +726,6 @@ void Companion::Process() {
                 continue;
             }
 
-
             // Parse horizontal assets
             if(assetNode["files"]){
                 auto segment = assetNode["segment"] ? assetNode["segment"].as<uint8_t>() : -1;
@@ -1081,6 +1135,7 @@ std::optional<YAML::Node> Companion::AddAsset(YAML::Node asset) {
 
     auto rom = this->GetRomData();
     auto factory = this->GetFactory(type);
+
     if(!factory.has_value()) {
         return std::nullopt;
     }

src/Companion.h

@@ -85,6 +85,9 @@ public:
     }
 
     void Init(ExportType type);
+
+    void PrepareCache(const std::string& string);
+
     void Process();
 
     bool IsOTRMode() const { return this->gConfig.otrMode; }
@@ -122,8 +125,10 @@ private:
     TorchConfig gConfig;
     YAML::Node gModdingConfig;
     fs::path gCurrentDirectory;
+    std::string gCurrentCacheHash;
     std::vector<uint8_t> gRomData;
     std::filesystem::path gRomPath;
+    bool gNodeIsCacheable;
     std::shared_ptr<N64::Cartridge> gCartridge;
     std::unordered_map<std::string, std::unordered_map<int32_t, std::string>> gEnums;
 
@@ -140,6 +145,7 @@ private:
     std::variant<std::vector<std::string>, std::string> gWriteOrder;
     std::unordered_map<std::string, std::shared_ptr<BaseFactory>> gFactories;
     std::unordered_map<std::string, std::string> gModdedAssetPaths;
+    std::unordered_map<std::string, std::vector<uint8_t>> gCacheData;
     std::unordered_map<std::string, std::map<std::string, std::vector<WriteEntry>>> gWriteMap;
     std::unordered_map<std::string, std::map<std::string, std::pair<YAML::Node, bool>>> gAssetDependencies;
     std::unordered_map<std::string, std::unordered_map<uint32_t, std::tuple<std::string, YAML::Node>>> gAddrMap;
@@ -148,5 +154,8 @@ private:
     void ParseModdingConfig();
     void ParseCurrentFileConfig(YAML::Node node);
     void RegisterFactory(const std::string& type, const std::shared_ptr<BaseFactory>& factory);
+
+    void StoreCache(const std::vector<uint8_t>& value);
+
     void ExtractNode(YAML::Node& node, std::string& name, SWrapper* binary);
 };

src/factories/BaseFactory.h

@@ -71,7 +71,16 @@ T GetSafeNode(YAML::Node& node, const std::string& key, const T& def) {
     return node[key].as<T>();
 }
 
-class IParsedData {};
+class IParsedData {
+public:
+    virtual std::optional<std::vector<uint8_t>> ToCacheBuffer() {
+        return std::nullopt;
+    }
+
+    virtual void FromCacheBuffer(std::vector<uint8_t>& buffer) {
+
+    }
+};
 
 template<typename T>
 class GenericData : public IParsedData {
@@ -103,7 +112,10 @@ public:
     }
     virtual uint32_t GetAlignment() {
         return 4;
-    };
+    }
+    virtual bool IsCacheable() {
+        return false;
+    }
 private:
     virtual std::unordered_map<ExportType, std::shared_ptr<BaseExporter>> GetExporters() = 0;
 };