Updated dkr_extractor to be multithreaded.

Makefile

@@ -15,6 +15,7 @@ VERSION := us_1.0
 ifneq ($(MAKECMDGOALS),clean)
 ifneq ($(MAKECMDGOALS),clean_lib)
 ifneq ($(MAKECMDGOALS),clean_src)
+ifneq ($(MAKECMDGOALS),reset)
 
 ########## QEMU_IRIX ###########
 
@@ -91,6 +92,7 @@ endif
 endif
 endif
 endif
+endif
 
 ################ Target Executable and Sources ###############
 
@@ -302,6 +304,18 @@ LD_SCRIPT = $(TARGET).ld
 
 all: $(BUILD_DIR)/$(TARGET).z64
 
+reset: 
+ifneq ($(wildcard ./build/.*),) 
+	rm -r build
+endif 
+ifneq ($(wildcard ./assets/.*),) 
+	rm -r assets
+endif 
+ifneq ($(wildcard ./ucode/.*),) 
+	rm -r ucode
+endif 
+	@echo "Done." 
+
 clean: 
 ifneq ($(wildcard ./build/.*),) 
 	rm -r build

tools/Makefile

@@ -15,7 +15,7 @@ dkr_decompressor_SOURCES := dkr_decompressor.cpp $(wildcard dkr_decompressor_src
 dkr_decompressor_CXXFLAGS := -lstdc++fs -lcrypto -lssl
 
 dkr_extractor_SOURCES := dkr_extractor.cpp $(wildcard dkr_decompressor_src/*.cpp dkr_decompressor_src/*.c dkr_extractor_classes/*.cpp n64graphics/*.c)
-dkr_extractor_CXXFLAGS := -lstdc++fs -lcrypto -lssl
+dkr_extractor_CXXFLAGS := -lstdc++fs -lcrypto -lssl -lpthread
 
 dkr_texbuilder_SOURCES := dkr_texbuilder.cpp $(wildcard dkr_decompressor_src/*.cpp dkr_decompressor_src/*.c n64graphics/*.c)
 dkr_texbuilder_CXXFLAGS := -lstdc++fs

tools/dkr_extractor_classes/ThreadPool.h

@@ -0,0 +1,98 @@
+#ifndef THREAD_POOL_H
+#define THREAD_POOL_H
+
+#include <vector>
+#include <queue>
+#include <memory>
+#include <thread>
+#include <mutex>
+#include <condition_variable>
+#include <future>
+#include <functional>
+#include <stdexcept>
+
+class ThreadPool {
+public:
+    ThreadPool(size_t);
+    template<class F, class... Args>
+    auto enqueue(F&& f, Args&&... args) 
+        -> std::future<typename std::result_of<F(Args...)>::type>;
+    ~ThreadPool();
+private:
+    // need to keep track of threads so we can join them
+    std::vector< std::thread > workers;
+    // the task queue
+    std::queue< std::function<void()> > tasks;
+    
+    // synchronization
+    std::mutex queue_mutex;
+    std::condition_variable condition;
+    bool stop;
+};
+ 
+// the constructor just launches some amount of workers
+inline ThreadPool::ThreadPool(size_t threads)
+    :   stop(false)
+{
+    for(size_t i = 0;i<threads;++i)
+        workers.emplace_back(
+            [this]
+            {
+                for(;;)
+                {
+                    std::function<void()> task;
+
+                    {
+                        std::unique_lock<std::mutex> lock(this->queue_mutex);
+                        this->condition.wait(lock,
+                            [this]{ return this->stop || !this->tasks.empty(); });
+                        if(this->stop && this->tasks.empty())
+                            return;
+                        task = std::move(this->tasks.front());
+                        this->tasks.pop();
+                    }
+
+                    task();
+                }
+            }
+        );
+}
+
+// add new work item to the pool
+template<class F, class... Args>
+auto ThreadPool::enqueue(F&& f, Args&&... args) 
+    -> std::future<typename std::result_of<F(Args...)>::type>
+{
+    using return_type = typename std::result_of<F(Args...)>::type;
+
+    auto task = std::make_shared< std::packaged_task<return_type()> >(
+            std::bind(std::forward<F>(f), std::forward<Args>(args)...)
+        );
+        
+    std::future<return_type> res = task->get_future();
+    {
+        std::unique_lock<std::mutex> lock(queue_mutex);
+
+        // don't allow enqueueing after stopping the pool
+        if(stop)
+            throw std::runtime_error("enqueue on stopped ThreadPool");
+
+        tasks.emplace([task](){ (*task)(); });
+    }
+    condition.notify_one();
+    return res;
+}
+
+// the destructor joins all threads
+inline ThreadPool::~ThreadPool()
+{
+    {
+        std::unique_lock<std::mutex> lock(queue_mutex);
+        stop = true;
+    }
+    condition.notify_all();
+    for(std::thread &worker: workers)
+        worker.join();
+}
+
+#endif

tools/dkr_extractor_classes/config.cpp

@@ -37,23 +37,7 @@ std::string get_extension_from_type(std::string type) {
 }
 
 void ExtractConfig::extract_file(ROM& rom, std::string type, std::string folder, std::string filename, std::vector<uint8_t> data) {
-    if(!fs::is_directory(folder)) {
-        fs::create_directories(folder);
-    }
     
-    std::string outFilepath = folder + "/" + filename;
-    
-    if(is_binary_type(type)) {
-        ExtractBinary(data, rom, outFilepath + ".bin");
-    } else if(is_compressed_type(type)) {
-        ExtractCompressed(data, rom, outFilepath + ".cbin");
-    } else if(type == "Textures") {
-        ExtractTextures(data, rom, outFilepath + ".png");
-    } else if(type == "Empty") {
-    } else if(type != "NoExtract") {
-        std::cout << "Unknown extraction type: " << type << std::endl;
-        throw 1;
-    }
 }
 
 uint32_t get_uint_from_table(std::vector<uint8_t>& table, int index) {
@@ -140,8 +124,8 @@ void ExtractConfig::extract(ROM& rom, json::JSON& assetsJson) {
         for(int j = 0; j < numFiles; j++) {
             json::JSON file = files[j];
             int length = std::stoul(file["length"].ToString(), nullptr, 16);
-            extract_file(rom, type, folder, file["filename"].ToString(), 
-                rom.get_bytes_from_range(currentROMOffset, length));
+            extractions.push_back(ExtractInfo(type, folder, file["filename"].ToString(), 
+                rom.get_bytes_from_range(currentROMOffset, length)));
             currentROMOffset += length;
         }
     }
@@ -214,7 +198,7 @@ void ExtractConfig::extract(ROM& rom, json::JSON& assetsJson) {
                     filename = filenameStream.str();
                 }
                 outputAsset["filenames"].append(filename + get_extension_from_type(type));
-                extract_file(rom, type, folder, filename, sectionFiles[j]);
+                extractions.push_back(ExtractInfo(type, folder, filename, sectionFiles[j]));
             }
         } else {
             // No lookup table associated, so the section is assumed to be 1 file.
@@ -228,11 +212,40 @@ void ExtractConfig::extract(ROM& rom, json::JSON& assetsJson) {
                 filename = filenameStream.str();
             }
             outputAsset["filenames"].append(filename + get_extension_from_type(type));
-            extract_file(rom, type, folder, filename, sectionsData[i]);
+            extractions.push_back(ExtractInfo(type, folder, filename, sectionsData[i]));
         }
         currentROMOffset += sectionsData[i].size();
         assetsJson["assets"].append(outputAsset);
     }
+    
+    ThreadPool pool(std::thread::hardware_concurrency());
+    
+    for(int i = 0; i < extractions.size(); i++) {
+        std::string type = extractions[i].type;
+        std::string folder = extractions[i].folder;
+        std::string filename = extractions[i].filename;
+        std::vector<uint8_t> data = extractions[i].data;
+        pool.enqueue([&rom, type, folder, filename, data] {
+            if(!fs::is_directory(folder)) {
+                fs::create_directories(folder);
+            }
+            
+            std::string outFilepath = folder + "/" + filename;
+            
+            if(is_binary_type(type)) {
+                ExtractBinary(data, rom, outFilepath + ".bin");
+            } else if(is_compressed_type(type)) {
+                ExtractCompressed(data, rom, outFilepath + ".cbin");
+            } else if(type == "Textures") {
+                ExtractTextures(data, rom, outFilepath + ".png");
+            } else if(type == "Empty") {
+            } else if(type != "NoExtract") {
+                std::cout << "Unknown extraction type: " << type << std::endl;
+                throw 1;
+            }
+        });
+    }
+    
 }
 
 bool ExtractConfig::is_supported(){

tools/dkr_extractor_classes/config.h

@@ -6,6 +6,7 @@
 #include <fstream>
 #include <sstream>
 #include <regex>
+#include <thread>
 
 // C++17
 #include <experimental/filesystem> 
@@ -17,6 +18,16 @@ namespace fs = std::experimental::filesystem;
 #include "extract_binary.h"
 #include "extract_compressed.h"
 #include "extract_textures.h"
+#include "ThreadPool.h"
+
+struct ExtractInfo {
+    std::string type;
+    std::string folder;
+    std::string filename;
+    std::vector<uint8_t> data;
+    ExtractInfo(std::string type, std::string folder, std::string filename, std::vector<uint8_t> data) 
+        : type(type), folder(folder), filename(filename), data(data) {}
+};
 
 class ExtractConfig {
 public:
@@ -43,5 +54,7 @@ private:
     std::string md5;
     std::string subfolder;
     
+    std::vector<ExtractInfo> extractions;
+    
     std::string read_file(std::string filename);
 };

tools/dkr_extractor_classes/extract.cpp

@@ -1,6 +1,6 @@
 #include "extract.h"
 
-Extract::Extract(std::vector<uint8_t>& data, ROM& rom, std::string outFilepath){
+Extract::Extract(std::vector<uint8_t> data, ROM& rom, std::string outFilepath){
 }
 
 Extract::~Extract(){
@@ -21,7 +21,9 @@ void Extract::write_text_file(std::string text, std::string filepath){
 }
 
 void Extract::print_extracted(std::string outFilepath) {
-    std::cout << "Extracted " << outFilepath << std::endl;
+    std::stringstream out;
+    out << "Extracted " << outFilepath << std::endl;
+    std::cout << out.str();
 }
 
 void Extract::to_lowercase(std::string& input) {

tools/dkr_extractor_classes/extract.h

@@ -17,7 +17,7 @@ namespace fs = std::experimental::filesystem;
 
 class Extract {
 public:
-    Extract(std::vector<uint8_t>& data, ROM& rom, std::string outFilepath);
+    Extract(std::vector<uint8_t> data, ROM& rom, std::string outFilepath);
     ~Extract();
     
     void write_binary_file(std::vector<uint8_t> data, std::string filepath);

tools/dkr_extractor_classes/extract_binary.cpp

@@ -1,6 +1,6 @@
 #include "extract_binary.h"
 
-ExtractBinary::ExtractBinary(std::vector<uint8_t>& data, ROM& rom, std::string outFilepath) 
+ExtractBinary::ExtractBinary(std::vector<uint8_t> data, ROM& rom, std::string outFilepath) 
 : Extract(data, rom, outFilepath) {
     write_binary_file(data, outFilepath);
     print_extracted(outFilepath);

tools/dkr_extractor_classes/extract_binary.h

@@ -4,6 +4,6 @@
 
 class ExtractBinary : Extract {
 public:
-    ExtractBinary(std::vector<uint8_t>& data, ROM& rom, std::string outFilepath);
+    ExtractBinary(std::vector<uint8_t> data, ROM& rom, std::string outFilepath);
     ~ExtractBinary();
 };

tools/dkr_extractor_classes/extract_compressed.cpp

@@ -1,6 +1,6 @@
 #include "extract_compressed.h"
 
-ExtractCompressed::ExtractCompressed(std::vector<uint8_t>& data, ROM& rom, std::string outFilepath) 
+ExtractCompressed::ExtractCompressed(std::vector<uint8_t> data, ROM& rom, std::string outFilepath) 
 : Extract(data, rom, outFilepath) {
     
     if(data.size() == 0) {

tools/dkr_extractor_classes/extract_compressed.h

@@ -5,6 +5,6 @@
 
 class ExtractCompressed : Extract {
 public:
-    ExtractCompressed(std::vector<uint8_t>& data, ROM& rom, std::string outFilepath);
+    ExtractCompressed(std::vector<uint8_t> data, ROM& rom, std::string outFilepath);
     ~ExtractCompressed();
 };

tools/dkr_extractor_classes/extract_textures.cpp

@@ -1,6 +1,6 @@
 #include "extract_textures.h"
 
-ExtractTextures::ExtractTextures(std::vector<uint8_t>& data, ROM& rom, std::string outFilepath) 
+ExtractTextures::ExtractTextures(std::vector<uint8_t> data, ROM& rom, std::string outFilepath) 
 : Extract(data, rom, outFilepath) {
     std::vector<uint8_t> header(data.begin(), data.begin() + TEX_HEADER_SIZE);
     std::vector<uint8_t> texData;

tools/dkr_extractor_classes/extract_textures.h

@@ -17,7 +17,7 @@
 
 class ExtractTextures : Extract {
 public:
-    ExtractTextures(std::vector<uint8_t>& data, ROM& rom, std::string outFilepath);
+    ExtractTextures(std::vector<uint8_t> data, ROM& rom, std::string outFilepath);
     ~ExtractTextures();
     
 private: