From 0afb876332803db08cf845967d1f9b9657f9c1b8 Mon Sep 17 00:00:00 2001 From: David Benepe Date: Tue, 25 Aug 2020 15:57:40 -0500 Subject: [PATCH] Updated dkr_extractor to be multithreaded. --- Makefile | 14 +++ tools/Makefile | 2 +- tools/dkr_extractor_classes/ThreadPool.h | 98 +++++++++++++++++++ tools/dkr_extractor_classes/config.cpp | 53 ++++++---- tools/dkr_extractor_classes/config.h | 13 +++ tools/dkr_extractor_classes/extract.cpp | 6 +- tools/dkr_extractor_classes/extract.h | 2 +- .../dkr_extractor_classes/extract_binary.cpp | 2 +- tools/dkr_extractor_classes/extract_binary.h | 2 +- .../extract_compressed.cpp | 2 +- .../extract_compressed.h | 2 +- .../extract_textures.cpp | 2 +- .../dkr_extractor_classes/extract_textures.h | 2 +- 13 files changed, 170 insertions(+), 30 deletions(-) create mode 100644 tools/dkr_extractor_classes/ThreadPool.h diff --git a/Makefile b/Makefile index 3564681a..d7f0ca6d 100755 --- a/Makefile +++ b/Makefile @@ -15,6 +15,7 @@ VERSION := us_1.0 ifneq ($(MAKECMDGOALS),clean) ifneq ($(MAKECMDGOALS),clean_lib) ifneq ($(MAKECMDGOALS),clean_src) +ifneq ($(MAKECMDGOALS),reset) ########## QEMU_IRIX ########### @@ -91,6 +92,7 @@ endif endif endif endif +endif ################ Target Executable and Sources ############### @@ -302,6 +304,18 @@ LD_SCRIPT = $(TARGET).ld all: $(BUILD_DIR)/$(TARGET).z64 +reset: +ifneq ($(wildcard ./build/.*),) + rm -r build +endif +ifneq ($(wildcard ./assets/.*),) + rm -r assets +endif +ifneq ($(wildcard ./ucode/.*),) + rm -r ucode +endif + @echo "Done." + clean: ifneq ($(wildcard ./build/.*),) rm -r build diff --git a/tools/Makefile b/tools/Makefile index 7e3a9a2b..13e9d826 100755 --- a/tools/Makefile +++ b/tools/Makefile @@ -15,7 +15,7 @@ dkr_decompressor_SOURCES := dkr_decompressor.cpp $(wildcard dkr_decompressor_src dkr_decompressor_CXXFLAGS := -lstdc++fs -lcrypto -lssl dkr_extractor_SOURCES := dkr_extractor.cpp $(wildcard dkr_decompressor_src/*.cpp dkr_decompressor_src/*.c dkr_extractor_classes/*.cpp n64graphics/*.c) -dkr_extractor_CXXFLAGS := -lstdc++fs -lcrypto -lssl +dkr_extractor_CXXFLAGS := -lstdc++fs -lcrypto -lssl -lpthread dkr_texbuilder_SOURCES := dkr_texbuilder.cpp $(wildcard dkr_decompressor_src/*.cpp dkr_decompressor_src/*.c n64graphics/*.c) dkr_texbuilder_CXXFLAGS := -lstdc++fs diff --git a/tools/dkr_extractor_classes/ThreadPool.h b/tools/dkr_extractor_classes/ThreadPool.h new file mode 100644 index 00000000..41832030 --- /dev/null +++ b/tools/dkr_extractor_classes/ThreadPool.h @@ -0,0 +1,98 @@ +#ifndef THREAD_POOL_H +#define THREAD_POOL_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +class ThreadPool { +public: + ThreadPool(size_t); + template + auto enqueue(F&& f, Args&&... args) + -> std::future::type>; + ~ThreadPool(); +private: + // need to keep track of threads so we can join them + std::vector< std::thread > workers; + // the task queue + std::queue< std::function > tasks; + + // synchronization + std::mutex queue_mutex; + std::condition_variable condition; + bool stop; +}; + +// the constructor just launches some amount of workers +inline ThreadPool::ThreadPool(size_t threads) + : stop(false) +{ + for(size_t i = 0;i task; + + { + std::unique_lock lock(this->queue_mutex); + this->condition.wait(lock, + [this]{ return this->stop || !this->tasks.empty(); }); + if(this->stop && this->tasks.empty()) + return; + task = std::move(this->tasks.front()); + this->tasks.pop(); + } + + task(); + } + } + ); +} + +// add new work item to the pool +template +auto ThreadPool::enqueue(F&& f, Args&&... args) + -> std::future::type> +{ + using return_type = typename std::result_of::type; + + auto task = std::make_shared< std::packaged_task >( + std::bind(std::forward(f), std::forward(args)...) + ); + + std::future res = task->get_future(); + { + std::unique_lock lock(queue_mutex); + + // don't allow enqueueing after stopping the pool + if(stop) + throw std::runtime_error("enqueue on stopped ThreadPool"); + + tasks.emplace([task](){ (*task)(); }); + } + condition.notify_one(); + return res; +} + +// the destructor joins all threads +inline ThreadPool::~ThreadPool() +{ + { + std::unique_lock lock(queue_mutex); + stop = true; + } + condition.notify_all(); + for(std::thread &worker: workers) + worker.join(); +} + +#endif diff --git a/tools/dkr_extractor_classes/config.cpp b/tools/dkr_extractor_classes/config.cpp index f8da2b5e..a403005a 100644 --- a/tools/dkr_extractor_classes/config.cpp +++ b/tools/dkr_extractor_classes/config.cpp @@ -37,23 +37,7 @@ std::string get_extension_from_type(std::string type) { } void ExtractConfig::extract_file(ROM& rom, std::string type, std::string folder, std::string filename, std::vector data) { - if(!fs::is_directory(folder)) { - fs::create_directories(folder); - } - std::string outFilepath = folder + "/" + filename; - - if(is_binary_type(type)) { - ExtractBinary(data, rom, outFilepath + ".bin"); - } else if(is_compressed_type(type)) { - ExtractCompressed(data, rom, outFilepath + ".cbin"); - } else if(type == "Textures") { - ExtractTextures(data, rom, outFilepath + ".png"); - } else if(type == "Empty") { - } else if(type != "NoExtract") { - std::cout << "Unknown extraction type: " << type << std::endl; - throw 1; - } } uint32_t get_uint_from_table(std::vector& table, int index) { @@ -140,8 +124,8 @@ void ExtractConfig::extract(ROM& rom, json::JSON& assetsJson) { for(int j = 0; j < numFiles; j++) { json::JSON file = files[j]; int length = std::stoul(file["length"].ToString(), nullptr, 16); - extract_file(rom, type, folder, file["filename"].ToString(), - rom.get_bytes_from_range(currentROMOffset, length)); + extractions.push_back(ExtractInfo(type, folder, file["filename"].ToString(), + rom.get_bytes_from_range(currentROMOffset, length))); currentROMOffset += length; } } @@ -214,7 +198,7 @@ void ExtractConfig::extract(ROM& rom, json::JSON& assetsJson) { filename = filenameStream.str(); } outputAsset["filenames"].append(filename + get_extension_from_type(type)); - extract_file(rom, type, folder, filename, sectionFiles[j]); + extractions.push_back(ExtractInfo(type, folder, filename, sectionFiles[j])); } } else { // No lookup table associated, so the section is assumed to be 1 file. @@ -228,11 +212,40 @@ void ExtractConfig::extract(ROM& rom, json::JSON& assetsJson) { filename = filenameStream.str(); } outputAsset["filenames"].append(filename + get_extension_from_type(type)); - extract_file(rom, type, folder, filename, sectionsData[i]); + extractions.push_back(ExtractInfo(type, folder, filename, sectionsData[i])); } currentROMOffset += sectionsData[i].size(); assetsJson["assets"].append(outputAsset); } + + ThreadPool pool(std::thread::hardware_concurrency()); + + for(int i = 0; i < extractions.size(); i++) { + std::string type = extractions[i].type; + std::string folder = extractions[i].folder; + std::string filename = extractions[i].filename; + std::vector data = extractions[i].data; + pool.enqueue([&rom, type, folder, filename, data] { + if(!fs::is_directory(folder)) { + fs::create_directories(folder); + } + + std::string outFilepath = folder + "/" + filename; + + if(is_binary_type(type)) { + ExtractBinary(data, rom, outFilepath + ".bin"); + } else if(is_compressed_type(type)) { + ExtractCompressed(data, rom, outFilepath + ".cbin"); + } else if(type == "Textures") { + ExtractTextures(data, rom, outFilepath + ".png"); + } else if(type == "Empty") { + } else if(type != "NoExtract") { + std::cout << "Unknown extraction type: " << type << std::endl; + throw 1; + } + }); + } + } bool ExtractConfig::is_supported(){ diff --git a/tools/dkr_extractor_classes/config.h b/tools/dkr_extractor_classes/config.h index c8ce8768..c30df4e5 100644 --- a/tools/dkr_extractor_classes/config.h +++ b/tools/dkr_extractor_classes/config.h @@ -6,6 +6,7 @@ #include #include #include +#include // C++17 #include @@ -17,6 +18,16 @@ namespace fs = std::experimental::filesystem; #include "extract_binary.h" #include "extract_compressed.h" #include "extract_textures.h" +#include "ThreadPool.h" + +struct ExtractInfo { + std::string type; + std::string folder; + std::string filename; + std::vector data; + ExtractInfo(std::string type, std::string folder, std::string filename, std::vector data) + : type(type), folder(folder), filename(filename), data(data) {} +}; class ExtractConfig { public: @@ -43,5 +54,7 @@ private: std::string md5; std::string subfolder; + std::vector extractions; + std::string read_file(std::string filename); }; diff --git a/tools/dkr_extractor_classes/extract.cpp b/tools/dkr_extractor_classes/extract.cpp index 7c4e8fca..7b647214 100644 --- a/tools/dkr_extractor_classes/extract.cpp +++ b/tools/dkr_extractor_classes/extract.cpp @@ -1,6 +1,6 @@ #include "extract.h" -Extract::Extract(std::vector& data, ROM& rom, std::string outFilepath){ +Extract::Extract(std::vector data, ROM& rom, std::string outFilepath){ } Extract::~Extract(){ @@ -21,7 +21,9 @@ void Extract::write_text_file(std::string text, std::string filepath){ } void Extract::print_extracted(std::string outFilepath) { - std::cout << "Extracted " << outFilepath << std::endl; + std::stringstream out; + out << "Extracted " << outFilepath << std::endl; + std::cout << out.str(); } void Extract::to_lowercase(std::string& input) { diff --git a/tools/dkr_extractor_classes/extract.h b/tools/dkr_extractor_classes/extract.h index b558b13a..021728e9 100644 --- a/tools/dkr_extractor_classes/extract.h +++ b/tools/dkr_extractor_classes/extract.h @@ -17,7 +17,7 @@ namespace fs = std::experimental::filesystem; class Extract { public: - Extract(std::vector& data, ROM& rom, std::string outFilepath); + Extract(std::vector data, ROM& rom, std::string outFilepath); ~Extract(); void write_binary_file(std::vector data, std::string filepath); diff --git a/tools/dkr_extractor_classes/extract_binary.cpp b/tools/dkr_extractor_classes/extract_binary.cpp index 1efd4a09..3e3a2fa1 100644 --- a/tools/dkr_extractor_classes/extract_binary.cpp +++ b/tools/dkr_extractor_classes/extract_binary.cpp @@ -1,6 +1,6 @@ #include "extract_binary.h" -ExtractBinary::ExtractBinary(std::vector& data, ROM& rom, std::string outFilepath) +ExtractBinary::ExtractBinary(std::vector data, ROM& rom, std::string outFilepath) : Extract(data, rom, outFilepath) { write_binary_file(data, outFilepath); print_extracted(outFilepath); diff --git a/tools/dkr_extractor_classes/extract_binary.h b/tools/dkr_extractor_classes/extract_binary.h index 61eff74c..8e2dba0a 100644 --- a/tools/dkr_extractor_classes/extract_binary.h +++ b/tools/dkr_extractor_classes/extract_binary.h @@ -4,6 +4,6 @@ class ExtractBinary : Extract { public: - ExtractBinary(std::vector& data, ROM& rom, std::string outFilepath); + ExtractBinary(std::vector data, ROM& rom, std::string outFilepath); ~ExtractBinary(); }; diff --git a/tools/dkr_extractor_classes/extract_compressed.cpp b/tools/dkr_extractor_classes/extract_compressed.cpp index 861b45b3..d405af40 100644 --- a/tools/dkr_extractor_classes/extract_compressed.cpp +++ b/tools/dkr_extractor_classes/extract_compressed.cpp @@ -1,6 +1,6 @@ #include "extract_compressed.h" -ExtractCompressed::ExtractCompressed(std::vector& data, ROM& rom, std::string outFilepath) +ExtractCompressed::ExtractCompressed(std::vector data, ROM& rom, std::string outFilepath) : Extract(data, rom, outFilepath) { if(data.size() == 0) { diff --git a/tools/dkr_extractor_classes/extract_compressed.h b/tools/dkr_extractor_classes/extract_compressed.h index 850a6c0a..54a8818d 100644 --- a/tools/dkr_extractor_classes/extract_compressed.h +++ b/tools/dkr_extractor_classes/extract_compressed.h @@ -5,6 +5,6 @@ class ExtractCompressed : Extract { public: - ExtractCompressed(std::vector& data, ROM& rom, std::string outFilepath); + ExtractCompressed(std::vector data, ROM& rom, std::string outFilepath); ~ExtractCompressed(); }; diff --git a/tools/dkr_extractor_classes/extract_textures.cpp b/tools/dkr_extractor_classes/extract_textures.cpp index cd8bad5a..1d53df89 100644 --- a/tools/dkr_extractor_classes/extract_textures.cpp +++ b/tools/dkr_extractor_classes/extract_textures.cpp @@ -1,6 +1,6 @@ #include "extract_textures.h" -ExtractTextures::ExtractTextures(std::vector& data, ROM& rom, std::string outFilepath) +ExtractTextures::ExtractTextures(std::vector data, ROM& rom, std::string outFilepath) : Extract(data, rom, outFilepath) { std::vector header(data.begin(), data.begin() + TEX_HEADER_SIZE); std::vector texData; diff --git a/tools/dkr_extractor_classes/extract_textures.h b/tools/dkr_extractor_classes/extract_textures.h index 8a0a8d44..4cd3b6a1 100644 --- a/tools/dkr_extractor_classes/extract_textures.h +++ b/tools/dkr_extractor_classes/extract_textures.h @@ -17,7 +17,7 @@ class ExtractTextures : Extract { public: - ExtractTextures(std::vector& data, ROM& rom, std::string outFilepath); + ExtractTextures(std::vector data, ROM& rom, std::string outFilepath); ~ExtractTextures(); private: