Improvements for the compiler.

Makefile

@@ -86,7 +86,11 @@ RECOMP_PROJECT := ./tools/ido-static-recomp/
 
 DUMMY != ls $(RECOMP_PROJECT)ido >/dev/null || echo FAIL
 ifeq ($(DUMMY),FAIL)
-  $(error Missing submodule ido-static-recomp. Please run 'git submodule update --init')
+DUMMY != git submodule update --init
+DUMMY != ls $(RECOMP_PROJECT)ido >/dev/null || echo FAIL
+ifeq ($(DUMMY),FAIL)
+$(error Missing submodule ido-static-recomp. Please run 'git submodule update --init')
+endif
 endif
 
 # List of IDO tools required for the repo.

doc/dkr_asset_spec.md

@@ -63,6 +63,34 @@
 
 ----
 
+## meta.json
+
+Every package should have a meta.json file in the root directory that describes the contents of the package.
+
+| Property Name | Type | Required | Comments |
+| -------- | -------- | -------- | -------- |
+| name | String | Yes | The name of the package. |
+| authors | Array | Yes | The list of authors for the package. |
+| pkg-version | String | Yes | The version text for the package. |
+| revision | Integer | Yes | The revision number for this package. Should match the revision number of dkr\_assets\_tool. |
+| revision-minor | Integer | Yes | The minor revision number for this package. Should match the minor revision number of dkr\_assets\_tool. |
+| description | String | No | Describes what the package does in detail. |
+| description-brief | String | No | A short form of the description. There is no maximum length, but try to keep it under 80 characters. |
+| thumbnail | String | No | A path to a PNG image or a base64 image uri. If you are using a path, put the thumbnail image in a folder called "meta" in the root directory. |
+
+----
+
+## Package Commands
+
+Every single JSON file has an option to tell the asset compiler how to write the JSON file. These commands start with the string "pkg-".
+
+| Property Name | Type | Required | Comments |
+| -------- | -------- | -------- | -------- | 
+| pkg-write | String | No | Either "replace" or "merge". "replace" will overwrite the existing JSON file if it exists. "merge" will combine the contents of the existing JSON file with the new one. Set to "replace" by default. |
+| pkg-rebuild | String | No | Either "onchange" or "always". "onchange" will only copy the final output if the md5 hash is different. "always" will copy the final output even if the md5 hash is the same. Set to "onchange" by default.
+
+----
+
 ## Fonts
 
 | Property Name | Type | Required | Comments |
@@ -93,7 +121,8 @@
 | special-character-width | Integer | Yes | Width for every ASCII character value smaller or equal to 0x20 (including space and excluding the tab character, which uses `tab-width`). |
 | tab-width | Integer | Yes | Width of the tab character `\t` |
 | textures | Array | Yes | An array of Texture IDs to use for this font. DKR has a maximum of 32 textures; future rare games extended this to 64. |
-| unknown-text | String | Yes | Leftover debug text from development? |
+| unknown-text | String | No | Leftover debug text from development? |
+| upper-case-only | Boolean | No | If set to true, then all lowercase nodes ("a", "b", "c", etc.) will only use their uppercase values. This is useful for custom fonts with only a single case in it. Set to `false` by default.  |
 | y-offset | Integer | Yes | Offsets the y position for each character in the font. (Might just be line height?) |
 
 | `character` Property Name | Type | Required | Comments |

tools/dkr_assets_tool_classes/builder/build_files/build_fonts.cpp

@@ -51,12 +51,18 @@ BuildFonts::BuildFonts(std::string srcPath, std::string dstPath) {
         
         std::string encoding = get_string_from_json(fontPath, "encoding", "type");
         make_uppercase(encoding);
+
+        bool uppercaseOnly = false;
+
+        if(json_has_key(fontPath, "uppercase-only")) {
+            uppercaseOnly = get_bool_from_json(fontPath, "uppercase-only");
+        }
         
         // Write character nodes
         if(encoding == "ASCII") {
-            write_ascii_char_nodes(out, fontPath, fontOffset + 0x100);
+            write_ascii_char_nodes(out, fontPath, fontOffset + 0x100, uppercaseOnly);
         } else if(encoding == "CUSTOM") {
-            write_custom_char_nodes(out, fontPath, fontOffset + 0x100);
+            write_custom_char_nodes(out, fontPath, fontOffset + 0x100, uppercaseOnly);
         } else {
             display_error_and_abort("Unsupported font encoding type: \"", encoding, "\"");
         }
@@ -71,17 +77,28 @@ BuildFonts::~BuildFonts() {
 }
 
 void write_char_node(std::vector<uint8_t> &out, std::string fontPath, int nodeOffset, std::string &chr) {
-    out[nodeOffset]     = get_int_from_json(fontPath, "characters", chr, "tex-index");
-    out[nodeOffset + 1] = get_int_from_json(fontPath, "characters", chr, "char-width");
-    out[nodeOffset + 2] = get_int_from_json(fontPath, "characters", chr, "offset", 0);   // X offset
-    out[nodeOffset + 3] = get_int_from_json(fontPath, "characters", chr, "offset", 1);   // Y offset
-    out[nodeOffset + 4] = get_int_from_json(fontPath, "characters", chr, "uv", 0);       // U coordinate
-    out[nodeOffset + 5] = get_int_from_json(fontPath, "characters", chr, "uv", 1);       // V coordinate
-    out[nodeOffset + 6] = get_int_from_json(fontPath, "characters", chr, "tex-size", 0); // Width
-    out[nodeOffset + 7] = get_int_from_json(fontPath, "characters", chr, "tex-size", 1); // Height
+    if(json_has_key(fontPath, "characters", chr)) {
+        out[nodeOffset]     = get_int_from_json(fontPath, "characters", chr, "tex-index");
+        out[nodeOffset + 1] = get_int_from_json(fontPath, "characters", chr, "char-width");
+        out[nodeOffset + 2] = get_int_from_json(fontPath, "characters", chr, "offset", 0);   // X offset
+        out[nodeOffset + 3] = get_int_from_json(fontPath, "characters", chr, "offset", 1);   // Y offset
+        out[nodeOffset + 4] = get_int_from_json(fontPath, "characters", chr, "uv", 0);       // U coordinate
+        out[nodeOffset + 5] = get_int_from_json(fontPath, "characters", chr, "uv", 1);       // V coordinate
+        out[nodeOffset + 6] = get_int_from_json(fontPath, "characters", chr, "tex-size", 0); // Width
+        out[nodeOffset + 7] = get_int_from_json(fontPath, "characters", chr, "tex-size", 1); // Height
+    } else { // Node doesn't exist, so just mark it as empty.
+        out[nodeOffset]     = -1;
+        out[nodeOffset + 1] =  0;
+        out[nodeOffset + 2] =  0;
+        out[nodeOffset + 3] =  0;
+        out[nodeOffset + 4] =  0;
+        out[nodeOffset + 5] =  0;
+        out[nodeOffset + 6] =  0;
+        out[nodeOffset + 7] =  0;
+    }
 }
 
-void BuildFonts::write_ascii_char_nodes(std::vector<uint8_t> &out, std::string fontPath, int offset) {
+void BuildFonts::write_ascii_char_nodes(std::vector<uint8_t> &out, std::string fontPath, int offset, bool uppercaseOnly) {
     for(int node = CHAR_NODE_START_OFFSET; node < CHAR_NODE_END_OFFSET; node++) {
         std::string chr;
         if(node == 127) {
@@ -94,11 +111,14 @@ void BuildFonts::write_ascii_char_nodes(std::vector<uint8_t> &out, std::string f
             chr = (char)node;
         }
         int nodeOffset = offset + (node - CHAR_NODE_START_OFFSET) * SIZEOF_CHAR_NODE;
+        if(uppercaseOnly && node >= (int)'a' && node <= (int)'z') {
+            make_uppercase(chr);
+        }
         write_char_node(out, fontPath, nodeOffset, chr);
     }
 }
 
-void BuildFonts::write_custom_char_nodes(std::vector<uint8_t> &out, std::string fontPath, int offset) {
+void BuildFonts::write_custom_char_nodes(std::vector<uint8_t> &out, std::string fontPath, int offset, bool uppercaseOnly) {
     int index = 0;
     std::vector<std::string> order = get_array_from_json(fontPath, "encoding", "order");
     for(std::string &chr : order) {

tools/dkr_assets_tool_classes/builder/build_files/build_fonts.h

@@ -8,6 +8,6 @@ public:
     BuildFonts(std::string srcPath, std::string dstPath);
     ~BuildFonts();
 private:
-    void write_ascii_char_nodes(std::vector<uint8_t> &out, std::string srcPath, int offset);
-    void write_custom_char_nodes(std::vector<uint8_t> &out, std::string srcPath, int offset);
+    void write_ascii_char_nodes(std::vector<uint8_t> &out, std::string srcPath, int offset, bool uppercaseOnly);
+    void write_custom_char_nodes(std::vector<uint8_t> &out, std::string srcPath, int offset, bool uppercaseOnly);
 };

tools/dkr_assets_tool_classes/common/revision_number.h

@@ -0,0 +1,5 @@
+#pragma once
+
+#define REVISION_NUMBER 2
+#define REVISION_MINOR_NUMBER 0
+

tools/dkr_assets_tool_classes/common/util/errorHelper.h

@@ -3,8 +3,14 @@
 #include <string>
 #include <iostream>
 
-// Requires C++17
+// Terminal colors
 #define RED_TEXT "\033[1;31m"
+#define GREEN_TEXT "\033[1;32m"
+#define YELLOW_TEXT "\033[1;33m"
+#define BLUE_TEXT "\033[1;34m"
+#define MAGENTA_TEXT "\033[1;35m"
+#define CYAN_TEXT "\033[1;36m"
+
 #define RESET_TEXT "\033[0m"
 
 // Requires C++17
@@ -23,4 +29,3 @@ inline void display_error_and_abort(Args... args) {
     throw 1;
 }
 
-

tools/dkr_assets_tool_classes/common/util/fileHelper.cpp

@@ -68,6 +68,12 @@ void path_must_exist(std::string path) {
     }
 }
 
+bool path_is_directory(std::string strPath) {
+    fs::path path(strPath);
+    std::error_code ec;
+    return fs::is_directory(path, ec);
+}
+
 std::vector<std::string> get_filenames_from_directory(std::string dirPath) {
     std::vector<std::string> out;
     for (const auto &file : fs::recursive_directory_iterator(dirPath)) {
@@ -117,6 +123,17 @@ std::vector<std::string> get_filenames_from_directory_with_extension(std::string
     return out;
 }
 
+std::vector<std::string> get_filenames_from_directory_without_extension(std::string dirPath, std::string extension) {
+    std::vector<std::string> out;
+    for (const auto &file : fs::recursive_directory_iterator(dirPath)) {
+        std::string path = file.path();
+        if(!ends_with(path, extension)) {
+            out.push_back(path);
+        }
+    }
+    return out;
+}
+
 void create_directory(std::string directory) {
     if(!fs::is_directory(directory)) {
         fs::create_directories(directory);
@@ -148,6 +165,11 @@ void ensure_that_path_exists(std::string path) {
     create_directory(get_directory_from_full_path(path));
 }
 
+
+void copy_file(std::string srcPath, std::string dstPath) {
+    fs::copy(srcPath, dstPath, fs::copy_options::overwrite_existing);
+}
+
 void recursively_copy_directory(std::string srcPath, std::string dstPath) {
     fs::copy(srcPath, dstPath, fs::copy_options::recursive | fs::copy_options::overwrite_existing);
 }
@@ -158,10 +180,24 @@ void delete_directory(std::string dirPath) {
     }
 }
 
+void delete_path(std::string path) {
+    if(path_exists(path)) {
+        if(path_is_directory(path)) {
+            fs::remove_all(path);
+        } else {
+            fs::remove(path);
+        }
+    }
+}
+
 std::string replace_extension(std::string path, std::string newExtension) {
     return path.substr(0, path.rfind(".")) + newExtension;
 }
 
+bool starts_with(const std::string &a, const std::string &b) {
+    return a.rfind(b, 0) == 0;
+}
+
 bool ends_with(const std::string &a, const std::string &b) {
     if (b.size() > a.size()) return false;
     return std::equal(a.begin() + a.size() - b.size(), a.end(), b.begin());
@@ -179,3 +215,36 @@ void make_uppercase(std::string& input) {
     }
 }
 
+std::string calculate_md5(std::vector<uint8_t> &bytes) {
+    uint8_t buffer[0x4000];
+    uint8_t digest[MD5_DIGEST_LENGTH];
+
+    std::stringstream ss;
+
+    MD5_CTX md5Context;
+
+    MD5_Init(&md5Context);
+    MD5_Update(&md5Context, &bytes[0], bytes.size()); 
+    int res = MD5_Final(digest, &md5Context);
+
+    if(res == 0) {
+        return "";
+    }
+
+    // set up stringstream format
+    ss << std::hex << std::setfill('0');
+
+
+    for(uint8_t uc: digest) {
+        ss << std::setw(2) << (int)uc;
+    }
+
+    return ss.str();
+}
+
+std::string calculate_file_md5(std::string filepath) {
+    std::vector<uint8_t> bytes = read_binary_file(filepath);
+    return calculate_md5(bytes);
+}
+
+

tools/dkr_assets_tool_classes/common/util/fileHelper.h

@@ -7,6 +7,7 @@
 #include <algorithm>
 #include <iterator>
 #include <iostream>
+#include <openssl/md5.h>
 
 // C++17
 #include <filesystem> 
@@ -26,19 +27,27 @@ std::string read_text_file(std::string filename);
 void write_text_file(std::string text, std::string filepath);
 bool path_exists(std::string path);
 void path_must_exist(std::string path);
+bool path_is_directory(std::string strPath);
 std::vector<std::string> get_filenames_from_directory(std::string dirPath);
 std::vector<std::string> get_filenames_from_directory_only(std::string dirPath);
 std::vector<fs::path> get_filenames_from_directory_only_with_extension(std::string dirPath, std::string extension);
 std::vector<fs::path> get_folders_from_directory_only(std::string dirPath);
 std::vector<std::string> get_filenames_from_directory_with_extension(std::string dirPath, std::string extension);
+std::vector<std::string> get_filenames_from_directory_without_extension(std::string dirPath, std::string extension);
 void create_directory(std::string directory);
 std::string get_filename_from_full_path(std::string path, bool excludeExtension);
 std::string get_directory_from_full_path(std::string path);
 void ensure_that_path_exists(std::string path);
+void copy_file(std::string srcPath, std::string dstPath);
 void recursively_copy_directory(std::string srcPath, std::string dstPath);
 void delete_directory(std::string dirPath);
+void delete_path(std::string dirPath);
 std::string replace_extension(std::string path, std::string newExtension);
+bool starts_with(const std::string &a, const std::string &b);
 bool ends_with(const std::string &a, const std::string &b);
 void make_lowercase(std::string &input);
 void make_uppercase(std::string &input);
 
+std::string calculate_md5(std::vector<uint8_t> &bytes);
+std::string calculate_file_md5(std::string filepath);
+

tools/dkr_assets_tool_classes/common/util/jsonHelper.cpp

@@ -171,6 +171,8 @@ void recursively_combine_json(json::JSON &mainJson, json::JSON &addJson) {
 }
 
 void combine_json_files(std::string src1Path, std::string src2Path, std::string outputPath) {
+    path_must_exist(src1Path);
+    path_must_exist(src2Path);
     json::JSON mainJson = json::JSON::Load(read_text_file(src1Path));
     json::JSON addJson = json::JSON::Load(read_text_file(src2Path));
     recursively_combine_json(mainJson, addJson);

tools/dkr_assets_tool_classes/compiler/compiler.cpp

@@ -3,11 +3,24 @@
 // Needs to be defined here.
 #include "../libs/zip_file.hpp"
 
+
+const std::string JSON_WRITE_MODE_REPLACE = "replace";
+const std::string JSON_WRITE_MODE_MERGE = "merge";
+const std::string DEFAULT_JSON_WRITE_MODE = JSON_WRITE_MODE_REPLACE;
+
 const std::string BASE_DIR_NAME = "vanilla";
 const std::string EXT_DIR_NAME = "packages";
 
+void print_text(std::string text) {
+    std::cout << text << std::endl;
+}
+
+void print_coloured_text(std::string color, std::string text) {
+    std::cout << color << text << RESET_TEXT << std::endl;
+}
+
 AssetCompiler::AssetCompiler() {
-    std::cout << "Compiling assets..." << std::endl;
+    print_text("Compiling assets...");
 
     std::string assetsDirectory = get_asset_folder_path();
     std::string version = get_version();
@@ -15,38 +28,86 @@ AssetCompiler::AssetCompiler() {
     std::string vanillaDirPath = assetsDirectory + "/" + BASE_DIR_NAME;
     std::string outDirPath = assetsDirectory + "/" + version;
     std::string packagesDirPath = assetsDirectory + "/" + EXT_DIR_NAME + "/" + version;
-    
-    // Deletes the output directory if it currently exists.
-    std::cout << "Deleting \"" << outDirPath << "\"..." << std::endl;
-    delete_directory(outDirPath); 
+    std::string tempDirPath = assetsDirectory + "/_temp";
 
-    // Copy vanilla folder to the output directory as the baseline.
-    std::cout << "Copying vanilla files..." << std::endl;
-    recursively_copy_directory(vanillaDirPath + "/" + version, outDirPath);
+    try {
+        // 1.) Create temporary directory for working files.
+        create_directory(tempDirPath);
 
-    std::cout << "Reading package order..." << std::endl;
-    std::string packageOrderFilepath = packagesDirPath + "/order.json";
-    if(path_exists(packageOrderFilepath)) {
-        std::vector<std::string> packageOrder = get_array_from_json(packageOrderFilepath);
-        for(std::string &packageName : packageOrder) {
-            std::cout << "Applying package: " << packageName << std::endl;
-            apply_package(packagesDirPath, packageName, outDirPath);
+        // 2.) Create "compiled" directory for output files
+        std::string compiledDir = tempDirPath + "/__compiled__";
+
+        // Copy vanilla folder to the compiled directory as the baseline.
+        print_text("Copying vanilla files...");
+        recursively_copy_directory(vanillaDirPath + "/" + version, compiledDir);
+
+        print_text("Reading package order...");
+        std::string packageOrderFilepath = packagesDirPath + "/order.json";
+
+        bool isVanilla = true;
+
+        if(path_exists(packageOrderFilepath)) {
+            std::vector<std::string> packageOrder = get_array_from_json(packageOrderFilepath);
+            for(std::string &packageName : packageOrder) {
+                print_text("Applying package \"" + packageName + "\"...");
+                if(packageName.length() == 0) {
+                    display_error_and_abort("The package \"\" does not exist!");
+                }
+                apply_package(packagesDirPath, packageName, tempDirPath, compiledDir);
+                isVanilla = false;
+            }
         }
-    }
-    
-    // Remove the temp directory if it exists.
-    if(path_exists(packagesDirPath + "/_temp")) {
-        delete_directory(packagesDirPath + "/_temp");
+
+        if(isVanilla) {
+            print_coloured_text(CYAN_TEXT, "Assets are vanilla.");
+        } else {
+            print_coloured_text(CYAN_TEXT, "Assets are modified.");
+        }
+
+        print_text("Merging modified files...");
+        merge_output(compiledDir, outDirPath);
+        
+        // Remove the temp directory
+        print_text("Cleaning up...");
+        delete_directory(tempDirPath);
+    } catch (int e) {
+        //delete_directory(tempDirPath);
+        throw e;
     }
 
-    std::cout << "Done!" << std::endl;
+    print_text("Done!");
 }
 
 AssetCompiler::~AssetCompiler() {
 
 }
 
-void AssetCompiler::apply_package(std::string &packagesDirPath, std::string &packageName, std::string &outDirPath) {
+void ensure_meta_json_has_key(std::string &packageName, std::string &metaJsonFilepath, std::string key) {
+    if(!json_has_key(metaJsonFilepath, key)) {
+        display_error_and_abort("\"", key, "\" not defined in ", packageName, "'s meta.json file!");
+    }
+}
+
+void AssetCompiler::check_meta_json(std::string &packageName, std::string metaJsonFilepath) {
+    if(!path_exists(metaJsonFilepath)) {
+        display_error_and_abort("The package \"", packageName, "\" is missing a meta.json file!");
+    }
+
+    ensure_meta_json_has_key(packageName, metaJsonFilepath, "name");
+    ensure_meta_json_has_key(packageName, metaJsonFilepath, "pkg-version");
+    ensure_meta_json_has_key(packageName, metaJsonFilepath, "authors");
+    ensure_meta_json_has_key(packageName, metaJsonFilepath, "revision");
+    ensure_meta_json_has_key(packageName, metaJsonFilepath, "revision-minor");
+
+    int revision = get_int_from_json(metaJsonFilepath, "revision");
+    int revisionMinor = get_int_from_json(metaJsonFilepath, "revision-minor");
+
+    if(revision != REVISION_NUMBER || revisionMinor != REVISION_MINOR_NUMBER) {
+        display_error_and_abort("The package \"", packageName, "\" is not supported. It's revision number is ", revision, ".", revisionMinor, ", when it should be ", REVISION_NUMBER, ".", REVISION_MINOR_NUMBER, "!");
+    }
+}
+
+void AssetCompiler::apply_package(std::string &packagesDirPath, std::string &packageName, std::string &tempDirPath, std::string &outDirPath) {
     std::string packageDir = packagesDirPath + "/" + packageName;
 
     bool isZip = false;
@@ -57,59 +118,137 @@ void AssetCompiler::apply_package(std::string &packagesDirPath, std::string &pac
     } else if(!path_exists(packageDir) && path_exists(packageDir + ".zip")) {
         packageDir += ".zip";
         isZip = true;
+    } else if(!path_exists(packageDir)) {
+        display_error_and_abort("The package \"", packageName, "\" does not exist!");
     }
 
-    // Extract zip to a temp directory and use that directory.
     if(isZip) {
-        std::string tempDirPath = packagesDirPath + "/_temp";
-        create_directory(tempDirPath);
         miniz_cpp::zip_file file(packageDir);
         packageDir = tempDirPath + "/" + packageName;
         // Make sure a subdirectory with the name of the package exists in the temp directory.
-        if(!file.has_file(packageName + "/")) {
+        if(file.has_file("meta.json")) {
+            std::cout << "A" << std::endl;
             create_directory(packageDir);
             file.extractall(packageDir);
         } else {
+            std::cout << "B" << std::endl;
             file.extractall(tempDirPath);
         }
     }
+    
+    check_meta_json(packageName, packageDir + "/meta.json");
 
-    copy_stuff_from_folders(packageDir, outDirPath);
-    append_manifest_files(packageDir, outDirPath);
-}
+    std::vector<std::string> jsonFiles = get_filenames_from_directory_with_extension(packageDir, ".json");
+    std::vector<std::string> otherFiles = get_filenames_from_directory_without_extension(packageDir, ".json");
+    for(int i = 0; i < jsonFiles.size(); i++) {
+        std::string localPath = jsonFiles[i].substr(packageDir.length() + 1);
 
-void AssetCompiler::copy_stuff_from_folders(std::string &packageDir, std::string &outDirPath) {
-    std::vector<fs::path> packageFolders = get_folders_from_directory_only(packageDir);
-    for(int i = 0; i < packageFolders.size(); i++) {
-        std::string folderPath = packageFolders[i].string();
-        verify_json_files(folderPath);
-        recursively_copy_directory(folderPath, outDirPath + "/" + packageFolders[i].filename().string());
+        if(localPath == "meta.json") {
+            continue; // Skip meta.json file.
+        }
+
+        std::string outPath = outDirPath + "/" + localPath;
+
+        std::string write_mode = get_string_from_json(jsonFiles[i], "pkg-write");
+        make_lowercase(write_mode);
+        if(write_mode == "") {
+            write_mode = DEFAULT_JSON_WRITE_MODE;
+        }
+
+        if(write_mode == JSON_WRITE_MODE_REPLACE) {
+            copy_file(jsonFiles[i], outPath);
+        } else if(write_mode == JSON_WRITE_MODE_MERGE) {
+            combine_json_files(outPath, jsonFiles[i], outPath);
+        } else {
+            display_error_and_abort("Unknown json write mode: \"", write_mode, "\"");
+        }
     }
-}
 
-void AssetCompiler::append_manifest_files(std::string &packageDir, std::string &outDirPath) {
-    std::vector<fs::path> packageFiles = get_filenames_from_directory_only_with_extension(packageDir, ".json");
-    std::string curDataPath, newDataPath;
-    for(int i = 0; i < packageFiles.size(); i++) {
-        std::string filename = packageFiles[i].filename().string();
-        // meta.json is a special file, so ignore it!
-        if(filename == "meta.json") { 
-            continue;
+    for(int i = 0; i < otherFiles.size(); i++) {
+        if(path_is_directory(otherFiles[i])) {
+            std::string localPath = otherFiles[i].substr(packageDir.length() + 1);
+            if(localPath == "meta") {
+                continue; // Skip meta folder if it exists.
+            }
+            std::string outPath = outDirPath + "/" + localPath;
+            create_directory(outPath);
+        } else {
+            std::string localPath = otherFiles[i].substr(packageDir.length() + 1);
+            std::string outPath = outDirPath + "/" + localPath;
+            copy_file(otherFiles[i], outPath);
         }
-        newDataPath = packageFiles[i].string();
-        curDataPath = outDirPath + "/" + filename;
-        combine_json_files(curDataPath, newDataPath, curDataPath);
     }
 }
 
-// TODO: Implement more checks. Maybe add option to skip verification?
-void AssetCompiler::verify_json_files(std::string &folderPath) {
-    std::vector<std::string> jsonFiles = get_filenames_from_directory_with_extension(folderPath, ".json");
-    for(std::string &jsonFilePath : jsonFiles) {
-        // Need to make sure that all the json files within directories have a type in them.
-        std::string jsonFileType = get_string_from_json(jsonFilePath, "type");
-        if(jsonFileType == "") {
-            display_error_and_abort("\"", jsonFilePath, "\" does not have a type.");
+void AssetCompiler::merge_output(std::string &tempDir, std::string &outDir) {
+    std::vector<std::string> tempFiles = get_filenames_from_directory(tempDir);
+    std::vector<std::string> curFiles = get_filenames_from_directory(outDir);
+    std::vector<std::string> sameFiles;
+
+    for(int i = 0; i < tempFiles.size(); i++) {
+        std::string tempFilepath = tempFiles[i];
+        std::string localPath = tempFilepath.substr(tempDir.length() + 1);
+        if(path_exists(outDir + "/" + localPath)) {
+            sameFiles.push_back(localPath);
+            tempFiles.erase(tempFiles.begin() + i);
+            i--;
+        }
+    }
+
+    for(int i = 0; i < curFiles.size(); i++) {
+        std::string curOutFilepath = curFiles[i];
+        std::string localPath = curOutFilepath.substr(outDir.length() + 1);
+        if(path_exists(tempDir + "/" + localPath)) {
+            curFiles.erase(curFiles.begin() + i);
+            i--;
+        }
+    }
+
+    // tempFiles now only has files that only exist in the temp directory. (Need to get added)
+    // curFiles now only has files that only exist in the current output directory. (Need to get deleted)
+    // sameFiles only has files that exist in both directories. (Need to compare their hashes)
+
+    for(std::string &pathToRemove : curFiles) {
+        delete_path(pathToRemove);
+    }
+    
+    for(std::string &pathToAdd : tempFiles) {
+        std::string localPath = pathToAdd.substr(tempDir.length() + 1);
+        std::string outAddPath = outDir + "/" + localPath;
+        if(path_is_directory(pathToAdd)) {
+            create_directory(outAddPath);
+        } else {
+            copy_file(pathToAdd, outAddPath);
+        }
+    }
+    
+    for(std::string &sameFileLocalPath : sameFiles) {
+        std::string tempFilePath = tempDir + "/" + sameFileLocalPath;
+        std::string curFilePath = outDir + "/" + sameFileLocalPath;
+        if(!path_is_directory(tempFilePath)) {
+            bool doHashCheck = true;
+            if(ends_with(tempFilePath, ".json")) {
+                std::string pkgRebuild = "onchange";
+                if(json_has_key(tempFilePath, "pkg-rebuild")) {
+                    pkgRebuild = get_string_from_json(tempFilePath, "pkg-rebuild");
+                    make_lowercase(pkgRebuild);
+                }
+                if(pkgRebuild == "always") {
+                    doHashCheck = false;
+                } else if(pkgRebuild == "onchange") {
+                    doHashCheck = true;
+                } else {
+                    display_error_and_abort("Unknown pkg-rebuild value \"", pkgRebuild, "\".");
+                }
+            }
+            if(doHashCheck) {
+                std::string tempFileHash = calculate_file_md5(tempFilePath);
+                std::string curFileHash = calculate_file_md5(curFilePath);
+                if(tempFileHash == curFileHash) {
+                    continue;
+                }
+            }
+            copy_file(tempFilePath, curFilePath);
         }
     }
 }

tools/dkr_assets_tool_classes/compiler/compiler.h

@@ -8,14 +8,14 @@
 
 #include "../common/util.h"
 #include "../common/types.h"
+#include "../common/revision_number.h"
 
 class AssetCompiler {
 public:
     AssetCompiler();
     ~AssetCompiler();
 private:
-    void apply_package(std::string &packagesDirPath, std::string &packageName, std::string &outDirPath);
-    void copy_stuff_from_folders(std::string &packagesDirPath, std::string &outDirPath);
-    void append_manifest_files(std::string &packagesDirPath, std::string &outDirPath);
-    void verify_json_files(std::string &folderPath);
+    void apply_package(std::string &packagesDirPath, std::string &packageName, std::string &tempDirPath, std::string &outDirPath);
+    void check_meta_json(std::string &packageName, std::string metaJsonFilepath);
+    void merge_output(std::string &tempDir, std::string &outDir);
 };

tools/dkr_assets_tool_classes/extractor/config.cpp

@@ -1,8 +1,5 @@
 #include "config.h"
 
-#define REVISION_NUMBER 2
-#define REVISION_MINOR_NUMBER 0
-
 const std::string BASE_DIR_NAME = "vanilla";
 const std::string EXT_DIR_NAME = "packages";
 

tools/dkr_assets_tool_classes/extractor/config.h

@@ -18,6 +18,7 @@ namespace fs = std::filesystem;
 
 #include "../common/util.h"
 #include "../common/types.h"
+#include "../common/revision_number.h"
 #include "../libs/json.hpp"
 #include "../libs/ThreadPool.h"
 

tools/dkr_assets_tool_classes/extractor/rom.cpp

@@ -7,7 +7,7 @@ ROM::ROM(std::string filename){
     romFilename = filename;
     
     test_endianness();
-    calculate_md5();
+    md5 = calculate_md5(bytes);
 }
 
 ROM::~ROM(){
@@ -84,34 +84,6 @@ void ROM::test_endianness(){
     }
 }
 
-void ROM::calculate_md5() {
-    uint8_t buffer[0x4000];
-    uint8_t digest[MD5_DIGEST_LENGTH];
-
-    std::stringstream ss;
-
-    MD5_CTX md5Context;
-
-    MD5_Init(&md5Context);
-    MD5_Update(&md5Context, &bytes[0], bytes.size()); 
-    int res = MD5_Final(digest, &md5Context);
-
-    if(res == 0){ // hash failed or raise an exception
-        md5 = "";
-        return;
-    }
-
-    // set up stringstream format
-    ss << std::hex << std::setfill('0');
-
-
-    for(uint8_t uc: digest) {
-        ss << std::setw(2) << (int)uc;
-    }
-
-    md5 = ss.str();
-}
-
 bool ROM::readROMFile(std::string filename) {
     try 
     {

tools/dkr_assets_tool_classes/extractor/rom.h

@@ -2,7 +2,6 @@
 
 #include "../common/util.h"
 #include <iomanip>
-#include <openssl/md5.h>
 
 class ROM {
 public:
@@ -13,14 +12,15 @@ public:
     uint8_t get_byte(int romOffset);
     uint32_t get_uint(int romOffset);
     
-    std::string get_md5();
     int get_size();
+
+
+    std::string get_md5();
     
     bool is_valid();
     
 private:
     void test_endianness();
-    void calculate_md5();
     bool readROMFile(std::string filename);
     bool writeROMFile(std::string filename);