diff --git a/Makefile b/Makefile index ab84a3e2..d9d76650 100755 --- a/Makefile +++ b/Makefile @@ -410,7 +410,7 @@ endif build_assets: $(info Building Assets...) - @$(TOOLS_DIR)/dkr_assets_tool build -o $(ASSETS_OUTPUT) -dkrv $(REGION).$(VERSION) $(MODDED_ARG) >&2 || echo FAIL + @$(TOOLS_DIR)/dkr_assets_tool build -o $(ASSETS_OUTPUT) -dkrv $(REGION).$(VERSION) $(MODDED_ARG) >&2 || exit 1 ############################### diff --git a/README.md b/README.md index 3a581e73..9ef7b2fe 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,9 @@ All versions are supported, and the US 1.0 version (SHA1 = 0cb115d8716dbbc2922fd As of June 11, 2025, this is our current score: -    Decomp progress: 90.62% +    Decomp progress: 90.87% -    Documentation progress: 59.08% +    Documentation progress: 59.40% --- @@ -119,28 +119,28 @@ s32 is_drumstick_unlocked(void) { As of June 11, 2025, this is our current score: ``` - =================================================================== - ADVENTURE ONE (ASM -> C Decompilation) - -------------- 90.62% Complete (91.40% NON_MATCHING) -------------- - # Decompiled functions: 1912 - # GLOBAL_ASM remaining: 32 + ==================================================================== + ADVENTURE ONE (ASM -> C Decompilation) + -------------- 90.87% Complete (91.65% NON_MATCHING) --------------- + # Decompiled functions: 1913 + # GLOBAL_ASM remaining: 37 # NON_MATCHING functions: 4 - # NON_EQUIVALENT WIP functions: 28 - --------------------------- Game Status --------------------------- + # NON_EQUIVALENT WIP functions: 30 + --------------------------- Game Status ---------------------------- Balloons: 42/47, Keys: 4/4, Trophies: 4/5 - T.T. Amulets: 4/4, Wizpig Amulets: 4/4 - ------------------------------------------------------------------- + T.T. Amulets: 4/4, Wizpig Amulets: 4/4 + -------------------------------------------------------------------- We are racing in Star City. (Lap 3/3) - =================================================================== + ==================================================================== ADVENTURE TWO (Cleanup & Documentation) - ------------------------- 59.08% Complete ------------------------- - # Documented functions: 1207 - # Undocumented remaining: 440 - --------------------------- Game Status --------------------------- - Balloons: 28/47, Keys: 3/4, Trophies: 2/5 - T.T. Amulets: 3/4, Wizpig Amulets: 2/4 - ------------------------------------------------------------------- - We are collecting silver coins in Pirate Lagoon. (8/8 silver coins) - =================================================================== + ------------------------- 59.40% Complete -------------------------- + # Documented functions: 1208 + # Undocumented remaining: 445 + --------------------------- Game Status ---------------------------- + Balloons: 29/47, Keys: 3/4, Trophies: 2/5 + T.T. Amulets: 3/4, Wizpig Amulets: 2/4 + -------------------------------------------------------------------- + We are collecting silver coins in Treasure Caves. (1/8 silver coins) + ==================================================================== ``` diff --git a/mods/assets/examples/CustomAnimatedModelTest/asset_object_models.meta.json b/mods/assets/examples/CustomAnimatedModelTest/asset_object_models.meta.json new file mode 100644 index 00000000..487ecfb6 --- /dev/null +++ b/mods/assets/examples/CustomAnimatedModelTest/asset_object_models.meta.json @@ -0,0 +1,12 @@ +{ + "files": { + "order": [ + "ASSET_CUSTOM_ANIMATED_TESTCUBE" + ], + "sections": { + "ASSET_CUSTOM_ANIMATED_TESTCUBE": { + "filename": "animatedBox.json" + } + } + } +} \ No newline at end of file diff --git a/mods/assets/examples/CustomAnimatedModelTest/meta.json b/mods/assets/examples/CustomAnimatedModelTest/meta.json new file mode 100644 index 00000000..5ef5c9f9 --- /dev/null +++ b/mods/assets/examples/CustomAnimatedModelTest/meta.json @@ -0,0 +1,5 @@ +{ + "name": "Custom Animated Model Test", + "authors" : ["Example"], + "description": "A simple cube model with a custom animation." +} \ No newline at end of file diff --git a/mods/assets/examples/CustomAnimatedModelTest/objects/models/animatedBox.bin b/mods/assets/examples/CustomAnimatedModelTest/objects/models/animatedBox.bin new file mode 100644 index 00000000..0d65e769 Binary files /dev/null and b/mods/assets/examples/CustomAnimatedModelTest/objects/models/animatedBox.bin differ diff --git a/mods/assets/examples/CustomAnimatedModelTest/objects/models/animatedBox.gltf b/mods/assets/examples/CustomAnimatedModelTest/objects/models/animatedBox.gltf new file mode 100644 index 00000000..c8af5d74 --- /dev/null +++ b/mods/assets/examples/CustomAnimatedModelTest/objects/models/animatedBox.gltf @@ -0,0 +1,288 @@ +{ + "asset":{ + "generator":"Khronos glTF Blender I/O v4.2.70", + "version":"2.0" + }, + "scene":0, + "scenes":[ + { + "name":"Scene", + "nodes":[ + 0 + ] + } + ], + "nodes":[ + { + "mesh":0, + "name":"Cube" + } + ], + "animations":[ + { + "channels":[ + { + "sampler":0, + "target":{ + "node":0, + "path":"weights" + } + } + ], + "name":"Key.001Action", + "samplers":[ + { + "input":9, + "interpolation":"LINEAR", + "output":10 + } + ] + } + ], + "materials":[ + { + "doubleSided":true, + "name":"Material.001", + "pbrMetallicRoughness":{ + "baseColorTexture":{ + "index":0 + }, + "metallicFactor":0, + "roughnessFactor":0.5 + } + } + ], + "meshes":[ + { + "extras":{ + "targetNames":[ + "Key 1", + "Key 2" + ] + }, + "name":"Cube.001", + "primitives":[ + { + "attributes":{ + "POSITION":0, + "NORMAL":1, + "TEXCOORD_0":2, + "COLOR_0":3 + }, + "indices":4, + "material":0, + "targets":[ + { + "POSITION":5, + "NORMAL":6 + }, + { + "POSITION":7, + "NORMAL":8 + } + ] + } + ], + "weights":[ + 0, + 0 + ] + } + ], + "textures":[ + { + "sampler":0, + "source":0 + } + ], + "images":[ + { + "mimeType":"image/png", + "name":"logo", + "uri":"logo.png" + } + ], + "accessors":[ + { + "bufferView":0, + "componentType":5126, + "count":24, + "max":[ + 1, + 1, + 1 + ], + "min":[ + -1, + -1, + -1 + ], + "type":"VEC3" + }, + { + "bufferView":1, + "componentType":5126, + "count":24, + "type":"VEC3" + }, + { + "bufferView":2, + "componentType":5126, + "count":24, + "type":"VEC2" + }, + { + "bufferView":3, + "componentType":5123, + "count":24, + "normalized":true, + "type":"VEC4" + }, + { + "bufferView":4, + "componentType":5123, + "count":36, + "type":"SCALAR" + }, + { + "bufferView":5, + "componentType":5126, + "count":24, + "max":[ + 0.5727667212486267, + 0, + 0.9693711996078491 + ], + "min":[ + -1.1323010921478271, + 0, + -0.820482611656189 + ], + "type":"VEC3" + }, + { + "bufferView":6, + "componentType":5126, + "count":24, + "type":"VEC3" + }, + { + "bufferView":7, + "componentType":5126, + "count":24, + "max":[ + 0.8063837289810181, + 0, + 1.0958211421966553 + ], + "min":[ + -0.9761124849319458, + 0, + -0.6933281421661377 + ], + "type":"VEC3" + }, + { + "bufferView":8, + "componentType":5126, + "count":24, + "type":"VEC3" + }, + { + "bufferView":9, + "componentType":5126, + "count":25, + "max":[ + 1 + ], + "min":[ + 0 + ], + "type":"SCALAR" + }, + { + "bufferView":10, + "componentType":5126, + "count":50, + "type":"SCALAR" + } + ], + "bufferViews":[ + { + "buffer":0, + "byteLength":288, + "byteOffset":0, + "target":34962 + }, + { + "buffer":0, + "byteLength":288, + "byteOffset":288, + "target":34962 + }, + { + "buffer":0, + "byteLength":192, + "byteOffset":576, + "target":34962 + }, + { + "buffer":0, + "byteLength":192, + "byteOffset":768, + "target":34962 + }, + { + "buffer":0, + "byteLength":72, + "byteOffset":960, + "target":34963 + }, + { + "buffer":0, + "byteLength":288, + "byteOffset":1032, + "target":34962 + }, + { + "buffer":0, + "byteLength":288, + "byteOffset":1320, + "target":34962 + }, + { + "buffer":0, + "byteLength":288, + "byteOffset":1608, + "target":34962 + }, + { + "buffer":0, + "byteLength":288, + "byteOffset":1896, + "target":34962 + }, + { + "buffer":0, + "byteLength":100, + "byteOffset":2184 + }, + { + "buffer":0, + "byteLength":200, + "byteOffset":2284 + } + ], + "samplers":[ + { + "magFilter":9729, + "minFilter":9987 + } + ], + "buffers":[ + { + "byteLength":2484, + "uri":"animatedBox.bin" + } + ] +} diff --git a/mods/assets/examples/CustomAnimatedModelTest/objects/models/animatedBox.json b/mods/assets/examples/CustomAnimatedModelTest/objects/models/animatedBox.json new file mode 100644 index 00000000..572ce8cb --- /dev/null +++ b/mods/assets/examples/CustomAnimatedModelTest/objects/models/animatedBox.json @@ -0,0 +1,8 @@ +{ + "model": "animatedBox.gltf", + "type": "ObjectModel", + "scale": 10.0, + "animation": { + "seconds-per-keyframe": 0.25 + } +} diff --git a/mods/assets/examples/CustomAnimatedModelTest/objects/models/logo.png b/mods/assets/examples/CustomAnimatedModelTest/objects/models/logo.png new file mode 100644 index 00000000..1cf5a1d0 Binary files /dev/null and b/mods/assets/examples/CustomAnimatedModelTest/objects/models/logo.png differ diff --git a/src/objects.c b/src/objects.c index 8394f958..f3fac4f1 100644 --- a/src/objects.c +++ b/src/objects.c @@ -6490,12 +6490,10 @@ void func_80019808(s32 updateRate) { } } } - if ((currentLevelHeader->race_type != RACETYPE_CHALLENGE_BATTLE && numFinishedRacers > 0) || ((((numHumanRacers == 1 && numHumanRacersFinished == 1) || (numHumanRacers >= 2 && numHumanRacersFinished >= numHumanRacers)) || numFinishedRacers >= 3))) { - for (i = 0; i < gNumRacers; i++) { if (currentLevelHeader->race_type == RACETYPE_CHALLENGE_BATTLE) { sp5C[i] = 10 - racer[i]->bananas; @@ -6542,7 +6540,6 @@ void func_80019808(s32 updateRate) { } while (racerIndex != -1); gSwapLeadPlayer = FALSE; - if (!is_in_tracks_mode() && (racer[0]->finishPosition == 1 || is_in_two_player_adventure() && racer[1]->finishPosition == 1) && diff --git a/tools/Makefile b/tools/Makefile index 84378c83..a717c329 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -1,5 +1,11 @@ # Make `OPT` -g for better debugging info. OPT := -O2 + +# If -g option was set, then add some debugging flags for the stack tracer. +ifneq (,$(findstring -g,$(OPT))) + OPT += -rdynamic -ldl -lelf -ldwarf -DDEBUG -I/usr/include/libdwarf +endif + CC := gcc CXX := g++ CFLAGS := -I . -Wall -Wextra -Wno-unused-parameter -pedantic -std=c99 $(OPT) -s diff --git a/tools/dkr_assets_tool_settings.json b/tools/dkr_assets_tool_settings.json index 27dfd4e9..125c2a8d 100644 --- a/tools/dkr_assets_tool_settings.json +++ b/tools/dkr_assets_tool_settings.json @@ -8,7 +8,7 @@ "build-subpath" : "build/", "data-subpath" : "data/", "model-scale": 0.001, - "threads" : 0, + "threads" : 1, "debug": { "keep-uncompressed": true, "write-assets-map": false diff --git a/tools/dkr_assets_tool_src/builder/buildInfo.cpp b/tools/dkr_assets_tool_src/builder/buildInfo.cpp index be98d3da..ea09434a 100644 --- a/tools/dkr_assets_tool_src/builder/buildInfo.cpp +++ b/tools/dkr_assets_tool_src/builder/buildInfo.cpp @@ -29,8 +29,9 @@ BuildInfo::BuildInfo(std::string buildId, std::string buildSectionId, const Json _type = jsonFile.get_string("/type", "NoType"); } -BuildInfo::BuildInfo(std::string buildId, std::string buildSectionId, const std::vector &outData, size_t fileIndex, const fs::path &dir, const BuildInfoContext &infoContext) - : out(outData), _dstPath(""), _localDirectory(dir), _jsonFile(std::nullopt), _infoContext(infoContext), _buildId(buildId), _buildSectionId(buildSectionId), _type("Binary"), _fileIndex(fileIndex) { +BuildInfo::BuildInfo(std::string buildId, std::string buildSectionId, std::string deferredFromBuildId, std::string deferredFromSectionId, const std::vector &outData, + size_t fileIndex, const fs::path &dir, const BuildInfoContext &infoContext) : out(outData), _dstPath(""), _localDirectory(dir), _jsonFile(std::nullopt), _infoContext(infoContext), + _buildId(buildId), _buildSectionId(buildSectionId), _deferredFromBuildId(deferredFromBuildId), _deferredFromSectionId(deferredFromSectionId), _type("Binary"), _fileIndex(fileIndex) { _buildType = BUILD_TO_BINARY; } @@ -116,10 +117,22 @@ std::string BuildInfo::get_section_build_id() const { return _buildSectionId; } +std::string BuildInfo::get_deferred_from_build_id() const { + return _deferredFromBuildId; +} + +std::string BuildInfo::get_deferred_from_section_build_id() const { + return _deferredFromSectionId; +} + size_t BuildInfo::get_file_index() const { return _fileIndex; } +bool BuildInfo::is_deferred() const { + return !_deferredFromSectionId.empty(); +} + bool BuildInfo::is_complete() const { return _complete; } diff --git a/tools/dkr_assets_tool_src/builder/buildInfo.h b/tools/dkr_assets_tool_src/builder/buildInfo.h index 7c7dd74a..ab75ca27 100644 --- a/tools/dkr_assets_tool_src/builder/buildInfo.h +++ b/tools/dkr_assets_tool_src/builder/buildInfo.h @@ -27,7 +27,8 @@ public: //BuildInfo(std::string buildId, JsonFile *src, const fs::path &dst, const fs::path &dir); // BUILD_TO_FILE, TODO //BuildInfo(std::string buildId, JsonFile *src, const fs::path &dir); // BUILD_TO_BINARY BuildInfo(std::string buildId, std::string buildSectionId, const JsonFile &src, size_t fileIndex, const fs::path &dir, const BuildInfoContext &infoContext); - BuildInfo(std::string buildId, std::string buildSectionId, const std::vector &outData, size_t fileIndex, const fs::path &dir, const BuildInfoContext &infoContext); + BuildInfo(std::string buildId, std::string buildSectionId, std::string deferredFromBuildId, std::string deferredFromSectionId, const std::vector &outData, + size_t fileIndex, const fs::path &dir, const BuildInfoContext &infoContext); ~BuildInfo(); std::vector out; @@ -56,8 +57,12 @@ public: std::string get_type() const; std::string get_build_id() const; std::string get_section_build_id() const; + std::string get_deferred_from_build_id() const; + std::string get_deferred_from_section_build_id() const; size_t get_file_index() const; + bool is_deferred() const; // Returns true if this asset was deferred from another section. + bool is_complete() const; void done(); @@ -75,6 +80,8 @@ private: std::string _buildId; std::string _buildSectionId; + std::string _deferredFromBuildId; + std::string _deferredFromSectionId; std::string _type; size_t _fileIndex; BuildInfoType _buildType; diff --git a/tools/dkr_assets_tool_src/builder/buildInfoCollection.cpp b/tools/dkr_assets_tool_src/builder/buildInfoCollection.cpp index f875ad43..8f54d814 100644 --- a/tools/dkr_assets_tool_src/builder/buildInfoCollection.cpp +++ b/tools/dkr_assets_tool_src/builder/buildInfoCollection.cpp @@ -46,7 +46,8 @@ void BuildInfoCollection::add_build_info(std::string sectionBuildId, std::string _buildInfoMutex.unlock(); } -size_t BuildInfoCollection::add_deferred_build_info(std::string sectionBuildId, std::string buildId, const std::vector &out, const fs::path &dir, const BuildInfoContext &infoContext) { +size_t BuildInfoCollection::add_deferred_build_info(std::string sectionBuildId, std::string buildId, std::string deferredFromBuildId, std::string deferredFromSectionId, + const std::vector &out, const fs::path &dir, const BuildInfoContext &infoContext) { _buildInfoMutex.lock(); if(_buildInfoSections.find(sectionBuildId) == _buildInfoSections.end()) { @@ -59,7 +60,7 @@ size_t BuildInfoCollection::add_deferred_build_info(std::string sectionBuildId, "(BuildInfoCollection::add_build_info) Max number of assets reached! Limit was ", MAX_NUMBER_OF_BUILD_INFOS); size_t fileIndex = _buildInfoSections[sectionBuildId].size(); - _buildInfos.emplace_back(buildId, sectionBuildId, out, fileIndex, dir, infoContext); + _buildInfos.emplace_back(buildId, sectionBuildId, deferredFromBuildId, deferredFromSectionId, out, fileIndex, dir, infoContext); // Do not process deferred assets. _buildInfos.back().done(); @@ -72,8 +73,8 @@ size_t BuildInfoCollection::add_deferred_build_info(std::string sectionBuildId, return fileIndex; } -void BuildInfoCollection::add_deferred_build_info(std::string sectionBuildId, std::string buildId, size_t fileIndex, const std::vector &out, - const fs::path &dir, const BuildInfoContext &infoContext) { +void BuildInfoCollection::add_deferred_build_info(std::string sectionBuildId, std::string buildId, std::string deferredFromBuildId, std::string deferredFromSectionId, + size_t fileIndex, const std::vector &out, const fs::path &dir, const BuildInfoContext &infoContext) { _buildInfoMutex.lock(); if(_buildInfoSections.find(sectionBuildId) == _buildInfoSections.end()) { @@ -89,7 +90,7 @@ void BuildInfoCollection::add_deferred_build_info(std::string sectionBuildId, st DebugHelper::assert_(index < MAX_NUMBER_OF_BUILD_INFOS, "(BuildInfoCollection::add_build_info) Max number of assets reached! Limit was ", MAX_NUMBER_OF_BUILD_INFOS); - _buildInfos.emplace_back(buildId, sectionBuildId, out, fileIndex, dir, infoContext); + _buildInfos.emplace_back(buildId, sectionBuildId, deferredFromBuildId, deferredFromSectionId, out, fileIndex, dir, infoContext); // Do not process deferred assets. _buildInfos.back().done(); diff --git a/tools/dkr_assets_tool_src/builder/buildInfoCollection.h b/tools/dkr_assets_tool_src/builder/buildInfoCollection.h index 147697a4..a9d3c251 100644 --- a/tools/dkr_assets_tool_src/builder/buildInfoCollection.h +++ b/tools/dkr_assets_tool_src/builder/buildInfoCollection.h @@ -19,12 +19,12 @@ public: void add_build_info(std::string sectionBuildId, std::string buildId, const JsonFile &src, const fs::path &dir, const BuildInfoContext &infoContext); // Adds deferred info to the end of the list. Returns the fileIndex of the added asset. - size_t add_deferred_build_info(std::string sectionBuildId, std::string buildId, const std::vector &out, - const fs::path &dir, const BuildInfoContext &infoContext); + size_t add_deferred_build_info(std::string sectionBuildId, std::string buildId, std::string deferredFromBuildId, std::string deferredFromSectionId, + const std::vector &out, const fs::path &dir, const BuildInfoContext &infoContext); // Adds deferred info to a specific file index. - void add_deferred_build_info(std::string sectionBuildId, std::string buildId, size_t fileIndex, const std::vector &out, - const fs::path &dir, const BuildInfoContext &infoContext); + void add_deferred_build_info(std::string sectionBuildId, std::string buildId, std::string deferredFromBuildId, std::string deferredFromSectionId, + size_t fileIndex, const std::vector &out, const fs::path &dir, const BuildInfoContext &infoContext); void run_builds(std::function callbackFunction, const std::vector §ionsOrder); diff --git a/tools/dkr_assets_tool_src/builder/buildTypes/buildJpFonts.cpp b/tools/dkr_assets_tool_src/builder/buildTypes/buildJpFonts.cpp index 20ffa775..8f60bfa0 100644 --- a/tools/dkr_assets_tool_src/builder/buildTypes/buildJpFonts.cpp +++ b/tools/dkr_assets_tool_src/builder/buildTypes/buildJpFonts.cpp @@ -121,8 +121,8 @@ void build_v77_beta_font(BuildInfo &info, const JsonFile &jsonFile) { //outJpFontTableView.print(0, outJpFontTable.size()); // Add jp font table to build collection. - collection.add_deferred_build_info("ASSET_JAPANESE_FONTS_TABLE", "ASSET_JAPANESE_FONTS_TABLE", outJpFontTable, - info.get_dst_folder() / deferInfo.outputPath, info.get_info_context()); + collection.add_deferred_build_info("ASSET_JAPANESE_FONTS_TABLE", "ASSET_JAPANESE_FONTS_TABLE", info.get_build_id(), + info.get_section_build_id(), outJpFontTable,info.get_dst_folder() / deferInfo.outputPath, info.get_info_context()); } @@ -282,8 +282,8 @@ void build_v79_jp_fonts(BuildInfo &info, const JsonFile &jsonFile) { } // Add jp font table to build collection. - collection.add_deferred_build_info("ASSET_JAPANESE_FONTS_TABLE", "ASSET_JAPANESE_FONTS_TABLE", outJpFontTable, - info.get_dst_folder() / deferInfo.outputPath, info.get_info_context()); + collection.add_deferred_build_info("ASSET_JAPANESE_FONTS_TABLE", "ASSET_JAPANESE_FONTS_TABLE", info.get_build_id(), + info.get_section_build_id(), outJpFontTable,info.get_dst_folder() / deferInfo.outputPath, info.get_info_context()); } void BuildJPFonts::build(BuildInfo &info) { diff --git a/tools/dkr_assets_tool_src/builder/buildTypes/buildLevelHeader.cpp b/tools/dkr_assets_tool_src/builder/buildTypes/buildLevelHeader.cpp index 030d0074..ef666b5b 100644 --- a/tools/dkr_assets_tool_src/builder/buildTypes/buildLevelHeader.cpp +++ b/tools/dkr_assets_tool_src/builder/buildTypes/buildLevelHeader.cpp @@ -89,8 +89,8 @@ void build_level_name(BuildInfo &info, const JsonFile &jsonFile) { BuildInfoCollection &collection = info.get_collection(); std::string buildId = info.get_build_id() + deferInfo.idPostfix; - collection.add_deferred_build_info(levelNamesSectionBuildId, buildId, info.get_file_index(), - out, info.get_dst_folder() / deferInfo.outputPath, info.get_info_context()); + collection.add_deferred_build_info(levelNamesSectionBuildId, buildId, info.get_build_id(), info.get_section_build_id(), + info.get_file_index(), out, info.get_dst_folder() / deferInfo.outputPath, info.get_info_context()); } } diff --git a/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModel.cpp b/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModel.cpp index 1096d8f9..edafee64 100644 --- a/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModel.cpp +++ b/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModel.cpp @@ -10,29 +10,42 @@ #include "fileTypes/vertex.hpp" #include "fileTypes/textureInfo.hpp" +#include "libs/bytes_view.hpp" + #include "misc/globalSettings.h" +#include "builder/buildInfoCollection.h" + using namespace DkrAssetsTool; void BuildModel::generate_object_model(BuildInfo &info) { + const JsonFile &modelJson = info.get_src_json_file(); + float modelScale = modelJson.get_float("/scale", 1000.0f); + size_t numberOfTriangles = _total_number_of_triangles(); size_t numberOfVertices = _total_number_of_vertices(); + size_t numberOfAnimatedVertices = _total_number_of_animated_vertices(); size_t numberOfBatches = _total_number_of_batches(); size_t numberOfBatchesToAllocate = _total_number_of_batches_to_allocate(); size_t numberOfTextures = _total_number_of_textures(); + size_t numberOfAttachPoints = _attachPoints.size(); size_t sizeOfHeader = DataHelper::align16(sizeof(ObjectModel)); size_t sizeOfTextures = DataHelper::align16(sizeof(DkrTextureInfo) * numberOfTextures); - size_t sizeOfVertices = DataHelper::align16(sizeof(DkrVertex) * numberOfVertices); + size_t sizeOfVertices = DataHelper::align16(sizeof(DkrVertex) * (numberOfVertices + numberOfAttachPoints)); size_t sizeOfTriangles = DataHelper::align16(sizeof(DkrTriangle) * numberOfTriangles); size_t sizeOfBatches = DataHelper::align16(sizeof(DkrBatch) * numberOfBatchesToAllocate); + size_t sizeOfAnimatedVertexIndices = _animations.empty() ? 0 : DataHelper::align16(sizeof(int16_t) * numberOfVertices); + size_t sizeOfAttachPoints = DataHelper::align16(sizeof(be_int16_t) * numberOfAttachPoints); // Calculated the file size for the binary model file. size_t calculatedFileSize = sizeOfHeader + sizeOfTextures + sizeOfVertices + sizeOfTriangles + - sizeOfBatches; + sizeOfBatches + + sizeOfAnimatedVertexIndices + + sizeOfAttachPoints; info.out.resize(calculatedFileSize); @@ -42,27 +55,47 @@ void BuildModel::generate_object_model(BuildInfo &info) { objectModelHeader->vertices = objectModelHeader->textures + sizeOfTextures; objectModelHeader->triangles = objectModelHeader->vertices + sizeOfVertices; objectModelHeader->batches = objectModelHeader->triangles + sizeOfTriangles; - objectModelHeader->numberOfTextures= (int16_t)numberOfTextures; + objectModelHeader->animatedVertexIndices = objectModelHeader->batches + sizeOfBatches; + objectModelHeader->attachPoints = objectModelHeader->animatedVertexIndices + sizeOfAnimatedVertexIndices; + objectModelHeader->numberOfTextures = (int16_t)numberOfTextures; objectModelHeader->numberOfTriangles = (int16_t)numberOfTriangles; objectModelHeader->numberOfVertices = (int16_t)numberOfVertices; objectModelHeader->numberOfBatches = (int16_t)numberOfBatches; + objectModelHeader->numberOfAttachPoints = (int16_t)numberOfAttachPoints; objectModelHeader->fileSize = (int16_t)calculatedFileSize; + objectModelHeader->numberOfAnimatedVertices = (int16_t)numberOfAnimatedVertices; DkrTextureInfo *textures = reinterpret_cast(&info.out[objectModelHeader->textures]); DkrVertex *vertices = reinterpret_cast(&info.out[objectModelHeader->vertices]); DkrTriangle *triangles = reinterpret_cast(&info.out[objectModelHeader->triangles]); DkrBatch *batches = reinterpret_cast(&info.out[objectModelHeader->batches]); + be_int16_t *animatedVertexIndices = reinterpret_cast(&info.out[objectModelHeader->animatedVertexIndices]); + be_int16_t *attachPoints = reinterpret_cast(&info.out[objectModelHeader->attachPoints]); bool hasAnimatedTexture = false; for(size_t materialIndex = 0; materialIndex < numberOfTextures; materialIndex++) { BuildModelMaterial &material = _materials[materialIndex]; + material.write_to(&textures[materialIndex], info); hasAnimatedTexture = hasAnimatedTexture || material.is_texture_animated(); } objectModelHeader->hasAnimatedTexture = (int)hasAnimatedTexture; for(auto &block : _blocks) { - block.write_batches(_materialIds, _materials, batches, vertices, triangles); + block.write_batches(_materialIds, _materials, batches, vertices, triangles, modelScale); + } + + // Does nothing with static models. + _generate_animations(info, objectModelHeader, animatedVertexIndices); + + vertices = reinterpret_cast(&info.out[objectModelHeader->vertices]); + for(size_t i = 0; i < numberOfAttachPoints; i++) { + attachPoints[i] = numberOfVertices + i; + + DkrVertex *vertex = &vertices[numberOfVertices + i]; + vertex->x = static_cast(_attachPoints[i].x * modelScale); + vertex->y = static_cast(_attachPoints[i].y * modelScale); + vertex->z = static_cast(_attachPoints[i].z * modelScale); } bool debugKeepUncompressed = GlobalSettings::get_value("/debug/keep-uncompressed", false); @@ -105,7 +138,16 @@ void BuildModel::_add_material(std::string id, fs::path texturePath) { return; } _materialIds[id] = _materials.size(); - _materials.emplace_back(texturePath); + _materials.emplace_back(texturePath, id); +} + +void BuildModel::_add_material(std::string id, uint8_t *imgData, size_t imgDataLength, std::string mimeType) { + if(imgData == nullptr || imgDataLength == 0) { + _materialIds[id] = -1; + return; + } + _materialIds[id] = _materials.size(); + _materials.emplace_back(imgData, imgDataLength, mimeType, id); } void BuildModel::_set_material(std::string id) { @@ -145,6 +187,24 @@ size_t BuildModel::_total_number_of_vertices() { return count; } +size_t BuildModel::_total_number_of_animated_vertices() { + if(_animations.empty()) { + return 0; + } + + size_t count = 0; + + for(auto &block : _blocks) { + block.process_vertices([&count](BuildModelVertex &vertex) { + if(vertex.is_animated()) { + count++; + } + }); + } + + return count; +} + size_t BuildModel::_total_number_of_textures() { size_t count = 0; @@ -176,3 +236,148 @@ size_t BuildModel::_total_number_of_batches_to_allocate() { return count; } + +template +inline void process_keyframe(T *keyframe, size_t &animDataOffset, const std::vector> &animatedVertices, AnimWeights &weights, + std::optional previousWeights, float modelScale) { + size_t numberOfAnimatedVertices = animatedVertices.size(); + Vec3f vertexDelta; + + for(size_t i = 0; i < numberOfAnimatedVertices; i++) { + const BuildModelVertex &vertex = animatedVertices.at(i); + if(previousWeights.has_value()) { + vertexDelta = vertex.animated_position(weights) - vertex.animated_position(previousWeights.value()); + } else { + vertexDelta = vertex.animated_position(weights) - vertex.position(); + } + + keyframe[i].offsetX = static_cast(vertexDelta.x * modelScale); + keyframe[i].offsetY = static_cast(vertexDelta.y * modelScale); + keyframe[i].offsetZ = static_cast(vertexDelta.z * modelScale); + } + + animDataOffset += sizeof(T) * numberOfAnimatedVertices; +} + +void BuildModel::_generate_animations(BuildInfo &info, ObjectModel *objectModelHeader, be_int16_t *outAnimatedIndicesPtr) { + if((_animations.empty()) || (objectModelHeader->numberOfAnimatedVertices < 1)) { + return; + } + + std::vector animatedVertexIndices; + std::vector> animatedVertices; + + animatedVertexIndices.reserve(objectModelHeader->numberOfVertices); + animatedVertices.reserve(objectModelHeader->numberOfVertices); // Should have no more than the number of vertices. + + for(auto &block : _blocks) { + block.process_vertices([&animatedVertexIndices, &animatedVertices](BuildModelVertex &vertex) { + if(!vertex.is_animated()) { + animatedVertexIndices.emplace_back(-1); + return; // Don't include this vertex. + } + animatedVertexIndices.emplace_back(animatedVertices.size()); + animatedVertices.emplace_back(vertex); + }); + } + + for(size_t i = 0; i < objectModelHeader->numberOfVertices; i++) { + outAnimatedIndicesPtr[i] = animatedVertexIndices[i]; + } + + std::optional weights = std::nullopt; + std::optional translations = std::nullopt; + std::optional rotations = std::nullopt; + std::optional scales = std::nullopt; + + std::optional previousWeights = std::nullopt; + + size_t numberOfAnimatedVertices = animatedVertices.size(); + + bool debugKeepUncompressed = GlobalSettings::get_value("/debug/keep-uncompressed", false); + + BuildInfoCollection &collection = info.get_collection(); + + std::vector animationIndices; + + const JsonFile &modelJson = info.get_src_json_file(); + float secondsPerKeyframe = modelJson.get_float("/animation/seconds-per-keyframe", 0.267f); + float modelScale = modelJson.get_float("/scale", 1000.0f); + + size_t animIndex = 0; + for(BuildModelAnimation &anim : _animations) { + size_t numberOfKeyframes = std::floor(anim.get_max_time() / secondsPerKeyframe) + 1; + + size_t animationDataSize = sizeof(ObjectModelAnimationHeader) + + (sizeof(ObjectModelAnimationKeyframeHeader) * numberOfKeyframes) + // Each keyframe has a header that is 12-bytes long. Mainly used for head tilting? + (sizeof(ObjectModelAnimationKeyframeVertexS16) * numberOfAnimatedVertices) + // First keyframe uses 16-bit values. + (sizeof(ObjectModelAnimationKeyframeVertexS8) * (numberOfAnimatedVertices - 1) * numberOfKeyframes); // Following keyframes use 8-bit values. + + std::vector animationData(animationDataSize); + BytesView animDataView(animationData); + size_t animDataOffset = 0; + + ObjectModelAnimationHeader *animHeader = animDataView.data_cast(animDataOffset); + animHeader->keyframeCount = numberOfKeyframes; + + animDataOffset += sizeof(ObjectModelAnimationHeader); + + bool skipAnimation = false; + + for(size_t keyframe = 0; keyframe < numberOfKeyframes; keyframe++) { + //ObjectModelAnimationKeyframeHeader *keyframeHeader = animDataView.data_cast(animDataOffset); + animDataOffset += sizeof(ObjectModelAnimationKeyframeHeader); + + float keyframeTime = keyframe * secondsPerKeyframe; + anim.get_inputs_for_time(keyframeTime, weights, translations, rotations, scales); + + if(keyframe == 0) { + if(translations.has_value() || rotations.has_value() || scales.has_value()) { + DebugHelper::warn("Only morph target/shape key animation is supported at the moment."); + } + if(!weights.has_value()) { + skipAnimation = true; + break; // Only Morph Targets are supported at the moment. + } + + ObjectModelAnimationKeyframeVertexS16 *firstKeyframe = animDataView.data_cast(animDataOffset); + process_keyframe(firstKeyframe, animDataOffset, animatedVertices, + weights.value(), std::nullopt, modelScale); + } else { + ObjectModelAnimationKeyframeVertexS8 *followingKeyframe = animDataView.data_cast(animDataOffset); + process_keyframe(followingKeyframe, animDataOffset, animatedVertices, + weights.value(), previousWeights, modelScale); + } + + previousWeights = weights; + } + + if(skipAnimation) { + continue; + } + + while((animationData.size() % 16) != 0) { + animationData.emplace_back(0); + } + + std::string buildId = info.get_build_id() + "_ANIM_" + std::to_string(animIndex); + + if(debugKeepUncompressed) { + fs::path outUncompressedPath = GlobalSettings::get_decomp_path("build_subpath", "build/"); + outUncompressedPath /= "debug/custom-animations"; + outUncompressedPath /= buildId + ".bin"; + FileHelper::write_binary_file(animationData, outUncompressedPath, true); + } + + animationData = GzipHelper::compress_gzip(animationData); + + size_t newAnimId = collection.add_deferred_build_info("ASSET_OBJECT_ANIMATIONS", buildId, info.get_build_id(), info.get_section_build_id(), + animationData, info.get_path_to_directory(), info.get_info_context()); + + animationIndices.emplace_back(newAnimId); + + animIndex++; + } + + +} diff --git a/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModel.h b/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModel.h index 8dee74f3..2d66f571 100644 --- a/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModel.h +++ b/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModel.h @@ -8,9 +8,11 @@ #include "helpers/fileHelper.h" #include "fileTypes/batch.hpp" +#include "fileTypes/objectModel.hpp" #include "buildModelBlock.h" #include "buildModelMaterial.h" +#include "buildModelAnimation.h" namespace DkrAssetsTool { @@ -26,21 +28,29 @@ protected: std::vector _materials; std::map _materialIds; + std::vector _animations; + + std::vector _attachPoints; + void _add_new_triangle_to_current_block(const BuildModelVertex& vertA, const BuildModelVertex& vertB, const BuildModelVertex& vertC, Vec2f uv0, Vec2f uv1, Vec2f uv2); void _add_new_triangle_to_current_block(const BuildModelVertex& vertA, const BuildModelVertex& vertB, const BuildModelVertex& vertC); void _new_block(); void _add_material(std::string id, fs::path texturePath=""); + void _add_material(std::string id, uint8_t *imgData, size_t imgDataLength, std::string mimeType); void _set_material(std::string id); std::optional> _try_get_material(std::string id); size_t _total_number_of_triangles(); size_t _total_number_of_vertices(); + size_t _total_number_of_animated_vertices(); size_t _total_number_of_textures(); size_t _total_number_of_batches(); size_t _total_number_of_batches_to_allocate(); + + void _generate_animations(BuildInfo &info, ObjectModel *objectModelHeader, be_int16_t *outAnimatedIndicesPtr); }; } \ No newline at end of file diff --git a/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModelAnimation.cpp b/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModelAnimation.cpp new file mode 100644 index 00000000..9167080f --- /dev/null +++ b/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModelAnimation.cpp @@ -0,0 +1,215 @@ +#include "buildModelAnimation.h" + +#include "helpers/dataHelper.h" +#include "helpers/debugHelper.h" +#include "helpers/gltf/gltfHelper.h" + +#include + +using namespace DkrAssetsTool; + +/***************************************************************************************************************/ + +AnimWeights BuildModelAnimationInputs::get_weights_at_time(float time) { + if(time < 0) { + time = 0; + } + + if(!weights.has_value()) { + return {}; + } + + std::map &inputWeights = weights.value(); + + if(inputWeights.find(time) != inputWeights.end()) { + // No need to interpolate! + return inputWeights.at(time); + } + + auto upperBound = inputWeights.upper_bound(time); + auto lowerBound = std::prev(upperBound); + + float lowerTime = lowerBound->first; + float upperTime = upperBound->first; + + DebugHelper::info(time, " is between ", lowerTime, " and ", upperTime); + + // Should handle the edgecases of time being outside timeline range. + if(lowerTime == upperTime) { + return upperBound->second; + } + + AnimWeights &lowerWeights = lowerBound->second; + AnimWeights &upperWeights = upperBound->second; + + // Percentage of interpolation. + float t = (time - lowerTime) / (upperTime - lowerTime); + + AnimWeights out(lowerWeights.size()); + + for(size_t i = 0; i < lowerWeights.size(); i++) { + // Linear interpolation of weights. + out[i] = lowerWeights[i] + t * (upperWeights[i] - lowerWeights[i]); + } + + return out; +} + +AnimTranslations BuildModelAnimationInputs::get_translations_at_time(float time) { + return {}; +} + +AnimRotations BuildModelAnimationInputs::get_rotations_at_time(float time) { + return {}; +} + +AnimScales BuildModelAnimationInputs::get_scales_at_time(float time) { + return {}; +} + +/***************************************************************************************************************/ + +BuildModelAnimation::BuildModelAnimation() { +} + +BuildModelAnimation::BuildModelAnimation(GltfFile &gltf, tinygltf::Animation &animation) { + for(size_t i = 0; i < animation.channels.size(); i++) { + tinygltf::AnimationChannel &channel = animation.channels[i]; + + if(channel.sampler < 0) { + continue; + } + + tinygltf::AnimationSampler &sampler = animation.samplers[channel.sampler]; + + if(sampler.interpolation != "LINEAR") { + DebugHelper::warn(sampler.interpolation + " is currently not supported. Will use LINEAR instead."); + } + + // I believe animation inputs are always floats? Couldn't find anything 100% definitive in the spec. + std::vector inputTimes = GltfHelper::get_data_from_accessor(gltf, sampler.input); + size_t numberOfInputs = inputTimes.size(); + + if(channel.target_path == "weights") { + // morph targets + auto tryGetWeights = GltfHelper::get_weights_from_accessor(gltf, sampler.output); + DebugHelper::assert_(tryGetWeights.has_value(), + "(BuildModelAnimation::BuildModelAnimation) Failed to get weights from gltf accessor ", sampler.output); + std::vector &weights = tryGetWeights.value(); + size_t numberOfWeights = weights.size() / numberOfInputs; + + for(size_t j = 0; j < numberOfInputs; j++) { + add_weights_input(inputTimes[j], weights, j * numberOfWeights, numberOfWeights); + } + } else if(channel.target_path == "translation") { + DebugHelper::warn("(BuildModelAnimation::BuildModelAnimation) TODO: Support translation channel.path"); + } else if(channel.target_path == "rotation") { + DebugHelper::warn("(BuildModelAnimation::BuildModelAnimation) TODO: Support rotation channel.path"); + } else if(channel.target_path == "scale") { + DebugHelper::warn("(BuildModelAnimation::BuildModelAnimation) TODO: Support scale channel.path"); + } + } +} + +void BuildModelAnimation::add_weights_input(float time, std::vector &values, std::optional offset, std::optional length) { + if(offset == std::nullopt) { + offset = 0; + } + if(length == std::nullopt) { + length = values.size() - offset.value(); + } + + if(_inputs.weights == std::nullopt) { + _inputs.weights = std::map>(); + } + + _inputs.weights.value()[time] = std::vector(values.begin() + offset.value(), values.begin() + offset.value() + length.value()); + + if(time > _maxTime) { + _maxTime = time; + } +} + +std::vector BuildModelAnimation::get_timeline() { + std::vector timeline; + int numberOfSections = 0; + + if(_inputs.weights.has_value()) { + for(auto &pair : _inputs.weights.value()) { + timeline.emplace_back(pair.first); + } + numberOfSections++; + } + + if(_inputs.translations.has_value()) { + for(auto &pair : _inputs.translations.value()) { + if(DataHelper::vector_has(timeline, pair.first)) { + continue; + } + timeline.emplace_back(pair.first); + } + numberOfSections++; + } + + if(_inputs.rotations.has_value()) { + for(auto &pair : _inputs.rotations.value()) { + if(DataHelper::vector_has(timeline, pair.first)) { + continue; + } + timeline.emplace_back(pair.first); + } + numberOfSections++; + } + + if(_inputs.scales.has_value()) { + for(auto &pair : _inputs.scales.value()) { + if(DataHelper::vector_has(timeline, pair.first)) { + continue; + } + timeline.emplace_back(pair.first); + } + numberOfSections++; + } + + if(numberOfSections > 1) { + std::sort(timeline.begin(), timeline.end()); + } + + return timeline; +} + +float BuildModelAnimation::get_max_time() const { + return _maxTime; +} + +// Returns weights/translations/rotations/scales for a specific time input. Values will be set to std::nullopt if there is no data. +// Values will be interpolated if there is data, but the time input was not specified. +void BuildModelAnimation::get_inputs_for_time(float time, std::optional &outWeights, std::optional &outTranslations, + std::optional &outRotations, std::optional &outScales) { + // Set new data + if(_inputs.weights.has_value()) { + outWeights = _inputs.get_weights_at_time(time); + } else { + outWeights = std::nullopt; + } + if(_inputs.translations.has_value()) { + outTranslations = _inputs.get_translations_at_time(time); + } else { + outTranslations = std::nullopt; + } + if(_inputs.rotations.has_value()) { + outRotations = _inputs.get_rotations_at_time(time); + } else { + outRotations = std::nullopt; + } + if(_inputs.scales.has_value()) { + outScales = _inputs.get_scales_at_time(time); + } else { + outScales = std::nullopt; + } +} + +bool BuildModelAnimation::only_has_morph_targets() const { + return _inputs.weights.has_value() && !_inputs.translations.has_value() && + !_inputs.rotations.has_value() && !_inputs.scales.has_value(); +} diff --git a/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModelAnimation.h b/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModelAnimation.h new file mode 100644 index 00000000..85e8adc0 --- /dev/null +++ b/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModelAnimation.h @@ -0,0 +1,62 @@ +#pragma once + +#include +#include +#include + +#include "helpers/mathHelper.h" +#include "helpers/gltf/gltfFile.h" + +#include "libs/bytes_view.hpp" + +namespace DkrAssetsTool { + +typedef std::vector AnimWeights; + +// Key: node index, Value: value for property +typedef std::map AnimTranslations; +typedef std::map AnimRotations; +typedef std::map AnimScales; + +struct BuildModelAnimationInputs { + // Key: time in seconds, value: array of weights for that moment in time. (Morph target animation only) + std::optional> weights = std::nullopt; + + // Key: time in seconds, value: array of translations for that moment in time. + std::optional> translations = std::nullopt; + // Key: time in seconds, value: array of rotation (Quaternions) for that moment in time. + std::optional> rotations = std::nullopt; + // Key: time in seconds, value: array of scale values for that moment in time. + std::optional> scales = std::nullopt; + + AnimWeights get_weights_at_time(float time); + AnimTranslations get_translations_at_time(float time); + AnimRotations get_rotations_at_time(float time); + AnimScales get_scales_at_time(float time); +}; + +class BuildModelAnimation { +public: + BuildModelAnimation(); + BuildModelAnimation(GltfFile &gltf, tinygltf::Animation &animation); + + void add_weights_input(float time, std::vector &values, std::optional offset=std::nullopt, std::optional length=std::nullopt); + + std::vector get_timeline(); + float get_max_time() const; + + // Returns weights/translations/rotations/scales for a specific time input. Values will be set to std::nullopt if there is no data. + // Values will be interpolated if there is data, but the time input was not specified. + void get_inputs_for_time(float time, std::optional &outWeights, std::optional &outTranslations, + std::optional &outRotations, std::optional &outScales); + + // The logic is a bit simpler if there is only morph target animation. + bool only_has_morph_targets() const; + +private: + BuildModelAnimationInputs _inputs; + + float _maxTime = 0.0f; +}; + +} diff --git a/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModelBatch.cpp b/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModelBatch.cpp index 33a26464..423ff13a 100644 --- a/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModelBatch.cpp +++ b/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModelBatch.cpp @@ -47,13 +47,16 @@ size_t BuildModelBatch::number_of_vertices() { } void BuildModelBatch::write_batch(const std::map &materialIds, std::vector &materials, - DkrBatch* outBatch, DkrVertex*& outVertices, DkrTriangle*& outTriangles) { - uint32_t flags = 0x00000003; + DkrBatch* outBatch, DkrVertex*& outVertices, DkrTriangle*& outTriangles, float modelScale) { + uint32_t flags = 0x03; // Default is (RENDER_ANTI_ALIASING | RENDER_Z_COMPARE) bool materialDoubleSided = false; if(_materialId.has_value()) { outBatch->textureIndex = materialIds.at(_materialId.value()); BuildModelMaterial &material = materials[outBatch->textureIndex]; + if(material.has_render_flags()) { + flags = material.get_render_flags(); + } if(material.is_texture_animated()) { flags |= 0x10000; // RENDER_TEX_ANIM } @@ -65,8 +68,8 @@ void BuildModelBatch::write_batch(const std::map &materialIds, outBatch->flags = flags; bool useVertexColors = false; - for(auto &vertex : _vertices) { - vertex.write_to(outVertices); + for(BuildModelVertex &vertex : _vertices) { + vertex.write_to(outVertices, modelScale); outVertices++; // Next out vertex. if(!vertex.is_color_white()) { useVertexColors = true; @@ -77,7 +80,7 @@ void BuildModelBatch::write_batch(const std::map &materialIds, uint8_t triFlags = materialDoubleSided ? 0x40 : 0x00; - for(auto &tri : _triangles) { + for(BuildModelTriangle &tri : _triangles) { outTriangles->flags = triFlags; outTriangles->vi0 = tri.a; outTriangles->vi1 = tri.b; @@ -92,6 +95,12 @@ void BuildModelBatch::write_batch(const std::map &materialIds, } } +void BuildModelBatch::process_vertices(std::function callbackFunction) { + for(BuildModelVertex &vertex : _vertices) { + callbackFunction(vertex); + } +} + std::optional BuildModelBatch::_index_of_vertex(const BuildModelVertex& vertex) { int index = DataHelper::vector_index_of(_vertices, vertex); if(index == -1) { diff --git a/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModelBatch.h b/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModelBatch.h index b0eabe5c..3e060c2f 100644 --- a/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModelBatch.h +++ b/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModelBatch.h @@ -37,7 +37,9 @@ public: size_t number_of_vertices(); void write_batch(const std::map &materialIds, std::vector &materials, - DkrBatch* outBatch, DkrVertex*& outVertices, DkrTriangle*& outTriangles); + DkrBatch* outBatch, DkrVertex*& outVertices, DkrTriangle*& outTriangles, float modelScale); + + void process_vertices(std::function callbackFunction); private: std::vector _triangles; diff --git a/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModelBlock.cpp b/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModelBlock.cpp index 9990d935..4736ed49 100644 --- a/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModelBlock.cpp +++ b/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModelBlock.cpp @@ -62,11 +62,11 @@ size_t BuildModelBlock::number_of_batches_to_allocate() { } void BuildModelBlock::write_batches(const std::map &materialIds, std::vector &materials, - DkrBatch*& outBatches, DkrVertex*& outVertices, DkrTriangle*& outTriangles) { + DkrBatch*& outBatches, DkrVertex*& outVertices, DkrTriangle*& outTriangles, float modelScale) { size_t vertexCount = 0; size_t triCount = 0; - for(auto &batch : _batches) { - batch.write_batch(materialIds, materials, outBatches, outVertices, outTriangles); + for(BuildModelBatch &batch : _batches) { + batch.write_batch(materialIds, materials, outBatches, outVertices, outTriangles, modelScale); outBatches->verticesOffset = vertexCount; outBatches->trianglesOffset = triCount; outBatches->unk1 = (int8_t)batch.number_of_vertices(); @@ -77,4 +77,10 @@ void BuildModelBlock::write_batches(const std::map &materialId // One extra batch for that extra entry. outBatches->verticesOffset = vertexCount; outBatches->trianglesOffset = triCount; -} \ No newline at end of file +} + +void BuildModelBlock::process_vertices(std::function callbackFunction) { + for(BuildModelBatch &batch : _batches) { + batch.process_vertices(callbackFunction); + } +} diff --git a/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModelBlock.h b/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModelBlock.h index 4d5c9f7c..7ae0e83f 100644 --- a/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModelBlock.h +++ b/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModelBlock.h @@ -3,6 +3,7 @@ #include #include #include +#include #include "buildModelVertex.h" #include "buildModelMaterial.h" @@ -27,7 +28,9 @@ public: size_t number_of_batches_to_allocate(); void write_batches(const std::map &materialIds, std::vector &materials, - DkrBatch*& outBatches, DkrVertex*& outVertices, DkrTriangle*& outTriangles); + DkrBatch*& outBatches, DkrVertex*& outVertices, DkrTriangle*& outTriangles, float modelScale); + + void process_vertices(std::function callbackFunction); private: std::vector _batches; diff --git a/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModelMaterial.cpp b/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModelMaterial.cpp index c4c12c7d..50cc67bf 100644 --- a/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModelMaterial.cpp +++ b/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModelMaterial.cpp @@ -3,38 +3,43 @@ #include "helpers/debugHelper.h" #include "helpers/imageHelper.h" +#include "libs/bytes_view.hpp" + #include "builder/buildTypes/buildTexture.h" using namespace DkrAssetsTool; -BuildModelMaterial::BuildModelMaterial() : _texturePath(std::nullopt) { +BuildModelMaterial::BuildModelMaterial() : _texturePath(std::nullopt), _name("") { } -BuildModelMaterial::BuildModelMaterial(fs::path texturePath) : _texturePath(texturePath) { +BuildModelMaterial::BuildModelMaterial(fs::path texturePath, std::string name) : _texturePath(texturePath), _name(name) { } + +BuildModelMaterial::BuildModelMaterial(uint8_t *imgData, size_t imgDataLength, std::string mimeType, std::string name) : + _imgData(imgData), _imgDataLength(imgDataLength), _mimeType(mimeType), _name(name) {} void BuildModelMaterial::write_to(DkrTextureInfo *outTexInfo, BuildInfo &info) { - DebugHelper::assert_(has_texture(), - "(BuildModelMaterial::write_to) Trying to write out to a DkrTextureInfo without a texture!"); - int width, height; - - ImageHelper::get_width_and_height(_texturePath.value(), width, height); + get_texture_width_and_height(width, height); DebugHelper::assert_((width > 0) && (width < 255), - "(BuildModelMaterial::write_to) Invalid width of ", width, " for texture ", _texturePath.value().filename()); + "(BuildModelMaterial::write_to) Invalid width of ", width, " for material \"", _name, "\""); DebugHelper::assert_((height > 0) && (height < 255), - "(BuildModelMaterial::write_to) Invalid height of ", height, " for texture ", _texturePath.value().filename()); + "(BuildModelMaterial::write_to) Invalid height of ", height, " for material \"", _name, "\""); outTexInfo->width = width; outTexInfo->height = height; - outTexInfo->format = 0x11; // 0x11 = Opaque RGBA16 texture. TODO: Add support for more formats! - outTexInfo->id = BuildTexture::build_deferred(info, _texturePath.value()); + outTexInfo->format = 0x11; // 0x11 = Opaque RGBA16 texture. Might not be used in-game? + if(is_embedded()) { + outTexInfo->id = BuildTexture::build_deferred(info, _imgData.value(), _imgDataLength, _mimeType, _name, _buildTextureInfo); + } else { + outTexInfo->id = BuildTexture::build_deferred(info, _texturePath.value(), _buildTextureInfo); + } } bool BuildModelMaterial::has_texture() const { - return _texturePath.has_value(); + return _texturePath.has_value() || _imgData.has_value(); } bool BuildModelMaterial::is_texture_animated() { @@ -48,16 +53,50 @@ bool BuildModelMaterial::is_texture_animated() { } bool BuildModelMaterial::is_texture_double_sided() { - if(!_texturePath.has_value()) { - return false; - } if(!_isTextureDoubleSided.has_value()) { + if(!_texturePath.has_value()) { + return false; + } _isTextureDoubleSided = ImageHelper::guess_if_texture_double_sided(_texturePath.value()); } return _isTextureDoubleSided.value(); } + +bool BuildModelMaterial::is_embedded() const { + return _imgData.has_value(); +} + +std::optional BuildModelMaterial::get_embedded_data(size_t &outLength, std::string &outMimeType) const { + if(!_imgData.has_value()) { + outLength = 0; + outMimeType = ""; + return std::nullopt; + } + outLength = _imgDataLength; + outMimeType = _mimeType; + return _imgData; +} + +std::string BuildModelMaterial::get_name() const { + return _name; +} void BuildModelMaterial::get_texture_width_and_height(int &outWidth, int &outHeight) { + if(is_embedded()) { + DebugHelper::assert_(_mimeType == "image/png", + "(BuildModelMaterial::get_texture_width_and_height) Sorry, only embedded png images are supported at the moment!"); + bool isPngData = (_imgData.value()[1] == 'P') && (_imgData.value()[2] == 'N') && (_imgData.value()[3] == 'G'); + DebugHelper::assert_(isPngData, + "(BuildModelMaterial::get_texture_width_and_height) embedded image data for material ", _name, " was NOT a .png image."); + + // Only need to see the two u32s at offsets 0x10 and 0x14 in the png data. + std::vector imgDataAsVector(_imgData.value() + 0x10, _imgData.value() + 0x18); + BytesView view(imgDataAsVector); + outWidth = view.get_u32_be(0); + outHeight = view.get_u32_be(4); + return; + } + if(!_texturePath.has_value()) { outWidth = 0; outHeight = 0; @@ -66,3 +105,36 @@ void BuildModelMaterial::get_texture_width_and_height(int &outWidth, int &outHei ImageHelper::get_width_and_height(_texturePath.value(), outWidth, outHeight); } + +void BuildModelMaterial::set_wrap_mode(std::string wrapS, std::string wrapT) { + _buildTextureInfo.wrapS = wrapS; + _buildTextureInfo.wrapT = wrapT; +} + +void BuildModelMaterial::set_format(std::string format) { + _buildTextureInfo.format = format; +} + +void BuildModelMaterial::set_rendermode(std::string renderMode) { + _buildTextureInfo.renderMode = renderMode; +} + +void BuildModelMaterial::set_flip_vertically(bool shouldFlipVertically) { + _buildTextureInfo.shouldFlip = shouldFlipVertically; +} + +void BuildModelMaterial::set_double_sided(bool doubleSided) { + _isTextureDoubleSided = doubleSided; +} + +void BuildModelMaterial::set_flag(uint32_t flag) { + _renderFlags |= flag; +} + +bool BuildModelMaterial::has_render_flags() const { + return _renderFlags != 0; +} + +uint32_t BuildModelMaterial::get_render_flags() const { + return _renderFlags; +} diff --git a/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModelMaterial.h b/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModelMaterial.h index 5ec5097c..1f725789 100644 --- a/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModelMaterial.h +++ b/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModelMaterial.h @@ -6,6 +6,8 @@ #include "builder/buildInfo.h" +#include "builder/buildTypes/buildTexture.h" // for DeferredTextureInfo + #include "fileTypes/textureInfo.hpp" namespace DkrAssetsTool { @@ -13,7 +15,8 @@ namespace DkrAssetsTool { class BuildModelMaterial { public: BuildModelMaterial(); - BuildModelMaterial(fs::path texturePath); // Path to texture. + BuildModelMaterial(fs::path texturePath, std::string name); // Path to texture. + BuildModelMaterial(uint8_t *imgData, size_t imgDataLength, std::string mimeType, std::string name); // Embedded texture void write_to(DkrTextureInfo *outTexInfo, BuildInfo &info); @@ -22,12 +25,33 @@ public: bool is_texture_animated(); bool is_texture_double_sided(); + bool is_embedded() const; + std::optional get_embedded_data(size_t &outLength, std::string &outMimeType) const; + + std::string get_name() const; + void get_texture_width_and_height(int &outWidth, int &outHeight); + void set_wrap_mode(std::string wrapS, std::string wrapT); + void set_format(std::string format); + void set_rendermode(std::string renderMode); + void set_flip_vertically(bool shouldFlipVertically); + void set_double_sided(bool doubleSided); + + void set_flag(uint32_t flag); + bool has_render_flags() const; + uint32_t get_render_flags() const; + private: - std::optional _texturePath; + uint32_t _renderFlags = 0; + BuildTexture::DeferredTextureInfo _buildTextureInfo; + std::optional _imgData = std::nullopt; + size_t _imgDataLength = 0; + std::string _mimeType = ""; + std::optional _texturePath = std::nullopt; std::optional _isTextureAnimated = std::nullopt; std::optional _isTextureDoubleSided = std::nullopt; + std::string _name; }; } diff --git a/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModelVertex.cpp b/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModelVertex.cpp index 70a197ff..89dc3c20 100644 --- a/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModelVertex.cpp +++ b/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModelVertex.cpp @@ -1,5 +1,7 @@ #include "buildModelVertex.h" +#include "helpers/debugHelper.h" + using namespace DkrAssetsTool; const Vec3f POSITION_ZERO = {}; @@ -38,11 +40,11 @@ bool BuildModelVertex::is_color_white() const { return _color == COLOR_WHITE; } -void BuildModelVertex::write_to(DkrVertex *outVertex) { +void BuildModelVertex::write_to(DkrVertex *outVertex, float modelScale) { // Write position - outVertex->x = static_cast(_position.x * 10); - outVertex->y = static_cast(_position.y * 10); - outVertex->z = static_cast(_position.z * 10); + outVertex->x = static_cast(_position.x * modelScale); + outVertex->y = static_cast(_position.y * modelScale); + outVertex->z = static_cast(_position.z * modelScale); // Write color outVertex->r = _color.get_red_as_byte(); @@ -55,3 +57,47 @@ void BuildModelVertex::write_to(DkrVertex *outVertex) { os << "{ pos: " << vertex.position() << ", color: 0x" << std::hex << vertex.color().get_as_rgba() << std::dec << " }"; return os; } + +void BuildModelVertex::add_morph_target(Vec3f newTarget) { + if(!_targetPositions.has_value()) { + _targetPositions = std::vector(); + } + _targetPositions.value().emplace_back(newTarget); +} + +bool BuildModelVertex::has_targets() const { + return _targetPositions.has_value(); +} + +bool BuildModelVertex::is_animated() const { + if(!_targetPositions.has_value()) { + return false; + } + const std::vector &targets = _targetPositions.value(); + for(const Vec3f &target : targets) { + if(!target.is_zero()) { + return true; + } + } + + return false; // If all targets are zero, then the vertex is not animated. +} + +Vec3f BuildModelVertex::animated_position(const std::vector &weights) const { + if(weights.size() == 0 || !_targetPositions.has_value()) { + return position(); + } + + DebugHelper::assert_(weights.size() == _targetPositions.value().size(), + "(BuildModelVertex::animated_position) Number of weights does not match number of morph targets!"); + + Vec3f out = position(); + + const std::vector &targets = _targetPositions.value(); + + for(size_t i = 0; i < weights.size(); i++) { + out += targets.at(i) * weights.at(i); + } + + return out; +} diff --git a/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModelVertex.h b/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModelVertex.h index 3695fae4..f6bb8c5a 100644 --- a/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModelVertex.h +++ b/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/buildModelVertex.h @@ -28,11 +28,19 @@ public: bool is_color_white() const; - void write_to(DkrVertex *outVertex); + void write_to(DkrVertex *outVertex, float modelScale); + + void add_morph_target(Vec3f newTarget); + bool has_targets() const; + bool is_animated() const; // Different from has_targets(), because it is possible that the targets don't move. + + Vec3f animated_position(const std::vector &weights) const; private: Vec3f _position; Vec4f _color; + + std::optional> _targetPositions = std::nullopt; }; } diff --git a/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/gltfModel.cpp b/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/gltfModel.cpp index e69de29b..d4ed9846 100644 --- a/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/gltfModel.cpp +++ b/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/gltfModel.cpp @@ -0,0 +1,348 @@ +#include "gltfModel.h" + +#include "helpers/fileHelper.h" +#include "helpers/dataHelper.h" +#include "helpers/debugHelper.h" +#include "helpers/mathHelper.h" + +#include +#include +#include +#include + +using namespace DkrAssetsTool; + +GltfBuildModel::GltfBuildModel(GltfFile &gltf, CContext &cContext, bool isObjectModel) : _isObjectModel(isObjectModel) { + if(gltf.empty()) { + return; + } + + // Ignore these nodes, since they are used for meta-data. + std::vector rootNodeBlacklist; + if(_isObjectModel) { + rootNodeBlacklist = { "collisionspheres", "attachpoints" }; + } else { + rootNodeBlacklist = { "bsp" }; + } + + std::vector rootNodeIndices = gltf.get_root_node_indices(rootNodeBlacklist); + + if(rootNodeIndices.empty()) { + return; + } + + // Process animation inputs + for(int i = 0; i < gltf.number_of_animations(); i++) { + tinygltf::Animation &animation = gltf.get_animation(i); + _animations.emplace_back(gltf, animation); + } + + // Process materials + for(int i = 0; i < gltf.number_of_materials(); i++) { + tinygltf::Material &material = gltf.get_material(i); + + bool hasExtras = material.extras.Type() != tinygltf::NULL_TYPE; + + if(material.name.empty()) { + material.name = "material_" + std::to_string(i); + } + + std::string materialId = material.name; + + int materialImageIndex = _get_image_index_for_material(gltf, i); + if(materialImageIndex == -1) { + // No image associated. + _add_material(materialId); + } else { + tinygltf::Image &image = gltf.get_image(materialImageIndex); + if(image.uri.empty()) { + if(image.bufferView == -1) { + // No image data found. + _add_material(materialId); + } else { + // Embedded image. + tinygltf::BufferView &embeddedImgBufferView = gltf.get_bufferview(image.bufferView); + tinygltf::Buffer &embeddedImgBuffer = gltf.get_buffer(embeddedImgBufferView.buffer); + uint8_t *imgData = GltfHelper::get_buffer_pointer(embeddedImgBufferView, embeddedImgBuffer, 0); + + _add_material(materialId, imgData, embeddedImgBufferView.byteLength, image.mimeType); + } + } else { + // image from a filepath. + auto tryGetFilepath = FileHelper::determine_full_filepath(image.uri, gltf.get_filepath().parent_path()); + if(!tryGetFilepath.has_value()) { + DebugHelper::error("Could not find the texture image: ", image.uri); + } + std::string pathToImage = tryGetFilepath.value(); + _add_material(materialId, pathToImage); + } + } + // Get the material we just added, to add some extras + BuildModelMaterial &buildMaterial = _materials.back(); + + if(hasExtras) { + if(GltfHelper::has_extra(material.extras, "/format")) { + buildMaterial.set_format(GltfHelper::get_extra(material.extras, "/format", "")); + } + if(GltfHelper::has_extra(material.extras, "/render-mode")) { + buildMaterial.set_rendermode(GltfHelper::get_extra(material.extras, "/render-mode", "")); + } + if(GltfHelper::has_extra(material.extras, "/flip-texture")) { + buildMaterial.set_flip_vertically(GltfHelper::get_extra(material.extras, "/flip-texture", false)); + } + if(GltfHelper::has_extra(material.extras, "/flags")) { + buildMaterial.set_flag(_get_render_flags_from_extras(material.extras, "/flags", cContext)); + } + } + + // Get the sampler for the material so we can get wrapS/wrapT for the texture. + int materialSamplerIndex = _get_image_sampler_index_for_material(gltf, i); + if(materialSamplerIndex >= 0) { + tinygltf::Sampler &sampler = gltf.get_sampler(materialSamplerIndex); + std::string wrapS = (sampler.wrapS == TINYGLTF_TEXTURE_WRAP_CLAMP_TO_EDGE) ? "clamp" : "wrap"; + std::string wrapT = (sampler.wrapT == TINYGLTF_TEXTURE_WRAP_CLAMP_TO_EDGE) ? "clamp" : "wrap"; + buildMaterial.set_wrap_mode(wrapS, wrapT); + } + + buildMaterial.set_double_sided(material.doubleSided); + + } + + Mat4x4f rootMatrix; + + // Process the base model + for(int rootNodeIndex : rootNodeIndices) { + _process_node_base(gltf, rootNodeIndex, rootMatrix); + } + + std::optional attachPointsIndex = gltf.get_root_node_from_name("attachpoints"); + + if(attachPointsIndex.has_value()) { + tinygltf::Node &attachPointsNode = gltf.get_node(attachPointsIndex.value()); + for(int childIndex : attachPointsNode.children) { + tinygltf::Node &attachPointNode = gltf.get_node(childIndex); + if(attachPointNode.translation.empty()) { + continue; + } + Vec3f position(attachPointNode.translation, 0); + _attachPoints.emplace_back(position); + } + } +} + +void GltfBuildModel::_process_node_base(GltfFile &gltf, int nodeIndex, const Mat4x4f &parentMatrix) { + tinygltf::Node &node = gltf.get_node(nodeIndex); + + Mat4x4f nodeMatrix = GltfHelper::calculate_node_matrix(node, parentMatrix); + + if(node.mesh >= 0) { + _process_mesh_base(gltf, node.mesh, nodeMatrix); + } + + for(int childIndex : node.children) { + _process_node_base(gltf, childIndex, nodeMatrix); + } +} + +void GltfBuildModel::_process_mesh_base(GltfFile &gltf, int meshIndex, const Mat4x4f &nodeMatrix) { + tinygltf::Mesh &mesh = gltf.get_mesh(meshIndex); + + for(auto &primitive : mesh.primitives) { + if(primitive.material == -1) { + _set_material(""); + } else { + tinygltf::Material &material = gltf.get_material(primitive.material); + _set_material(material.name); + } + + std::optional> positions = GltfHelper::get_positions_from_primitive(gltf, primitive); + std::optional> colors = GltfHelper::get_colors_from_primitive(gltf, primitive); + + std::optional> morphTargets = std::nullopt; + + if(_isObjectModel) { + // Only parse morph targets data for object models. + morphTargets = GltfHelper::get_morph_targets_from_primitive(gltf, primitive); + } + + std::vector vertices = _make_vertices_from_primitive_data(positions, colors, morphTargets, nodeMatrix); + + std::optional> gltfUVs = GltfHelper::get_uvs_from_primitive(gltf, primitive); + std::vector uvs = _adjust_uvs(gltf, primitive.material, gltfUVs); + + bool hasUVs = (uvs.size() > 0) && (primitive.material >= 0); + + std::optional> indices = GltfHelper::get_indices_from_primitive(gltf, primitive); + if(indices.has_value()) { + std::vector &triIndices = indices.value(); + size_t numberOfTriangles = triIndices.size() / 3; + + for(size_t i = 0; i < numberOfTriangles; i++) { + BuildModelVertex &vertexA = vertices[triIndices[i * 3 + 0]]; + BuildModelVertex &vertexB = vertices[triIndices[i * 3 + 1]]; + BuildModelVertex &vertexC = vertices[triIndices[i * 3 + 2]]; + + if(!hasUVs) { + _add_new_triangle_to_current_block(vertexA, vertexB, vertexC); + } else { + _add_new_triangle_to_current_block(vertexA, vertexB, vertexC, + uvs[triIndices[i * 3 + 0]], uvs[triIndices[i * 3 + 1]], uvs[triIndices[i * 3 + 2]]); + } + } + } else { + // Each 3 positions makes a triangle. + size_t numberOfTriangles = vertices.size() / 3; + + for(size_t i = 0; i < numberOfTriangles; i++) { + BuildModelVertex &vertexA = vertices[i * 3 + 0]; + BuildModelVertex &vertexB = vertices[i * 3 + 1]; + BuildModelVertex &vertexC = vertices[i * 3 + 2]; + + if(!hasUVs) { + _add_new_triangle_to_current_block(vertexA, vertexB, vertexC); + } else { + _add_new_triangle_to_current_block(vertexA, vertexB, vertexC, + uvs[i * 3 + 0], uvs[i * 3 + 1], uvs[i * 3 + 2]); + } + } + } + } +} + +std::vector GltfBuildModel::_make_vertices_from_primitive_data(std::optional> &positions, + std::optional> &colors, std::optional> &morphTargets, + const Mat4x4f &nodeMatrix) { + if(!positions.has_value()) { + return {}; + } + + std::vector &positionValues = positions.value(); + + size_t numberOfVertices = positionValues.size() / 3; + + std::vector out; + out.reserve(numberOfVertices); + + // TODO: Apply nodeMatrix to position Vec3f + + if(colors.has_value()) { + std::vector &colorValues = colors.value(); + for(size_t i = 0; i < numberOfVertices; i++) { + out.emplace_back(Vec3f(positionValues, i * 3), Vec4f(colorValues, i * 4)); + } + } else { + for(size_t i = 0; i < numberOfVertices; i++) { + out.emplace_back(Vec3f(positionValues, i * 3)); + } + } + + if(morphTargets.has_value()){ + std::vector &targets = morphTargets.value(); + size_t numberOfValuesPerKeyframe = positionValues.size(); + size_t numberOfKeyframes = targets.size() / numberOfValuesPerKeyframe; + for(size_t keyframe = 0; keyframe < numberOfKeyframes; keyframe++) { + for(size_t i = 0; i < numberOfVertices; i++) { + out[i].add_morph_target(Vec3f(targets, (keyframe * numberOfValuesPerKeyframe) + (i * 3))); + } + } + } + + return out; +} + +int GltfBuildModel::_get_image_index_for_material(GltfFile &gltf, int materialIndex) { + if(materialIndex == -1) { + return -1; + } + + if(_imageIndexForMaterialCache.find(materialIndex) == _imageIndexForMaterialCache.end()) { + tinygltf::Material &material = gltf.get_material(materialIndex); + int textureIndex = material.pbrMetallicRoughness.baseColorTexture.index; + + if(textureIndex == -1) { + _imageIndexForMaterialCache[materialIndex] = -1; + } else { + tinygltf::Texture &texture = gltf.get_texture(textureIndex); + _imageIndexForMaterialCache[materialIndex] = texture.source; + } + } + + return _imageIndexForMaterialCache.at(materialIndex); +} + +int GltfBuildModel::_get_image_sampler_index_for_material(GltfFile &gltf, int materialIndex) { + if(materialIndex == -1) { + return -1; + } + + if(_imageSamplerIndexForMaterialCache.find(materialIndex) == _imageSamplerIndexForMaterialCache.end()) { + tinygltf::Material &material = gltf.get_material(materialIndex); + int textureIndex = material.pbrMetallicRoughness.baseColorTexture.index; + + if(textureIndex == -1) { + _imageSamplerIndexForMaterialCache[materialIndex] = -1; + } else { + tinygltf::Texture &texture = gltf.get_texture(textureIndex); + _imageSamplerIndexForMaterialCache[materialIndex] = texture.sampler; + } + } + + return _imageSamplerIndexForMaterialCache.at(materialIndex); +} + +std::vector GltfBuildModel::_adjust_uvs(GltfFile &gltf, int materialIndex, std::optional> &uvs) { + if(!uvs.has_value() || uvs.value().size() == 0) { + return {}; + } + + std::vector out(uvs.value().size() / 2); + + int imageIndex = _get_image_index_for_material(gltf, materialIndex); + + if(imageIndex == -1) { + return out; + } + + std::vector &uvValues = uvs.value(); + tinygltf::Image &image = gltf.get_image(imageIndex); // Need the image info to get width & height of image. + + if(image.width == 0 || image.height == 0) { + DebugHelper::warn("Could not process UVs properly! Tex width = ", image.width, ", Tex height = ", image.height); + return out; + } + + for(size_t i = 0; i < out.size(); i++) { + out[i].u = uvValues[i * 2 + 0] * (image.width * 32); + out[i].v = uvValues[i * 2 + 1] * (image.height * 32); + } + + return out; +} + +uint32_t GltfBuildModel::_get_render_flags_from_extras(tinygltf::Value& extras, std::string jsonPointer, CContext &cContext) { + std::vector flags; + switch(GltfHelper::get_extra_type(extras, "/flags")) { + case tinygltf::ARRAY_TYPE: + { + GltfHelper::get_extra_array(extras, "/flags", flags); + break; + } + case tinygltf::STRING_TYPE: + { + std::string flagsString = GltfHelper::get_extra(extras, "/flags", ""); + StringHelper::split_and_trim(flagsString, {',','|'}, flags); // Flags are separated by either commas or vertical pipes. + break; + } + default: + // Unsupported. + return 0; + } + uint32_t out = 0; + for(const std::string &flag : flags) { + if(flag.empty()) { + continue; + } + out |= cContext.get_int_value_of_symbol(flag); + } + return out; +} diff --git a/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/gltfModel.h b/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/gltfModel.h index 3f59c932..8a7664ef 100644 --- a/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/gltfModel.h +++ b/tools/dkr_assets_tool_src/builder/buildTypes/buildModel/gltfModel.h @@ -1,2 +1,40 @@ #pragma once +#include +#include "helpers/fileHelper.h" + +#include "helpers/gltf/gltfFile.h" +#include "helpers/gltf/gltfHelper.h" + +#include "buildModel.h" +#include "buildModelAnimation.h" + +namespace DkrAssetsTool { + +class GltfBuildModel : public BuildModel { +public: + GltfBuildModel(GltfFile &gltf, CContext &cContext, bool isObjectModel = false); +private: + void _process_node_base(GltfFile &gltf, int nodeIndex, const Mat4x4f &parentMatrix); + void _process_mesh_base(GltfFile &gltf, int meshIndex, const Mat4x4f &nodeMatrix); + + std::vector _make_vertices_from_primitive_data(std::optional> &positions, + std::optional> &colors, std::optional> &morphTargets, + const Mat4x4f &nodeMatrix); + int _get_image_index_for_material(GltfFile &gltf, int materialIndex); + int _get_image_sampler_index_for_material(GltfFile &gltf, int materialIndex); + std::vector _adjust_uvs(GltfFile &gltf, int materialIndex, std::optional> &uvs); + + uint32_t _get_render_flags_from_extras(tinygltf::Value& extras, std::string jsonPointer, CContext &cContext); + + bool _isObjectModel = false; + + std::unordered_map _imageIndexForMaterialCache; + std::unordered_map _imageSamplerIndexForMaterialCache; + + int _lastMaterialChecked = -1; + + std::unordered_map _gltfMaterialIndexToId; +}; + +} diff --git a/tools/dkr_assets_tool_src/builder/buildTypes/buildObjectMap.cpp b/tools/dkr_assets_tool_src/builder/buildTypes/buildObjectMap.cpp index 4a882035..7050329c 100644 --- a/tools/dkr_assets_tool_src/builder/buildTypes/buildObjectMap.cpp +++ b/tools/dkr_assets_tool_src/builder/buildTypes/buildObjectMap.cpp @@ -20,38 +20,39 @@ void search_for_objects_node() { } -void set_obj_position(GltfFileNode *objNode, uint8_t *bytes, CStruct *entryCommon, int nodeIndex) { +void set_obj_position(tinygltf::Node &objNode, uint8_t *bytes, CStruct *entryCommon) { CStructEntry *memberX = entryCommon->get_entry(2); CStructEntry *memberY = entryCommon->get_entry(3); CStructEntry *memberZ = entryCommon->get_entry(4); - double scaledX, scaledY, scaledZ; - objNode->get_position(scaledX, scaledY, scaledZ); + //double scaledX, scaledY, scaledZ; + //objNode->get_position(scaledX, scaledY, scaledZ); + Vec3f position = GltfHelper::get_node_position(objNode); - int xPos = scaledX; //_settings.adjust_from_model_scale(scaledX); - int yPos = scaledY; //_settings.adjust_from_model_scale(scaledY); - int zPos = scaledZ; //_settings.adjust_from_model_scale(scaledZ); + int xPos = position.x; //_settings.adjust_from_model_scale(scaledX); + int yPos = position.y; //_settings.adjust_from_model_scale(scaledY); + int zPos = position.z; //_settings.adjust_from_model_scale(scaledZ); memberX->set_integer_to_data(bytes, xPos); memberY->set_integer_to_data(bytes, yPos); memberZ->set_integer_to_data(bytes, zPos); } -size_t calculate_size(GltfFileNode *objectsNode, BuildInfo &info, CContext &cContext, std::mutex &globalMutex) { +size_t calculate_size(GltfFile &gltf, tinygltf::Node &objectsNode, BuildInfo &info, CContext &cContext, std::mutex &globalMutex) { const BuildInfoContext &infoContext = info.get_info_context(); size_t out = 0; - int numberOfChildNodes = objectsNode->get_child_count(); + size_t numberOfChildNodes = objectsNode.children.size(); globalMutex.lock(); - for(int i = 0; i < numberOfChildNodes; i++) { - GltfFileNode *objNode = objectsNode->get_child_node_by_index(i); - std::string objectBuildId = objNode->get_extra("id", ""); + for(size_t i = 0; i < numberOfChildNodes; i++) { + tinygltf::Node &objNode = gltf.get_node(objectsNode.children[i]); + std::string objectBuildId = GltfHelper::get_extra(objNode.extras, "/id", ""); DebugHelper::assert_(!objectBuildId.empty(), - "(calculate_size) id was not defined in node: ", objNode->get_name()); + "(calculate_size) id was not defined in node: ", objNode.name); // First check if we have already calcuated the size of this object build id. if(_objectIdStructSizeCache.find(objectBuildId) != _objectIdStructSizeCache.end()) { @@ -120,10 +121,10 @@ void BuildObjectMap::build(BuildInfo &info) { DebugHelper::assert_(gltf.search_for_node_by_name("objects", objectsNodeIndex), "(BuildObjectMap::build) .glTF file does not contain a node named \"objects\"!"); - GltfFileNode *objectsNode = gltf.get_node(objectsNodeIndex); + tinygltf::Node &objectsNode = gltf.get_node(objectsNodeIndex); // calculate_size() also fills the `_objectIdToStructCache` map. - size_t dataSize = calculate_size(objectsNode, info, cContext, globalMutex); + size_t dataSize = calculate_size(gltf, objectsNode, info, cContext, globalMutex); size_t outSize = DataHelper::align8(dataSize + sizeof(LevelObjectMapHeader)); @@ -135,14 +136,14 @@ void BuildObjectMap::build(BuildInfo &info) { objectMap->header.fileSize = dataSize; uint8_t *bytes = &objectMap->entriesData[0]; - int numberOfChildNodes = objectsNode->get_child_count(); + int numberOfChildNodes = objectsNode.children.size(); for(int i = 0; i < numberOfChildNodes; i++) { - GltfFileNode *objNode = objectsNode->get_child_node_by_index(i); + tinygltf::Node &objNode = gltf.get_node(objectsNode.children[i]); - std::string objectBuildId = objNode->get_extra("id", ""); + std::string objectBuildId = GltfHelper::get_extra(objNode.extras, "/id", ""); - DebugHelper::assert_(!objectBuildId.empty(), "(BuildObjectMap::build) Node", i, + DebugHelper::assert_(!objectBuildId.empty(), "(BuildObjectMap::build) Child node ", i, " does not have id defined!"); int id = transTable.get_index_of_elem_in_array("/table", objectBuildId); @@ -167,7 +168,7 @@ void BuildObjectMap::build(BuildInfo &info) { entryCommon = cContext.get_struct("LevelObjectEntryCommon"); } - set_obj_position(objNode, bytes, entryCommon, i); + set_obj_position(objNode, bytes, entryCommon); if(entryCommon != entryStruct) { for(size_t entry = 0; entry < entryStruct->entry_count(); entry++) { diff --git a/tools/dkr_assets_tool_src/builder/buildTypes/buildObjectModel.cpp b/tools/dkr_assets_tool_src/builder/buildTypes/buildObjectModel.cpp index a4b10aaf..c7e07f37 100644 --- a/tools/dkr_assets_tool_src/builder/buildTypes/buildObjectModel.cpp +++ b/tools/dkr_assets_tool_src/builder/buildTypes/buildObjectModel.cpp @@ -8,6 +8,10 @@ using namespace DkrAssetsTool; +#include + +std::mutex mut; + void BuildObjectModel::build(BuildInfo &info) { const JsonFile &jsonFile = info.get_src_json_file(); @@ -43,8 +47,12 @@ void BuildObjectModel::build(BuildInfo &info) { if(modelExtension == ".obj") { ObjBuildModel objModel(modelPath); objModel.generate_object_model(info); - } else if(modelExtension == ".gltf") { - DebugHelper::error("(BuildObjectModel::build) TODO: GLTF support"); + } else if(modelExtension == ".gltf" || modelExtension == ".glb") { + CContext &cContext = info.get_c_context(); + // `gltf` needs to be here! If it's in the GltfBuildModel constructor, then the gltf data will be deallocated by the time it gets to generate_object_model(). + GltfFile gltf(modelPath); + GltfBuildModel gltfModel(gltf, cContext, true); + gltfModel.generate_object_model(info); } else { DebugHelper::error("(BuildObjectModel::build) Unsupported model type: \"", modelExtension, "\""); } diff --git a/tools/dkr_assets_tool_src/builder/buildTypes/buildTexture.cpp b/tools/dkr_assets_tool_src/builder/buildTypes/buildTexture.cpp index 04ab9ea5..356fe670 100644 --- a/tools/dkr_assets_tool_src/builder/buildTypes/buildTexture.cpp +++ b/tools/dkr_assets_tool_src/builder/buildTypes/buildTexture.cpp @@ -1,6 +1,7 @@ #include "buildTexture.h" -#include // for std::round +#include +#include #include "helpers/assetsHelper.h" #include "helpers/dataHelper.h" @@ -14,6 +15,7 @@ #include "builder/buildInfoCollection.h" + using namespace DkrAssetsTool; size_t calculate_out_size(BuildInfo &info, const JsonFile &jsonFile) { @@ -130,7 +132,7 @@ void BuildTexture::build(BuildInfo &info) { if(is3dTexture) { // Calculate sha1 of texture data, then add it to the texture cache. Need this so that we can reuse existing textures. std::string textureSha1 = DataHelper::make_sha1_hash_of_bytes(BytesView(info.out)); - textureCache.add_texture_to_sha1_cache(textureSha1, info.get_build_id()); + textureCache.add_texture_to_sha1_cache(textureSha1, info.get_file_index()); } bool debugKeepUncompressed = GlobalSettings::get_value("/debug/keep-uncompressed", false); @@ -162,44 +164,17 @@ void BuildTexture::build(BuildInfo &info) { } } -size_t BuildTexture::build_deferred(BuildInfo &baseInfo, fs::path pathToImage) { - std::string format, renderMode; - ImageHelper::guess_texture_format_and_render_mode(pathToImage, format, renderMode); +std::mutex buildDeferredTexMutex; + +size_t build_deferred_common(BuildInfo &baseInfo, std::vector &images, BuildTexture::DeferredTextureInfo &textureInfo) { + std::vector out(textureInfo.totalSize); - std::vector texturePaths = ImageHelper::get_multiple_textures_from_one(pathToImage); - DebugHelper::assert_(texturePaths.size() > 0, - "(BuildTexture::build_deferred) ImageHelper::get_multiple_textures_from_one() failed for ", pathToImage.filename()); - - size_t totalSize = 0; - - for(auto &path : texturePaths) { - int width, height; - ImageHelper::get_width_and_height(path, width, height); - totalSize += DataHelper::align16(sizeof(TextureHeader) + ImageHelper::image_size(width, height, format)); - } - - DebugHelper::assert_(totalSize > 0, - "(BuildTexture::build_deferred) Could not calculate texture size for ", pathToImage.filename()); - - int frameAdvanceDelay = 0; - - if(texturePaths.size() > 1) { - float animTexTiming = ImageHelper::guess_animated_texture_timing(pathToImage); - frameAdvanceDelay = std::round(animTexTiming * TEXTURE_FPS); - } - - std::string wrapS, wrapT; - ImageHelper::guess_texture_wrap_mode(pathToImage, wrapS, wrapT); - bool wrapSClamped = wrapS == "clamp"; - bool wrapTClamped = wrapT == "clamp"; - - std::vector out(totalSize); + bool wrapSClamped = textureInfo.wrapS == "clamp"; + bool wrapTClamped = textureInfo.wrapT == "clamp"; size_t offset = 0; - for(auto &path : texturePaths) { - N64Image img(path, format); - + for(N64Image &img : images) { int imgWidth = img.get_width(); int imgHeight = img.get_height(); @@ -207,18 +182,18 @@ size_t BuildTexture::build_deferred(BuildInfo &baseInfo, fs::path pathToImage) { offset += sizeof(TextureHeader); uint8_t *textureData = &out[offset]; - uint8_t formatVal = DataHelper::vector_index_of(TEXTURE_FORMAT_INT_TO_STRING, format); - uint8_t renderModeVal = DataHelper::vector_index_of(TEXTURE_RENDER_MODES, renderMode); + uint8_t formatVal = DataHelper::vector_index_of(TEXTURE_FORMAT_INT_TO_STRING, textureInfo.format); + uint8_t renderModeVal = DataHelper::vector_index_of(TEXTURE_RENDER_MODES, textureInfo.renderMode); header->width = imgWidth; header->height = imgHeight; header->format = (renderModeVal << 4) | formatVal; header->numberOfInstances = 1; - header->numOfTextures = texturePaths.size(); + header->numOfTextures = images.size(); header->textureSize = DataHelper::align16(img.size() + sizeof(TextureHeader)); if(header->numOfTextures > 1) { - header->frameAdvanceDelay = frameAdvanceDelay; + header->frameAdvanceDelay = textureInfo.frameAdvanceDelay; } int16_t flags = 0; @@ -231,7 +206,9 @@ size_t BuildTexture::build_deferred(BuildInfo &baseInfo, fs::path pathToImage) { flags |= (1 << 7); } - img.flip_vertically(); + if(textureInfo.shouldFlip) { + img.flip_vertically(); + } if(!DataHelper::is_power_of_two(imgWidth)) { flags |= (1 << 10); @@ -244,34 +221,109 @@ size_t BuildTexture::build_deferred(BuildInfo &baseInfo, fs::path pathToImage) { offset += img.size(); } - // Need to check the texture cache to see if the texture already exists or not. // Avoiding duplicate textures saves a lot of space in the ROM file. + BuildTextureCache &textureCache = baseInfo.get_texture_cache(); std::string textureSha1 = DataHelper::make_sha1_hash_of_bytes(BytesView(out)); - std::optional foundTexBuildId = textureCache.get_build_id_of_texture_hash(textureSha1); - if(foundTexBuildId.has_value()) { - DebugHelper::info_verbose("Found existing texture for ", pathToImage.filename(), ". Will use ID \"", foundTexBuildId.value(), "\"."); - return AssetsHelper::get_asset_index("ASSET_TEXTURES_3D", foundTexBuildId.value()); + std::optional foundTexIndex = textureCache.get_index_of_texture_hash(textureSha1); + if(foundTexIndex.has_value()) { + return foundTexIndex.value(); } - // Add the new texture to the cache. - std::string buildId = baseInfo.get_build_id() + "_" + pathToImage.stem().generic_string(); - DebugHelper::info_verbose("Created new texture: ", pathToImage.filename(), ", ID: ", buildId); - textureCache.add_texture_to_sha1_cache(textureSha1, buildId); - - BuildInfoCollection &collection = baseInfo.get_collection(); - - StringHelper::make_uppercase(buildId); - bool debugKeepUncompressed = GlobalSettings::get_value("/debug/keep-uncompressed", false); if(debugKeepUncompressed) { fs::path outUncompressedPath = GlobalSettings::get_decomp_path("build_subpath", "build/"); outUncompressedPath /= "debug/custom-textures"; - outUncompressedPath /= buildId + ".bin"; + outUncompressedPath /= textureInfo.buildId + ".bin"; FileHelper::write_binary_file(out, outUncompressedPath, true); } - return collection.add_deferred_build_info("ASSET_TEXTURES_3D", buildId, out, baseInfo.get_path_to_directory(), baseInfo.get_info_context()); + size_t outIndex = baseInfo.get_collection().add_deferred_build_info("ASSET_TEXTURES_3D", textureInfo.buildId, baseInfo.get_build_id(), baseInfo.get_section_build_id(), out, + baseInfo.get_path_to_directory(), baseInfo.get_info_context()); + + // Add the new texture to the cache. + textureCache.add_texture_to_sha1_cache(textureSha1, outIndex); + + return outIndex; +} + +size_t BuildTexture::build_deferred(BuildInfo &baseInfo, fs::path pathToImage, DeferredTextureInfo textureInfo) { + if(textureInfo.format.empty() || textureInfo.renderMode.empty()) { + ImageHelper::guess_texture_format_and_render_mode(pathToImage, textureInfo.format, textureInfo.renderMode); + } + + std::vector texturePaths = ImageHelper::get_multiple_textures_from_one(pathToImage); + + DebugHelper::assert_(texturePaths.size() > 0, + "(BuildTexture::build_deferred) ImageHelper::get_multiple_textures_from_one() failed for ", pathToImage.filename()); + + std::vector images; + images.reserve(texturePaths.size()); + + for(auto &path : texturePaths) { + int width, height; + ImageHelper::get_width_and_height(path, width, height); + textureInfo.totalSize += DataHelper::align16(sizeof(TextureHeader) + ImageHelper::image_size(width, height, textureInfo.format)); + images.emplace_back(path, textureInfo.format, true); + } + + DebugHelper::assert_(textureInfo.totalSize > 0, + "(BuildTexture::build_deferred) Could not calculate texture size for ", pathToImage.filename()); + + if(texturePaths.size() > 1) { + float animTexTiming = ImageHelper::guess_animated_texture_timing(pathToImage); + textureInfo.frameAdvanceDelay = std::round(animTexTiming * TEXTURE_FPS); + } + + if(textureInfo.wrapS.empty() || textureInfo.wrapT.empty()) { + ImageHelper::guess_texture_wrap_mode(pathToImage, textureInfo.wrapS, textureInfo.wrapT); + } + + textureInfo.buildId = baseInfo.get_build_id() + "_" + pathToImage.stem().generic_string(); + StringHelper::make_uppercase(textureInfo.buildId); + + return build_deferred_common(baseInfo, images, textureInfo); +} + +size_t BuildTexture::build_deferred(BuildInfo &baseInfo, uint8_t *imgData, size_t imgDataLength, std::string mimeType, + std::string imgName, DeferredTextureInfo textureInfo) { + DebugHelper::assert_(mimeType == "image/png", + "(BuildTexture::build_deferred) Only embedded .png files are supported! mimeType was \"", mimeType, "\""); + + int width, height; + void *data = pngdata2rgba(imgData, (int)imgDataLength, &width, &height); + + DebugHelper::assert_(data != NULL, + "(BuildTexture::build_deferred) Failed to load color PNG data from ", imgName); + + if(textureInfo.format.empty() || textureInfo.renderMode.empty()) { + ImageHelper::guess_texture_format_and_render_mode((rgba*)data, width, height, textureInfo.format, textureInfo.renderMode); + } + + bool isColourImage = (textureInfo.format == "RGBA16") || (textureInfo.format == "RGBA32") || (textureInfo.format == "CI4") || (textureInfo.format == "CI8"); + + if(!isColourImage) { + // Free colour data, then load it as grayscale data instead. + free(data); + data = pngdata2ia(imgData, (int)imgDataLength, &width, &height); + DebugHelper::assert_(data != NULL, + "(BuildTexture::build_deferred) Failed to load grayscale PNG data from ", imgName); + } + + // Embedded images are (currently) assumed to not be animated. + std::vector images; + images.reserve(1); + + textureInfo.totalSize = DataHelper::align16(sizeof(TextureHeader) + ImageHelper::image_size(width, height, textureInfo.format)); + + images.emplace_back(data, width, height, textureInfo.format, true); + + free(data); + + textureInfo.buildId = baseInfo.get_build_id() + "_" + imgName; + StringHelper::make_uppercase(textureInfo.buildId); + + return build_deferred_common(baseInfo, images, textureInfo); } diff --git a/tools/dkr_assets_tool_src/builder/buildTypes/buildTexture.h b/tools/dkr_assets_tool_src/builder/buildTypes/buildTexture.h index ea7c0b06..acc970cb 100644 --- a/tools/dkr_assets_tool_src/builder/buildTypes/buildTexture.h +++ b/tools/dkr_assets_tool_src/builder/buildTypes/buildTexture.h @@ -1,13 +1,29 @@ #pragma once +#include #include "builder/buildInfo.h" namespace DkrAssetsTool { namespace BuildTexture { + void build(BuildInfo &info); + + struct DeferredTextureInfo { + std::string buildId = ""; + std::string format = ""; + std::string renderMode = ""; + size_t totalSize = 0; + int frameAdvanceDelay = 0; + std::string wrapS = ""; + std::string wrapT = ""; + bool shouldFlip = false; + }; // Called from object/level model builders to add new textures. // Returns the fileIndex of the new texture asset. - size_t build_deferred(BuildInfo &baseInfo, fs::path pathToImage); + size_t build_deferred(BuildInfo &baseInfo, fs::path pathToImage, DeferredTextureInfo textureInfo = {}); + size_t build_deferred(BuildInfo &baseInfo, uint8_t *imgData, size_t imgDataLength, std::string mimeType, + std::string imgName, DeferredTextureInfo textureInfo = {}); + } } diff --git a/tools/dkr_assets_tool_src/builder/builder.cpp b/tools/dkr_assets_tool_src/builder/builder.cpp index 2547f86b..24b19e3a 100644 --- a/tools/dkr_assets_tool_src/builder/builder.cpp +++ b/tools/dkr_assets_tool_src/builder/builder.cpp @@ -212,7 +212,9 @@ void AssetBuilder::build_all(const fs::path &dstPath) { BuildInfoContext infoContext(cContext, textureCache, collection); fs::path includeFolder = GlobalSettings::get_decomp_path("include_subpath", "include/"); + fs::path srcFolder = GlobalSettings::get_decomp_path("src_subpath", "src/"); CEnumsHelper::load_enums_from_file(cContext, includeFolder / "enums.h"); + CEnumsHelper::load_enums_from_file(cContext, srcFolder / "textures_sprites.h"); // For RenderFlags enum. // Need to do this here for Object Maps to get built correctly. CEnumsHelper::load_enums_from_file(cContext, includeFolder / "object_behaviors.h"); @@ -250,13 +252,19 @@ void AssetBuilder::build_all(const fs::path &dstPath) { collection.add_build_info(sectionBuildId, fileBuildId, assetJson, assetJson.get_filepath().parent_path(), infoContext); } } - + // Build all the assets collection.run_builds([](BuildInfo &info) { - std::string type = info.get_type(); - builderMap[type](info); + if(DebugHelper::get().has_error_occured()) { + return; // Do NOT process any more if an error occured. + } + builderMap[info.get_type()](info); }, sectionIds); + if(DebugHelper::get().has_error_occured()) { + throw 1; // End the program if one of the threads had an error. + } + // Second loop to create the tables for the asset sections. for(size_t sectionIndex = 0; sectionIndex < numberOfAssetSections; sectionIndex++) { std::string sectionBuildId = sectionIds[sectionIndex]; @@ -295,12 +303,24 @@ void AssetBuilder::build_all(const fs::path &dstPath) { table.add_entry(info.out.size(), highestBitSet); } else { // ObjectAnimationIdsTable - const JsonFile &infoJson = info.get_src_json_file(); - std::string modelBuildId = infoJson.get_string("/for-model"); - int modelIndex = AssetsHelper::get_asset_index("ASSET_OBJECT_MODELS", modelBuildId); - DebugHelper::assert_(modelIndex >= 0, - "(AssetBuilder::build_all) Could not find object model \"", modelBuildId, "\" in ASSETS_OBJECT_MODELS section."); - table.add_object_animation_ids_entry(modelIndex); + if(info.is_deferred()) { + std::string deferredSectionId = info.get_deferred_from_section_build_id(); + if(deferredSectionId != "ASSET_OBJECT_MODELS") { + return; // Skip if not an object model. (Should never be the case?) + } + std::string modelBuildId = info.get_deferred_from_build_id(); + int modelIndex = AssetsHelper::get_asset_index("ASSET_OBJECT_MODELS", modelBuildId); + DebugHelper::assert_(modelIndex >= 0, + "(AssetBuilder::build_all) Could not find object model \"", modelBuildId, "\" in ASSETS_OBJECT_MODELS section."); + table.add_object_animation_ids_entry(modelIndex); + } else { + const JsonFile &infoJson = info.get_src_json_file(); + std::string modelBuildId = infoJson.get_string("/for-model"); + int modelIndex = AssetsHelper::get_asset_index("ASSET_OBJECT_MODELS", modelBuildId); + DebugHelper::assert_(modelIndex >= 0, + "(AssetBuilder::build_all) Could not find object model \"", modelBuildId, "\" in ASSETS_OBJECT_MODELS section."); + table.add_object_animation_ids_entry(modelIndex); + } } }); @@ -308,7 +328,7 @@ void AssetBuilder::build_all(const fs::path &dstPath) { const std::vector &tableData = tableView.data_vector(); - collection.add_deferred_build_info(sectionBuildId, sectionBuildId, tableData, "", infoContext); + collection.add_deferred_build_info(sectionBuildId, sectionBuildId, "", "", tableData, "", infoContext); } //collection.print_section_counts(); diff --git a/tools/dkr_assets_tool_src/builder/textureCache.cpp b/tools/dkr_assets_tool_src/builder/textureCache.cpp index 559d06ca..98b05059 100644 --- a/tools/dkr_assets_tool_src/builder/textureCache.cpp +++ b/tools/dkr_assets_tool_src/builder/textureCache.cpp @@ -4,23 +4,23 @@ using namespace DkrAssetsTool; -std::optional BuildTextureCache::get_build_id_of_texture_hash(std::string textureSha1) { +std::optional BuildTextureCache::get_index_of_texture_hash(std::string textureSha1) { _mutex.lock(); if(_textureCache.find(textureSha1) == _textureCache.end()) { _mutex.unlock(); return std::nullopt; } - std::string result = _textureCache.at(textureSha1); + size_t result = _textureCache.at(textureSha1); _mutex.unlock(); return result; } -void BuildTextureCache::add_texture_to_sha1_cache(std::string textureSha1, std::string buildId) { +void BuildTextureCache::add_texture_to_sha1_cache(std::string textureSha1, size_t index) { _mutex.lock(); if(_textureCache.find(textureSha1) != _textureCache.end()) { _mutex.unlock(); return; // If an entry already exists, then exit early to not overwrite it. } - _textureCache[textureSha1] = buildId; + _textureCache[textureSha1] = index; _mutex.unlock(); } diff --git a/tools/dkr_assets_tool_src/builder/textureCache.h b/tools/dkr_assets_tool_src/builder/textureCache.h index 56269d25..aae50ec7 100644 --- a/tools/dkr_assets_tool_src/builder/textureCache.h +++ b/tools/dkr_assets_tool_src/builder/textureCache.h @@ -10,14 +10,14 @@ namespace DkrAssetsTool { class BuildTextureCache { public: // Used in custom models to see if a texture already exists or not. Retruns std::nullopt if the texture does not exist. - std::optional get_build_id_of_texture_hash(std::string textureSha1); + std::optional get_index_of_texture_hash(std::string textureSha1); // Note: Will not overwrite a existing id if one already exists. - void add_texture_to_sha1_cache(std::string textureSha1, std::string buildId); + void add_texture_to_sha1_cache(std::string textureSha1, size_t index); private: - // Key: sha1 hash of a image, Value: Build id of that image. - std::unordered_map _textureCache; + // Key: sha1 hash of a image, Value: index of that texture in the ASSETS_TEXTURES_3D section. + std::unordered_map _textureCache; std::mutex _mutex; }; diff --git a/tools/dkr_assets_tool_src/helpers/c/cEnumsHelper.cpp b/tools/dkr_assets_tool_src/helpers/c/cEnumsHelper.cpp index acf7c513..7c8c1f22 100644 --- a/tools/dkr_assets_tool_src/helpers/c/cEnumsHelper.cpp +++ b/tools/dkr_assets_tool_src/helpers/c/cEnumsHelper.cpp @@ -53,9 +53,11 @@ int CEnum::_evaluate_enum(std::string enumValue) { int CEnum::_parse_enum_value(std::string &enumValue) { try { // Should work for most basic cases. - return std::stoi(enumValue, 0, 0); + return std::stoll(enumValue, 0, 0); } catch (const std::invalid_argument& ia) { // Not a simple conversion, continue. + } catch (const std::out_of_range& err) { + DebugHelper::error(err.what()); } return _evaluate_enum(enumValue); } @@ -276,6 +278,9 @@ void CEnumsHelper::get_enums_from_code(CContext &context, const std::string &cod for(RegexMatch &match : matches) { // Get the C code for this enum. std::string enumCode = match.get_group(0).get_text(); + + enumCode = StringHelper::remove_comments_from_c_code(enumCode); + // Output new CEnum out.push_back(new CEnum(context, enumCode)); } diff --git a/tools/dkr_assets_tool_src/helpers/c/cStructGltfHelper.cpp b/tools/dkr_assets_tool_src/helpers/c/cStructGltfHelper.cpp index 6257e1e5..068df236 100644 --- a/tools/dkr_assets_tool_src/helpers/c/cStructGltfHelper.cpp +++ b/tools/dkr_assets_tool_src/helpers/c/cStructGltfHelper.cpp @@ -56,7 +56,7 @@ int get_value_from_hint_enum(CStructEntry *structMember, std::string &enumValue) DebugHelper::assert_(hintEnum->get_value_of_member(enumValue, value), "(get_value_from_hint_enum) Enum ", enumName, - " does not have a value for the symbol ", enumValue); + " does not have a value for the symbol \"", enumValue, "\""); return value; } @@ -306,7 +306,7 @@ void CStructGltfHelper::put_struct_entry_into_gltf_node_extra(CStructEntry *stru } } -void CStructGltfHelper::put_gltf_node_extra_into_struct_entry(CStructEntry *structMember, GltfFileNode *objNode, uint8_t *bytes) { +void CStructGltfHelper::put_gltf_node_extra_into_struct_entry(CStructEntry *structMember, tinygltf::Node &objNode, uint8_t *bytes) { CStructEntry::InternalType internalType = structMember->get_internal_type(); StructHintType hintType = StructHintType::NO_HINT; @@ -314,23 +314,25 @@ void CStructGltfHelper::put_gltf_node_extra_into_struct_entry(CStructEntry *stru hintType = hintTypes[structMember->get_hint_type()]; } + std::string structMemberNamePtr = "/" + structMember->name; + switch(internalType) { case CStructEntry::InternalType::INTEGER: { int value = 0; switch(hintType) { case StructHintType::NO_HINT: - value = objNode->get_extra(structMember->name, 0); + value = GltfHelper::get_extra(objNode.extras, structMemberNamePtr, 0); break; case StructHintType::ENUM: { - std::string enumValue = objNode->get_extra(structMember->name, ""); + std::string enumValue = GltfHelper::get_extra(objNode.extras, structMemberNamePtr, ""); value = get_value_from_hint_enum(structMember, enumValue); break; } case StructHintType::ASSET_ID: { - std::string buildId = objNode->get_extra(structMember->name, ""); + std::string buildId = GltfHelper::get_extra(objNode.extras, structMemberNamePtr, ""); if(buildId.empty()) { value = -1; } else { @@ -340,19 +342,19 @@ void CStructGltfHelper::put_gltf_node_extra_into_struct_entry(CStructEntry *stru } case StructHintType::ANGLE: { - double angle = objNode->get_extra(structMember->name, 0.0); + double angle = GltfHelper::get_extra(objNode.extras, structMemberNamePtr, 0.0); value = get_value_from_hint_angle(structMember, angle); break; } case StructHintType::SCALE: { - double scale = objNode->get_extra(structMember->name, 0.0); + double scale = GltfHelper::get_extra(objNode.extras, structMemberNamePtr, 0.0); value = get_value_from_hint_scale(structMember, scale); break; } case StructHintType::OBJECT: { - std::string buildId = objNode->get_extra(structMember->name, ""); + std::string buildId = GltfHelper::get_extra(objNode.extras, structMemberNamePtr, ""); if(buildId.empty()) { value = -1; } else { @@ -362,7 +364,7 @@ void CStructGltfHelper::put_gltf_node_extra_into_struct_entry(CStructEntry *stru } case StructHintType::TIME: { - double timeSeconds = objNode->get_extra(structMember->name, -1.0); + double timeSeconds = GltfHelper::get_extra(objNode.extras, structMemberNamePtr, -1.0); if(timeSeconds == -1.0) { value = -1; break; @@ -379,7 +381,7 @@ void CStructGltfHelper::put_gltf_node_extra_into_struct_entry(CStructEntry *stru } case CStructEntry::InternalType::ARRAY_INTEGER: { - tinygltf::Value::Array arrayOfValues = objNode->get_extra(structMember->name, {}); + tinygltf::Value::Array arrayOfValues = GltfHelper::get_extra(objNode.extras, structMemberNamePtr, {}); std::vector values; for(tinygltf::Value &val : arrayOfValues) { switch(hintType) { diff --git a/tools/dkr_assets_tool_src/helpers/c/cStructGltfHelper.h b/tools/dkr_assets_tool_src/helpers/c/cStructGltfHelper.h index 43a68330..13621c02 100644 --- a/tools/dkr_assets_tool_src/helpers/c/cStructGltfHelper.h +++ b/tools/dkr_assets_tool_src/helpers/c/cStructGltfHelper.h @@ -11,6 +11,6 @@ namespace CStructGltfHelper { int gltfNode, uint8_t *bytes); // gltf node extra -> struct entry - void put_gltf_node_extra_into_struct_entry(CStructEntry *structMember, GltfFileNode *objNode, uint8_t *bytes); + void put_gltf_node_extra_into_struct_entry(CStructEntry *structMember, tinygltf::Node &objNode, uint8_t *bytes); } } \ No newline at end of file diff --git a/tools/dkr_assets_tool_src/helpers/debugHelper.cpp b/tools/dkr_assets_tool_src/helpers/debugHelper.cpp index 45b57ad8..57400c25 100644 --- a/tools/dkr_assets_tool_src/helpers/debugHelper.cpp +++ b/tools/dkr_assets_tool_src/helpers/debugHelper.cpp @@ -17,6 +17,18 @@ bool DebugHelper::using_asserts() { return _assertsEnabled; } +void DebugHelper::set_error_occured() { + _mutex.lock(); + _errorOccured = true; + _mutex.unlock(); +} +bool DebugHelper::has_error_occured() { + _mutex.lock(); + bool result = _errorOccured; + _mutex.unlock(); + return result; +} + DebugHelper::DebugTimer::DebugTimer() : m_beg(clock_::now()) { } diff --git a/tools/dkr_assets_tool_src/helpers/debugHelper.h b/tools/dkr_assets_tool_src/helpers/debugHelper.h index 1aace80b..d6e90dab 100644 --- a/tools/dkr_assets_tool_src/helpers/debugHelper.h +++ b/tools/dkr_assets_tool_src/helpers/debugHelper.h @@ -1,16 +1,34 @@ #pragma once +#if defined(__GNUG__) && defined(DEBUG) +// If you want to turn off the stack trace, then set this to 0. +#define INCLUDE_STACK_TRACE 1 +#else +#define INCLUDE_STACK_TRACE 0 +#endif + #include #include #include #include #include #include +#include #include +#include + +#if INCLUDE_STACK_TRACE +// Need to have libdwarf-dev installed if you want filenames & line numbers. +#if __has_include () +#define BACKWARD_HAS_DWARF 1 +#endif +#include "libs/backward.hpp" +#endif // Terminal colors #define RED_TEXT "\033[1;31m" #define GREEN_TEXT "\033[1;32m" +#define FOREST_GREEN_TEXT "\033[38;5;35m" #define YELLOW_TEXT "\033[1;33m" #define BLUE_TEXT "\033[1;34m" #define MAGENTA_TEXT "\033[1;35m" @@ -79,11 +97,16 @@ public: template static void error(Args... args) { + get().set_error_occured(); std::stringstream ss; ss << RED_TEXT << "ERROR: "; (ss << ... << args); ss << RESET_TEXT << std::endl; +#if INCLUDE_STACK_TRACE + _print_error_with_stack_trace(ss.str()); +#else std::cout << ss.str(); +#endif throw 1; } @@ -111,12 +134,12 @@ public: } template - static std::string vector_to_string(std::vector vec, int maxLength=-1) { + static std::string vector_to_string(std::vector vec, int startOffset = 0, int maxLength=-1) { std::stringstream ss; size_t len = (maxLength < 1) ? vec.size() : maxLength; ss << "["; - for(size_t i = 0; i < len; i++) { + for(size_t i = startOffset; i < len; i++) { ss << vec[i]; if(i < len - 1) { ss << ", "; @@ -127,6 +150,15 @@ public: return ss.str(); } + template + static void print_vector(std::vector vec, int startOffset = 0, int maxLength=-1) { + info(vector_to_string(vec, startOffset, maxLength)); + } + + static void sleep(float ms) { + std::this_thread::sleep_for(std::chrono::duration(ms)); + } + static std::string bytes_to_hex_string(const uint8_t *data, uint32_t dataLength) { std::stringstream ss; ss << std::hex << std::setfill('0'); @@ -141,13 +173,138 @@ public: return ss.str(); } + static std::thread::id get_current_thread_id() { + return std::this_thread::get_id(); + } + void set_verbose(bool isVerbose = true); bool is_verbose(); void set_asserts(bool useAsserts = true); bool using_asserts(); + void set_error_occured(); + bool has_error_occured(); + private: bool _verbose = false; bool _assertsEnabled = false; + bool _errorOccured = false; + std::mutex _mutex; int timerChannel = 0; + + +#if INCLUDE_STACK_TRACE + static bool _str_starts_with(const std::string &str, std::string substr) { + if(substr.length() > str.length()) { + return false; + } + for(size_t i = 0; i < substr.length(); i++) { + if(str[i] != substr[i]) { + return false; + } + } + return true; + } + + static void _print_stack_func(std::stringstream &ss, std::string functionSignature) { + // Don't need the "DkrAssetsTool::" part, since that applies to pretty much all of my code. + if(_str_starts_with(functionSignature, "DkrAssetsTool::")) { + functionSignature = functionSignature.substr(15); + } + + if(functionSignature.length() > 100) { + functionSignature = functionSignature.substr(0, 97) + "..."; + } + + ss << " "; + + size_t findOpen = functionSignature.find('('); + if(findOpen != std::string::npos) { + size_t start = findOpen; + while((functionSignature[start] != ':') && (start > 0)) start--; + char curChar = functionSignature[start]; + while(!((curChar >= 'A' && curChar <= 'Z') || (curChar >= 'a' && curChar <= 'z') || (curChar == '_'))) { + curChar = functionSignature[++start]; + } + + ss << functionSignature.substr(0, start); + ss << YELLOW_TEXT << functionSignature.substr(start, findOpen - start) << RESET_TEXT; + ss << functionSignature.substr(findOpen); + } else { + ss << functionSignature; + } + } + + static void _print_error_with_stack_trace(std::string error) { + using namespace backward; + StackTrace st; + st.load_here(20); + TraceResolver tr; + tr.load_stacktrace(st); + std::stringstream ss; + + ss << error; + ss << YELLOW_TEXT << "Thread: " << std::hex << DebugHelper::get_current_thread_id() << std::dec << RESET_TEXT << std::endl; + + int startOffset = 0; + for (size_t i = 0; i < st.size(); ++i) { + ResolvedTrace trace = tr.resolve(st[i]); + + if((trace.object_function.find("backward::") != std::string::npos) || + (trace.object_function.find("DkrAssetsTool::DebugHelper::") != std::string::npos)) { + // Skip the first few lines, since they include stack-trace calls. + startOffset++; + continue; + } + + bool hasSourceInfo = trace.source.line > 0; + + // Don't bother printing functions that come from system libraries. + bool isThirdParty = (hasSourceInfo) ? _str_starts_with(trace.source.filename, "/usr/include/") : false; + + if(isThirdParty) { + continue; + } + + bool printFuncSignature = (!trace.object_function.empty()) && (!isThirdParty); + + if(!printFuncSignature && !hasSourceInfo) { + continue; // Not useful, so skip. + } + + ss << BLUE_TEXT << "#" << std::setw(2) << std::setfill('0') << (i - startOffset) << RESET_TEXT << ":"; + if(printFuncSignature) { + _print_stack_func(ss, trace.object_function); + } else if (!trace.inliners.empty()) { + for(auto &inl : trace.inliners) { + ss << inl.function << std::endl; + } + } + if(hasSourceInfo) { + if(printFuncSignature) { + ss << std::endl; + } else if (!isThirdParty) { + _print_stack_func(ss, trace.source.function); + ss << std::endl; + } + + std::string sourceFilename = trace.source.filename; + size_t startFolderIndex = sourceFilename.find("dkr_assets_tool_src"); + if(startFolderIndex != std::string::npos) { + // Don't need the entire directory, just local to the src folder. + sourceFilename = sourceFilename.substr(startFolderIndex + 19); + } + if(sourceFilename.length() > 100) { + // Unlike the function name, the end of the filename string is more important than the beginning. + sourceFilename = "..." + sourceFilename.substr(sourceFilename.length() - 97); + } + + ss << " at " << FOREST_GREEN_TEXT << sourceFilename << RESET_TEXT << ":" << CYAN_TEXT << trace.source.line << RESET_TEXT; + } + ss << std::endl; + + } + std::cout << ss.str() << std::endl; + } +#endif }; } diff --git a/tools/dkr_assets_tool_src/helpers/gltf/gltfDkrTriangle.cpp b/tools/dkr_assets_tool_src/helpers/gltf/gltfDkrTriangle.cpp deleted file mode 100644 index b1f3b1e5..00000000 --- a/tools/dkr_assets_tool_src/helpers/gltf/gltfDkrTriangle.cpp +++ /dev/null @@ -1,42 +0,0 @@ -#include "gltfDkrTriangle.h" - -using namespace DkrAssetsTool; - -#include - -GltfDkrTriangle::GltfDkrTriangle(DkrTriangle &tri) { - vi0 = tri.vi0; - vi1 = tri.vi1; - vi2 = tri.vi2; -} - -GltfDkrTriangle::GltfDkrTriangle(uint16_t a, uint16_t b, uint16_t c) { - vi0 = a; - vi1 = b; - vi2 = c; -} - -// Determine if another triangle is the backface of this one. -bool GltfDkrTriangle::is_backface(const GltfDkrTriangle &other) { - return this->vi0 == other.vi0 && this->vi1 == other.vi2 && this->vi2 == other.vi1; -} - -// Returns number of bytes written. -int GltfDkrTriangle::write_indices(uint16_t *indices) { - indices[0] = vi0; - indices[1] = vi1; - indices[2] = vi2; - return get_size(); -} - -uint16_t GltfDkrTriangle::get_minimum_index() { - return std::min(vi0, std::min(vi1, vi2)); -} - -uint16_t GltfDkrTriangle::get_maximum_index() { - return std::max(vi0, std::max(vi1, vi2)); -} - -size_t GltfDkrTriangle::get_size() { - return sizeof(uint16_t) * 3; -} \ No newline at end of file diff --git a/tools/dkr_assets_tool_src/helpers/gltf/gltfDkrTriangle.h b/tools/dkr_assets_tool_src/helpers/gltf/gltfDkrTriangle.h deleted file mode 100644 index 9b098230..00000000 --- a/tools/dkr_assets_tool_src/helpers/gltf/gltfDkrTriangle.h +++ /dev/null @@ -1,26 +0,0 @@ -#pragma once - -#include "fileTypes/triangle.hpp" - -namespace DkrAssetsTool { - -class GltfDkrTriangle { -public: - uint16_t vi0, vi1, vi2; - - GltfDkrTriangle(DkrTriangle &tri); - GltfDkrTriangle(uint16_t a, uint16_t b, uint16_t c); - - // Determine if another triangle is the backface of this one. - bool is_backface(const GltfDkrTriangle &other); - - // Returns number of bytes written. - int write_indices(uint16_t *indices); - - uint16_t get_minimum_index(); - uint16_t get_maximum_index(); - - static size_t get_size(); -}; - -} diff --git a/tools/dkr_assets_tool_src/helpers/gltf/gltfDkrVertex.cpp b/tools/dkr_assets_tool_src/helpers/gltf/gltfDkrVertex.cpp deleted file mode 100644 index 6929d9eb..00000000 --- a/tools/dkr_assets_tool_src/helpers/gltf/gltfDkrVertex.cpp +++ /dev/null @@ -1,98 +0,0 @@ -#include "gltfDkrVertex.h" - -using namespace DkrAssetsTool; - -GltfDkrVertex::GltfDkrVertex(DkrVertex *vertex, DkrTexCoords *uv, int texWidth, int texHeight){ - x = (float)vertex->x * MODEL_SCALE_GLTF; - y = (float)vertex->y * MODEL_SCALE_GLTF; - z = (float)vertex->z * MODEL_SCALE_GLTF; - r = (float)vertex->r / 255.0f; - g = (float)vertex->g / 255.0f; - b = (float)vertex->b / 255.0f; - a = (float)vertex->a / 255.0f; - if (uv == nullptr || texWidth == 0 || texHeight == 0) { - u = 0; - v = 0; - return; - } - u = ((float)uv->u / 32.0f) / (float)texWidth; - v = ((float)uv->v / 32.0f) / (float)texHeight; -} - -GltfDkrVertex::GltfDkrVertex(int16_t xPos, int16_t yPos, int16_t zPos, - uint8_t red, uint8_t green, uint8_t blue, uint8_t alpha, - int16_t uOffset, int16_t vOffset, int texWidth, int texHeight) { - x = (float)xPos * MODEL_SCALE_GLTF; - y = (float)yPos * MODEL_SCALE_GLTF; - z = (float)zPos * MODEL_SCALE_GLTF; - r = (float)red / 255.0f; - g = (float)green / 255.0f; - b = (float)blue / 255.0f; - a = (float)alpha / 255.0f; - if(texWidth == 0 || texHeight == 0) { - u = 0; - v = 0; - return; - } - u = ((float)uOffset / 32.0f) / (float)texWidth; - v = ((float)vOffset / 32.0f) / (float)texHeight; -} - -GltfDkrVertex::GltfDkrVertex(float xPos, float yPos, float zPos, - float red, float green, float blue, float alpha, float uOffset, float vOffset) { - x = xPos * MODEL_SCALE_GLTF_INVERSE; - y = yPos * MODEL_SCALE_GLTF_INVERSE; - z = zPos * MODEL_SCALE_GLTF_INVERSE; - r = red; - g = green; - b = blue; - a = alpha; - u = uOffset; - v = vOffset; -} - -bool GltfDkrVertex::operator==(const GltfDkrVertex& other){ - return x == other.x && y == other.y && z == other.z && - r == other.r && g == other.g && b == other.b && a == other.a && - u == other.u && v == other.v; -} - -// Returns number of bytes written. -int GltfDkrVertex::write_position(float *positions) { - positions[0] = x; - positions[1] = y; - positions[2] = z; - return get_positions_size(); -} - -// Returns number of bytes written. -int GltfDkrVertex::write_color(float *colors) { - colors[0] = r; - colors[1] = g; - colors[2] = b; - colors[3] = a; - return get_colors_size(); -} - -// Returns number of bytes written. -int GltfDkrVertex::write_uv(float *uvs) { - uvs[0] = u; - uvs[1] = v; - return get_uv_size(); -} - -size_t GltfDkrVertex::get_positions_size() { - return sizeof(float) * 3; -} - -size_t GltfDkrVertex::get_colors_size() { - return sizeof(float) * 4; -} - -size_t GltfDkrVertex::get_uv_size() { - return sizeof(float) * 2; -} - -size_t GltfDkrVertex::get_size() { - return get_positions_size() + get_colors_size() + get_uv_size(); -} diff --git a/tools/dkr_assets_tool_src/helpers/gltf/gltfDkrVertex.h b/tools/dkr_assets_tool_src/helpers/gltf/gltfDkrVertex.h deleted file mode 100644 index 96d6b05b..00000000 --- a/tools/dkr_assets_tool_src/helpers/gltf/gltfDkrVertex.h +++ /dev/null @@ -1,43 +0,0 @@ -#pragma once - -#include "fileTypes/vertex.hpp" -#include "fileTypes/texCoords.hpp" - -namespace DkrAssetsTool { - -// TODO: Move this into a singleton class. -#define MODEL_SCALE_GLTF (1.0f / 1000.0f) -#define MODEL_SCALE_GLTF_INVERSE (1.0f / MODEL_SCALE_GLTF) - -// Class used for converting DKR's vertex information to the glTF format. -class GltfDkrVertex { -public: - float x, y, z; - float r, g, b, a; - float u, v; - - GltfDkrVertex(DkrVertex *vertex, DkrTexCoords *uv=nullptr, int texWidth=0, int texHeight=0); - GltfDkrVertex(int16_t xPos, int16_t yPos, int16_t zPos, - uint8_t red = 255, uint8_t green = 255, uint8_t blue = 255, uint8_t alpha = 255, - int16_t uOffset = 0, int16_t vOffset = 0, int texWidth = 0, int texHeight = 0); - GltfDkrVertex(float xPos, float yPos, float zPos, - float red = 1.0f, float green = 1.0f, float blue = 1.0f, float alpha = 1.0f, - float uOffset = 0, float vOffset = 0); - - bool operator==(const GltfDkrVertex& other); - bool equals(DkrVertex *vertex, DkrTexCoords *uv=nullptr, int texWidth=0, int texHeight=0); - - int write_position(float *positions); // Returns number of bytes written. - int write_color(float *colors); // Returns number of bytes written. - int write_uv(float *uvs); // Returns number of bytes written. - - static size_t get_positions_size(); - - static size_t get_colors_size(); - - static size_t get_uv_size(); - - static size_t get_size(); -}; - -} \ No newline at end of file diff --git a/tools/dkr_assets_tool_src/helpers/gltf/gltfFile.cpp b/tools/dkr_assets_tool_src/helpers/gltf/gltfFile.cpp new file mode 100644 index 00000000..576ccf51 --- /dev/null +++ b/tools/dkr_assets_tool_src/helpers/gltf/gltfFile.cpp @@ -0,0 +1,345 @@ +#include "gltfHelper.h" + +#include "helpers/stringHelper.h" + +#include + +// Defines for the tinygltf library. +#define TINYGLTF_IMPLEMENTATION +#define TINYGLTF_USE_CPP14 +#include "libs/tiny_gltf.h" + +using namespace DkrAssetsTool; + +GltfFile::GltfFile(const fs::path &filepath) : _filepath(filepath) { + std::string err; + std::string warn; + + bool loadResult; + + if(filepath.extension() == ".gltf") { + loadResult = _tinygltf.LoadASCIIFromFile(&_model, &err, &warn, filepath); + } else { + // .glb + loadResult = _tinygltf.LoadBinaryFromFile(&_model, &err, &warn, filepath); + } + + if(!warn.empty()) { + DebugHelper::warn(warn); + } + + if(!err.empty()) { + DebugHelper::error_no_throw(err); + } + + DebugHelper::assert_(loadResult, "(GltfFile::GltfFile) Failed to load GLTF file ", filepath); + + // Model has successfully loaded. +} + +GltfFile::~GltfFile() { +} + +fs::path GltfFile::get_filepath() { + return _filepath; +} + +bool GltfFile::empty() { + return (_model.scenes.size() == 0) || (_model.nodes.size() == 0); +} + +std::vector GltfFile::get_root_node_indices(std::optional> blacklist) { + std::vector indices; + + for(auto &scene : _model.scenes) { + indices.insert(indices.end(), scene.nodes.begin(), scene.nodes.end()); + } + + if(blacklist.has_value()) { + std::vector &blist = blacklist.value(); + + for(size_t i = 0; i < indices.size(); i++) { + std::string nodeName = StringHelper::simplify(_model.nodes[indices[i]].name); + bool removeThis = false; + for(const std::string &blistEntry : blist) { + if(StringHelper::starts_with(nodeName, blistEntry)) { + removeThis = true; + break; + } + } + if(removeThis) { + indices.erase(indices.begin() + i); + i--; + } + } + } + + return indices; +} + +std::optional GltfFile::get_root_node_from_name(std::string lookForName) { + lookForName = StringHelper::simplify(lookForName); + + std::vector childrenToSearch; + + for(auto &scene : _model.scenes) { + for(auto nodeIndex : scene.nodes) { + std::string nodeName = StringHelper::simplify(_model.nodes[nodeIndex].name); + if(StringHelper::starts_with(nodeName, lookForName)) { + return nodeIndex; + } + if(_model.nodes[nodeIndex].mesh == -1) { + for(int childIndex : _model.nodes[nodeIndex].children) { + childrenToSearch.emplace_back(childIndex); + } + } + } + } + + for(int childIndex : childrenToSearch) { + std::string nodeName = StringHelper::simplify(_model.nodes[childIndex].name); + if(StringHelper::starts_with(nodeName, lookForName)) { + return childIndex; + } + } + + return std::nullopt; // No node found. +} + +size_t GltfFile::get_node_count() { + return _model.nodes.size(); +} + +template +inline std::optional> try_get_element(int index, std::vector &elementArray) { + if((index < 0) || (index >= (int)elementArray.size())) { + return std::nullopt; + } + + return elementArray[index]; +} + +std::optional> GltfFile::try_get_node(int index) { + return try_get_element(index, _model.nodes); +} + +std::optional> GltfFile::try_get_mesh(int index) { + return try_get_element(index, _model.meshes); +} + +std::optional> GltfFile::try_get_buffer(int index) { + return try_get_element(index, _model.buffers); +} + +std::optional> GltfFile::try_get_bufferview(int index) { + return try_get_element(index, _model.bufferViews); +} + +std::optional> GltfFile::try_get_accessor(int index) { + return try_get_element(index, _model.accessors); +} + +std::optional> GltfFile::try_get_material(int index) { + return try_get_element(index, _model.materials); +} + +std::optional> GltfFile::try_get_image(int index) { + return try_get_element(index, _model.images); +} + +std::optional> GltfFile::try_get_sampler(int index) { + return try_get_element(index, _model.samplers); +} + +std::optional> GltfFile::try_get_texture(int index) { + return try_get_element(index, _model.textures); +} + +std::optional> GltfFile::try_get_animation(int index) { + return try_get_element(index, _model.animations); +} + +template +inline T &get_element(int index, const std::function>(int)> &try_get_func, const char* name) { + auto tryGetElement = try_get_func(index); + DebugHelper::assert_(tryGetElement.has_value(), + "(GltfFile::get_element) Failed to get ", name, " at index ", index); + return tryGetElement.value(); +} + +#define GET_ELEMENT_LAMBDA(funcName) [this](int i) { return this->funcName(i); } + +tinygltf::Node &GltfFile::get_node(int index) { + return get_element(index, GET_ELEMENT_LAMBDA(try_get_node), "node"); +} + +tinygltf::Mesh &GltfFile::get_mesh(int index) { + return get_element(index, GET_ELEMENT_LAMBDA(try_get_mesh), "mesh"); +} + +tinygltf::Buffer &GltfFile::get_buffer(int index) { + return get_element(index, GET_ELEMENT_LAMBDA(try_get_buffer), "buffer"); +} + +tinygltf::BufferView &GltfFile::get_bufferview(int index) { + return get_element(index, GET_ELEMENT_LAMBDA(try_get_bufferview), "buffer view"); +} + +tinygltf::Accessor &GltfFile::get_accessor(int index) { + return get_element(index, GET_ELEMENT_LAMBDA(try_get_accessor), "accessor"); +} + +tinygltf::Material &GltfFile::get_material(int index) { + return get_element(index, GET_ELEMENT_LAMBDA(try_get_material), "material"); +} + +tinygltf::Image &GltfFile::get_image(int index) { + return get_element(index, GET_ELEMENT_LAMBDA(try_get_image), "image"); +} + +tinygltf::Sampler &GltfFile::get_sampler(int index) { + return get_element(index, GET_ELEMENT_LAMBDA(try_get_sampler), "sampler"); +} + +tinygltf::Texture &GltfFile::get_texture(int index) { + return get_element(index, GET_ELEMENT_LAMBDA(try_get_texture), "texture"); +} + +tinygltf::Animation &GltfFile::get_animation(int index) { + return get_element(index, GET_ELEMENT_LAMBDA(try_get_animation), "animation"); +} + +size_t GltfFile::number_of_nodes() const { + return _model.nodes.size(); +} + +size_t GltfFile::number_of_mesh() const { + return _model.meshes.size(); +} + +size_t GltfFile::number_of_buffer() const { + return _model.buffers.size(); +} + +size_t GltfFile::number_of_bufferview() const { + return _model.bufferViews.size(); +} + +size_t GltfFile::number_of_accessors() const { + return _model.accessors.size(); +} + +size_t GltfFile::number_of_materials() const { + return _model.materials.size(); +} + +size_t GltfFile::number_of_images() const { + return _model.images.size(); +} + +size_t GltfFile::number_of_samplers() const { + return _model.samplers.size(); +} + +size_t GltfFile::number_of_textures() const { + return _model.textures.size(); +} + +size_t GltfFile::number_of_animations() const { + return _model.animations.size(); +} + + +bool GltfFile::search_for_node_by_name(const std::string nodeName, int &outIndex) { + size_t numNodes = get_node_count(); + for(size_t i = 0; i < numNodes; i++) { + if(_model.nodes[i].name == nodeName) { + outIndex = i; + return true; + } + } + // No node with the name was found. + outIndex = -1; + return false; +} + +/***************************************************************************/ + +WriteableGltfFile::WriteableGltfFile(std::string rootName) { + tinygltf::Node rootNode; + tinygltf::Scene scene; + tinygltf::Asset asset; + + rootNode.name = rootName; + + // Root node is node 0. + _model.nodes.push_back(rootNode); + + // Scene should just have the root node. + scene.nodes.push_back(0); + + _model.scenes.push_back(scene); +} + +WriteableGltfFile::~WriteableGltfFile() { + +} + +int WriteableGltfFile::new_node(const std::string nodeName, int parentNode) { + tinygltf::Node newNode; + + int thisIndex = (int)_model.nodes.size(); + + newNode.name = nodeName; + + _model.nodes[parentNode].children.push_back(thisIndex); + + _model.nodes.push_back(newNode); + + return thisIndex; +} + +void WriteableGltfFile::set_node_position(int node, double x, double y, double z) { + if(_model.nodes[node].translation.size() == 0) { + _model.nodes[node].translation.resize(3); + } + _model.nodes[node].translation[0] = x; + _model.nodes[node].translation[1] = y; + _model.nodes[node].translation[2] = z; +} + +void WriteableGltfFile::set_node_scale(int node, double x, double y, double z) { + if(_model.nodes[node].scale.size() == 0) { + _model.nodes[node].scale.resize(3); + } + _model.nodes[node].scale[0] = x; + _model.nodes[node].scale[1] = y; + _model.nodes[node].scale[2] = z; +} + +void WriteableGltfFile::set_node_rotation(int node, double x, double y, double z, double w) { + if(_model.nodes[node].rotation.size() == 0) { + _model.nodes[node].rotation.resize(4); + } + _model.nodes[node].rotation[0] = x; + _model.nodes[node].rotation[1] = y; + _model.nodes[node].rotation[2] = z; + _model.nodes[node].rotation[3] = w; +} + +void WriteableGltfFile::set_node_rotation_y(int node, double y) { + if(_model.nodes[node].rotation.size() == 0) { + _model.nodes[node].rotation.resize(4); + _model.nodes[node].rotation[3] = 1.0; + } + _model.nodes[node].rotation[1] = y; +} + +void WriteableGltfFile::save(const std::string filepath) { + FileHelper::write_folder_if_it_does_not_exist(filepath); + _tinygltf.WriteGltfSceneToFile(&_model, filepath, + true, // embedImages + true, // embedBuffers + true, // pretty print + false); // write binary +} diff --git a/tools/dkr_assets_tool_src/helpers/gltf/gltfFile.h b/tools/dkr_assets_tool_src/helpers/gltf/gltfFile.h new file mode 100644 index 00000000..d5961f78 --- /dev/null +++ b/tools/dkr_assets_tool_src/helpers/gltf/gltfFile.h @@ -0,0 +1,109 @@ +#pragma once + +#include "helpers/debugHelper.h" +#include "helpers/fileHelper.h" +#include "helpers/mathHelper.h" +#include +#include +#include +#include "libs/tiny_gltf.h" + +namespace DkrAssetsTool { + +// Read-Only .gltf file +class GltfFile { +public: + GltfFile(const fs::path &filepath); + ~GltfFile(); + + fs::path get_filepath(); + + // Returns true if there are no root nodes. + bool empty(); + + // Returns a list of indices to root nodes from each scene in the model. + // A blacklist is useful for ignore certain sections (Meta data like collision spheres, attach points, etc.) + std::vector get_root_node_indices(std::optional> blacklist = std::nullopt); + + std::optional get_root_node_from_name(std::string lookForName); + + size_t get_node_count(); + + // Returns std::nullopt if element does not exist. + std::optional> try_get_node(int index); + std::optional> try_get_mesh(int index); + std::optional> try_get_buffer(int index); + std::optional> try_get_bufferview(int index); + std::optional> try_get_accessor(int index); + std::optional> try_get_material(int index); + std::optional> try_get_image(int index); + std::optional> try_get_sampler(int index); + std::optional> try_get_texture(int index); + std::optional> try_get_animation(int index); + + tinygltf::Node &get_node(int index); + tinygltf::Mesh &get_mesh(int index); + tinygltf::Buffer &get_buffer(int index); + tinygltf::BufferView &get_bufferview(int index); + tinygltf::Accessor &get_accessor(int index); + tinygltf::Material &get_material(int index); + tinygltf::Image &get_image(int index); + tinygltf::Sampler &get_sampler(int index); + tinygltf::Texture &get_texture(int index); + tinygltf::Animation &get_animation(int index); + + size_t number_of_nodes() const; + size_t number_of_mesh() const; + size_t number_of_buffer() const; + size_t number_of_bufferview() const; + size_t number_of_accessors() const; + size_t number_of_materials() const; + size_t number_of_images() const; + size_t number_of_samplers() const; + size_t number_of_textures() const; + size_t number_of_animations() const; + + bool search_for_node_by_name(const std::string nodeName, int &outIndex); + +private: + tinygltf::Model _model; + tinygltf::TinyGLTF _tinygltf; + fs::path _filepath; +}; + +// Write-Only .gltf file +class WriteableGltfFile { +public: + WriteableGltfFile(std::string rootNode); + ~WriteableGltfFile(); + + // Node stuff + static const int ROOT_NODE = 0; + + // returns index of the new node. + int new_node(const std::string nodeName, int parentNode = ROOT_NODE); + + void set_node_position(int node, double x, double y, double z); + void set_node_scale(int node, double x, double y, double z); + void set_node_rotation(int node, double x, double y, double z, double w); + void set_node_rotation_y(int node, double y); + + template + void set_node_extra(int node, std::string key, T value) { + if((int)_model.nodes[node].extras.Type() == 0) { + tinygltf::Value::Object initObj; + _model.nodes[node].extras = tinygltf::Value(initObj); + } + + tinygltf::Value::Object &obj = _model.nodes[node].extras.Get(); + + obj[key] = tinygltf::Value(value); + }; + + void save(const std::string filepath); + +private: + tinygltf::Model _model; + tinygltf::TinyGLTF _tinygltf; +}; +} \ No newline at end of file diff --git a/tools/dkr_assets_tool_src/helpers/gltf/gltfHelper.cpp b/tools/dkr_assets_tool_src/helpers/gltf/gltfHelper.cpp index 555a958a..948cbf0b 100644 --- a/tools/dkr_assets_tool_src/helpers/gltf/gltfHelper.cpp +++ b/tools/dkr_assets_tool_src/helpers/gltf/gltfHelper.cpp @@ -1,170 +1,659 @@ #include "gltfHelper.h" +#include "helpers/debugHelper.h" +#include "helpers/stringHelper.h" +#include +#include + using namespace DkrAssetsTool; -// Defines for the tinygltf library. -#define TINYGLTF_IMPLEMENTATION -#define TINYGLTF_USE_CPP14 -#include "libs/tiny_gltf.h" +// The "normalized" functions are taken directly from the glTF 2.0 spec: +/* ++------------------------+----------------------------+------------------------+ +| accessor.componentType | int-to-float | float-to-int | ++------------------------+----------------------------+------------------------+ +| signed byte | f = max(c / 127.0, -1.0) | c = round(f * 127.0) | +| unsigned byte | f = c / 255.0 | c = round(f * 255.0) | +| signed short | f = max(c / 32767.0, -1.0) | c = round(f * 32767.0) | +| unsigned short | f = c / 65535.0 | c = round(f * 65535.0) | ++------------------------+----------------------------+------------------------+ +*/ -GltfFileNode::GltfFileNode(GltfFile *file, tinygltf::Node *node) : _file(file), _node(node) { +inline float GltfHelper::normalized_byte_to_float(int8_t value) { + return std::fmax(((float)value) / 127.0f, -1.0); } -GltfFileNode::~GltfFileNode(){ +inline float GltfHelper::normalized_ubyte_to_float(uint8_t value) { + return ((float)value) / 255.0f; } -size_t GltfFileNode::get_child_count() { - return _node->children.size(); +inline float GltfHelper::normalized_short_to_float(int16_t value) { + return std::fmax(((float)value) / 32767.0f, -1.0); } -GltfFileNode *GltfFileNode::get_child_node_by_index(int nodeIndex) { - return _file->get_node(_node->children[nodeIndex]); +inline float GltfHelper::normalized_ushort_to_float(uint16_t value) { + return ((float)value) / 65535.0f; } -void GltfFileNode::get_position(double &outX, double &outY, double &outZ) { - outX = _node->translation[0]; - outY = _node->translation[1]; - outZ = _node->translation[2]; +inline int8_t GltfHelper::float_to_normalized_byte(float value) { + return std::roundf(value * 127.0f); } -std::string GltfFileNode::get_name() { - return _node->name; +inline uint8_t GltfHelper::float_to_normalized_ubyte(float value) { + return std::roundf(value * 255.0f); } - -GltfFile::GltfFile(const fs::path &filepath) { - std::string err; - std::string warn; - bool loadResult = _tinygltf.LoadASCIIFromFile(&_model, &err, &warn, filepath); - - if(!warn.empty()) { - DebugHelper::warn(warn); + +inline int16_t GltfHelper::float_to_normalized_short(float value) { + return std::roundf(value * 32767.0f); +} + +inline uint16_t GltfHelper::float_to_normalized_ushort(float value) { + return std::roundf(value * 65535.0f); +} + +inline uint8_t GltfHelper::normalized_byte_to_normalized_ubyte(int8_t value) { + // byte -> float -> ubyte + return float_to_normalized_ubyte(normalized_byte_to_float(value)); +} + +inline uint8_t GltfHelper::normalized_short_to_normalized_ubyte(int16_t value) { + // short -> float -> ubyte + return float_to_normalized_ubyte(normalized_short_to_float(value)); +} + +inline uint8_t GltfHelper::normalized_ushort_to_normalized_ubyte(uint16_t value) { + // ushort -> float -> ubyte + return float_to_normalized_ubyte(normalized_ushort_to_float(value)); +} + +/***************************************************************************************************************/ + +Vec3f GltfHelper::get_node_position(tinygltf::Node &node) { + if(!node.matrix.empty()) { + return Vec3f(node.matrix[3], node.matrix[7], node.matrix[11]); } - if(!err.empty()) { - DebugHelper::error_no_throw(err); + if(node.translation.empty()) { + return Vec3f(); } - DebugHelper::assert_(loadResult, "(GltfFile::GltfFile) Failed to load GLTF file ", filepath); - - // Model has successfully loaded. + return Vec3f(node.translation[0], node.translation[1], node.translation[2]); } -GltfFile::~GltfFile() { - // Free node cache memory. - for (auto &pair : _nodeCache) { - if(pair.second == nullptr) { +/***************************************************************************************************************/ + +size_t GltfHelper::get_accessor_count(GltfFile& gltf, int accessorIndex) { + if (accessorIndex < 0 || accessorIndex >= gltf.number_of_accessors()) { + return 0; + } + + tinygltf::Accessor &accessor = gltf.get_accessor(accessorIndex); + return accessor.count; +} + +std::optional> GltfHelper::get_indices_from_primitive(GltfFile& gltf, tinygltf::Primitive& gltfPrimitive) { + int accessorIndex = gltfPrimitive.indices; + if (accessorIndex < 0) { + return std::nullopt; + } + + tinygltf::Accessor &gltfAccessor = gltf.get_accessor(accessorIndex); + + size_t numberOfIndices = gltfAccessor.count; + + switch (gltfAccessor.componentType) { + case TINYGLTF_COMPONENT_TYPE_UNSIGNED_BYTE: + { + return get_data_from_accessor(gltf, accessorIndex); + } + break; + case TINYGLTF_COMPONENT_TYPE_UNSIGNED_SHORT: + { + std::vector gltfData = get_data_from_accessor(gltf, accessorIndex); + std::vector out(numberOfIndices); + for (size_t i = 0; i < numberOfIndices; i++) { + DebugHelper::assert_(gltfData[i] <= 255, + "(GltfHelper::get_indices_from_primitive) uint32_t index is greater than 255: ", gltfData[i]); + out[i] = static_cast(gltfData[i]); + } + return out; + } + break; + case TINYGLTF_COMPONENT_TYPE_UNSIGNED_INT: + { + std::vector gltfData = get_data_from_accessor(gltf, accessorIndex); + std::vector out(numberOfIndices); + for (size_t i = 0; i < numberOfIndices; i++) { + DebugHelper::assert_(gltfData[i] <= 255, + "(GltfHelper::get_indices_from_primitive) uint32_t index is greater than 255: ", gltfData[i]); + out[i] = static_cast(gltfData[i]); + } + return out; + } + break; + default: + DebugHelper::error("(GltfHelper::get_indices_from_primitive) Invalid component type for indices: ", gltfAccessor.componentType); + break; + } + + return std::nullopt; +} + +std::optional> GltfHelper::get_positions_from_primitive(GltfFile& gltf, tinygltf::Primitive& gltfPrimitive) { + int accessorIndex = gltfPrimitive.attributes["POSITION"]; + if (accessorIndex < 0) { + return std::nullopt; + } + return get_data_from_accessor(gltf, accessorIndex); +} + +// Returns RGBA unsigned byte array of colors. +std::optional> GltfHelper::get_colors_from_primitive(GltfFile& gltf, tinygltf::Primitive& gltfPrimitive) { + int accessorIndex = -1; + + // First check if the primitive has a color attribute + if (gltfPrimitive.attributes.find("COLOR_0") != gltfPrimitive.attributes.end()) { + accessorIndex = gltfPrimitive.attributes["COLOR_0"]; + } + + if (accessorIndex < 0) { + // No color accessor found? return null. + return std::nullopt; + } + + tinygltf::Accessor &gltfAccessor = gltf.get_accessor(accessorIndex); + + size_t numberOfColors = gltfAccessor.count; + + // SPEC: Colors can be VEC3(RGB) or VEC4(RGBA) with floats, normalized ubytes, or normalized ushorts. + // Alpha is always 1.0 (255) for the VEC3 type + int type = gltfAccessor.type; + + bool isVec3 = (type == TINYGLTF_TYPE_VEC3), isVec4 = (type == TINYGLTF_TYPE_VEC4); + DebugHelper::assert_(isVec3 || isVec4, + "(GltfHelper::get_colors_from_primitive) Invalid accessor type for colors: ", type); + + int dataStride = (isVec4) ? 4 : 3; + + switch (gltfAccessor.componentType) { + case TINYGLTF_COMPONENT_TYPE_UNSIGNED_BYTE: + { + if (isVec4) { + // Can be gotten directly. + return get_data_from_accessor(gltf, accessorIndex); + } else { + std::vector gltfData = get_data_from_accessor(gltf, accessorIndex); + std::vector out(numberOfColors * 4); + for (size_t i = 0; i < numberOfColors; i++) { + out[i * 4 + 0] = gltfData[i * dataStride + 0]; + out[i * 4 + 1] = gltfData[i * dataStride + 1]; + out[i * 4 + 2] = gltfData[i * dataStride + 2]; + out[i * 4 + 3] = 255; + } + return out; + } + } + case TINYGLTF_COMPONENT_TYPE_UNSIGNED_SHORT: + { + // Downconverting from 16-bit color to 8-bit color. + std::vector gltfData = get_data_from_accessor(gltf, accessorIndex); + std::vector out(numberOfColors * 4); + for (size_t i = 0; i < numberOfColors; i++) { + out[i * 4 + 0] = gltfData[i * dataStride + 0] >> 8; + out[i * 4 + 1] = gltfData[i * dataStride + 1] >> 8; + out[i * 4 + 2] = gltfData[i * dataStride + 2] >> 8; + out[i * 4 + 3] = (isVec4) ? gltfData[i * dataStride + 3] >> 8 : 255; + } + return out; + } + case TINYGLTF_COMPONENT_TYPE_FLOAT: + { + std::vector gltfData = get_data_from_accessor(gltf, accessorIndex); + std::vector out(numberOfColors * 4); + for (size_t i = 0; i < numberOfColors; i++) { + out[i * 4 + 0] = float_to_normalized_ubyte(gltfData[i * dataStride + 0]); + out[i * 4 + 1] = float_to_normalized_ubyte(gltfData[i * dataStride + 1]); + out[i * 4 + 2] = float_to_normalized_ubyte(gltfData[i * dataStride + 2]); + out[i * 4 + 3] = (isVec4) ? float_to_normalized_ubyte(gltfData[i * dataStride + 3]) : 255; + } + return out; + } + default: + DebugHelper::error("(GltfHelper::get_colors_from_primitive) Invalid accessor component type for colors: ", gltfAccessor.componentType); + break; + } + + return std::nullopt; +} + +std::optional> GltfHelper::get_uvs_from_primitive(GltfFile& gltf, tinygltf::Primitive& gltfPrimitive) { + int accessorIndex = -1; + if (gltfPrimitive.attributes.find("TEXCOORD_0") != gltfPrimitive.attributes.end()) { + accessorIndex = gltfPrimitive.attributes["TEXCOORD_0"]; + } + + if (accessorIndex < 0) { + return std::nullopt; + } + + tinygltf::Accessor &gltfAccessor = gltf.get_accessor(accessorIndex); + + size_t numberOfUVs = gltfAccessor.count; + + // SPEC: texture coordinates use VEC2 floats, normalized bytes/ubytes, or normalized shorts/ushorts + + switch (gltfAccessor.componentType) { + case TINYGLTF_COMPONENT_TYPE_BYTE: + { + std::vector gltfData = get_data_from_accessor(gltf, accessorIndex); + std::vector out(numberOfUVs * 2); + for (size_t i = 0; i < numberOfUVs; i++) { + out[i * 2 + 0] = normalized_byte_to_float(gltfData[i * 2 + 0]); + out[i * 2 + 1] = normalized_byte_to_float(gltfData[i * 2 + 1]); + } + return out; + } + case TINYGLTF_COMPONENT_TYPE_UNSIGNED_BYTE: + { + std::vector gltfData = get_data_from_accessor(gltf, accessorIndex); + std::vector out(numberOfUVs * 2); + for (size_t i = 0; i < numberOfUVs; i++) { + out[i * 2 + 0] = normalized_ubyte_to_float(gltfData[i * 2 + 0]); + out[i * 2 + 1] = normalized_ubyte_to_float(gltfData[i * 2 + 1]); + } + return out; + } + case TINYGLTF_COMPONENT_TYPE_SHORT: + { + std::vector gltfData = get_data_from_accessor(gltf, accessorIndex); + std::vector out(numberOfUVs * 2); + for (size_t i = 0; i < numberOfUVs; i++) { + out[i * 2 + 0] = normalized_short_to_float(gltfData[i * 2 + 0]); + out[i * 2 + 1] = normalized_short_to_float(gltfData[i * 2 + 1]); + } + return out; + } + case TINYGLTF_COMPONENT_TYPE_UNSIGNED_SHORT: + { + std::vector gltfData = get_data_from_accessor(gltf, accessorIndex); + std::vector out(numberOfUVs * 2); + for (size_t i = 0; i < numberOfUVs; i++) { + out[i * 2 + 0] = normalized_ushort_to_float(gltfData[i * 2 + 0]); + out[i * 2 + 1] = normalized_ushort_to_float(gltfData[i * 2 + 1]); + } + return out; + } + case TINYGLTF_COMPONENT_TYPE_FLOAT: + { + // Can be gotten directly. + return get_data_from_accessor(gltf, accessorIndex); + } + default: + DebugHelper::error("(GltfHelper::get_colors_from_primitive) Invalid accessor component type for uvs: ", gltfAccessor.componentType); + break; + } + + return std::nullopt; +} + +std::optional> GltfHelper::get_morph_targets_from_primitive(GltfFile& gltf, tinygltf::Primitive& gltfPrimitive) { + if(gltfPrimitive.targets.empty()) { + return std::nullopt; + } + + // POSITION is required to exist according to the gltf spec. + int positionsAccessorIndex = gltfPrimitive.attributes["POSITION"]; + size_t numberOfPositions = get_accessor_count(gltf, positionsAccessorIndex) * 3; + + size_t numberOfTargetsWithPositions = 0; + + for(auto &targetMap : gltfPrimitive.targets) { + if(targetMap.find("POSITION") == targetMap.end()) { continue; } - delete pair.second; + numberOfTargetsWithPositions++; } -} - -size_t GltfFile::get_node_count() { - return _model.nodes.size(); -} - -GltfFileNode *GltfFile::get_node(int nodeIndex) { - DebugHelper::assert_(nodeIndex < (int)_model.nodes.size(), - "(GltfFile::get_node) Node index ", nodeIndex, " is out of range!"); - - tinygltf::Node *tinygltfNode = &_model.nodes[nodeIndex]; - if(_nodeCache.find(tinygltfNode) == _nodeCache.end()) { - _nodeCache[tinygltfNode] = new GltfFileNode(this, tinygltfNode); + if(numberOfTargetsWithPositions == 0) { + return std::nullopt; } + + std::vector out(numberOfPositions * numberOfTargetsWithPositions); + + size_t currentTargetWithPositions = 0; + for(auto &targetMap : gltfPrimitive.targets) { + if(targetMap.find("POSITION") == targetMap.end()) { + continue; + } - return _nodeCache[tinygltfNode]; + int targetAccessor = targetMap.at("POSITION"); + + if((targetAccessor < 0)) { + continue; + } + + tinygltf::Accessor &gltfAccessor = gltf.get_accessor(targetAccessor); + + DebugHelper::assert_(gltfAccessor.componentType == TINYGLTF_COMPONENT_TYPE_FLOAT, + "(GltfHelper::get_morph_targets_from_primitive) Morph target positions can only use floats!"); + DebugHelper::assert_(gltfAccessor.type == TINYGLTF_TYPE_VEC3, + "(GltfHelper::get_morph_targets_from_primitive) Morph target positions can only be VEC3!"); + + std::vector targetValues = get_data_from_accessor(gltf, targetAccessor); + + std::copy(targetValues.begin(), targetValues.end(), out.begin() + (currentTargetWithPositions * numberOfPositions)); + + currentTargetWithPositions++; + } + + return out; } +// Weights can be in multiple formats, not just floats, so this is necessary. +std::optional> GltfHelper::get_weights_from_accessor(GltfFile& gltf, int accessorIndex) { + tinygltf::Accessor &gltfAccessor = gltf.get_accessor(accessorIndex); -bool GltfFile::search_for_node_by_name(const std::string nodeName, int &outIndex) { - size_t numNodes = get_node_count(); - for(size_t i = 0; i < numNodes; i++) { - if(_model.nodes[i].name == nodeName) { - outIndex = i; - return true; + size_t numberOfWeights = gltfAccessor.count; + + int compType = gltfAccessor.componentType; + + switch (compType) { + case TINYGLTF_COMPONENT_TYPE_BYTE: + { + std::vector gltfData = get_data_from_accessor(gltf, accessorIndex); + std::vector out(numberOfWeights); + for (size_t i = 0; i < numberOfWeights; i++) { + out[i] = normalized_byte_to_float(gltfData[i]); + } + return out; + } + case TINYGLTF_COMPONENT_TYPE_UNSIGNED_BYTE: + { + std::vector gltfData = get_data_from_accessor(gltf, accessorIndex); + std::vector out(numberOfWeights); + for (size_t i = 0; i < numberOfWeights; i++) { + out[i] = normalized_ubyte_to_float(gltfData[i]); + } + return out; + } + case TINYGLTF_COMPONENT_TYPE_SHORT: + { + std::vector gltfData = get_data_from_accessor(gltf, accessorIndex); + std::vector out(numberOfWeights); + for (size_t i = 0; i < numberOfWeights; i++) { + out[i] = normalized_short_to_float(gltfData[i]); + } + return out; + } + case TINYGLTF_COMPONENT_TYPE_UNSIGNED_SHORT: + { + std::vector gltfData = get_data_from_accessor(gltf, accessorIndex); + std::vector out(numberOfWeights); + for (size_t i = 0; i < numberOfWeights; i++) { + out[i] = normalized_ushort_to_float(gltfData[i]); + } + return out; + } + case TINYGLTF_COMPONENT_TYPE_FLOAT: + { + // Can be gotten directly. + return get_data_from_accessor(gltf, accessorIndex); + } + default: + DebugHelper::error("(GltfHelper::get_weights_from_accessor) Invalid accessor component type for weights: ", gltfAccessor.componentType); + break; + } + + return std::nullopt; +} + +/***************************************************************************************************************/ + +// Gotta do the json pointer logic myself, because tinygltf doesn't support it. + +static inline std::string get_next_section_from_json_pointer(const std::string& jsonPointer) { + size_t indexOfNextSlash = jsonPointer.find('/', 1); + if (indexOfNextSlash == jsonPointer.npos) { + return "/"; + } + return jsonPointer.substr(indexOfNextSlash); +} + +static inline std::string get_current_section_name(const std::string& jsonPointer) { + size_t indexOfNextSlash = jsonPointer.find('/', 1); + if (indexOfNextSlash == jsonPointer.npos) { + return jsonPointer.substr(1); + } + return jsonPointer.substr(1, indexOfNextSlash - 1); +} + +static tinygltf::Value& get_value_using_json_pointer(tinygltf::Value& parentSection, bool parentIsArray, std::string jsonPointer, bool createUndefinedSections) { + DebugHelper::assert_(jsonPointer[0] == '/', + "(GltfHelper::get_value_using_json_pointer) Invalid json pointer \"", jsonPointer, "\", must start with forward slash."); + + std::string currentSectionName = get_current_section_name(jsonPointer); + bool isANumber = StringHelper::is_a_number(currentSectionName); + std::string childJsonPointer = get_next_section_from_json_pointer(jsonPointer); + std::string childSectionName = get_current_section_name(childJsonPointer); + bool childIsANumber = StringHelper::is_a_number(childSectionName); + bool isLastSection = childJsonPointer == "/"; + + if (parentIsArray) { + DebugHelper::assert_(isANumber, "(GltfHelper::set_extension_value) Invalid number: ", currentSectionName); + tinygltf::Value::Array& parentArray = parentSection.Get(); + int currentIndex = std::stoi(currentSectionName); + if (parentArray.size() <= currentIndex) { + parentArray.resize(currentIndex + 1); + } + if (parentArray[currentIndex].Type() == tinygltf::NULL_TYPE) { + if (!createUndefinedSections) { + throw 1; + } + if (childIsANumber) { + int childIndex = std::stoi(childSectionName); + parentArray[currentIndex] = tinygltf::Value(tinygltf::Value::Array(childIndex + 1)); + } else { + parentArray[currentIndex] = tinygltf::Value(tinygltf::Value::Object()); + } + } + if (isLastSection) { + return parentArray[currentIndex]; + } else { + return get_value_using_json_pointer(parentArray[currentIndex], childIsANumber, childJsonPointer, createUndefinedSections); + } + } else { + DebugHelper::assert_(parentSection.IsObject(), + "(GltfHelper::get_value_using_json_pointer) Parent is not an object, it is type: ", (int)parentSection.Type()); + tinygltf::Value::Object& parentObject = parentSection.Get(); + if (!parentSection.Has(currentSectionName)) { + if (!createUndefinedSections) { + throw 1; + } + if (childIsANumber) { + int childIndex = std::stoi(childSectionName); + parentObject[currentSectionName] = tinygltf::Value(tinygltf::Value::Array(childIndex + 1)); + } else { + parentObject[currentSectionName] = tinygltf::Value(tinygltf::Value::Object()); + } + } + if (isLastSection) { + return parentObject[currentSectionName]; + } else { + return get_value_using_json_pointer(parentObject[currentSectionName], childIsANumber, childJsonPointer, createUndefinedSections); } } - // No node with the name was found. - outIndex = -1; - return false; } -WriteableGltfFile::WriteableGltfFile(std::string rootName) { - tinygltf::Node rootNode; - tinygltf::Scene scene; - tinygltf::Asset asset; - - rootNode.name = rootName; - - // Root node is node 0. - _model.nodes.push_back(rootNode); - - // Scene should just have the root node. - scene.nodes.push_back(0); - - _model.scenes.push_back(scene); -} +/***************************************************************************************************************/ -WriteableGltfFile::~WriteableGltfFile() { - -} - -int WriteableGltfFile::new_node(const std::string nodeName, int parentNode) { - tinygltf::Node newNode; - - int thisIndex = (int)_model.nodes.size(); - - newNode.name = nodeName; - - _model.nodes[parentNode].children.push_back(thisIndex); - - _model.nodes.push_back(newNode); - - return thisIndex; -} - -void WriteableGltfFile::set_node_position(int node, double x, double y, double z) { - if(_model.nodes[node].translation.size() == 0) { - _model.nodes[node].translation.resize(3); +Mat4x4f GltfHelper::calculate_node_matrix(tinygltf::Node &node, const Mat4x4f &parentMatrix) { + if(node.matrix.size() > 0) { + return parentMatrix * Mat4x4f(node.matrix); } - _model.nodes[node].translation[0] = x; - _model.nodes[node].translation[1] = y; - _model.nodes[node].translation[2] = z; -} - -void WriteableGltfFile::set_node_scale(int node, double x, double y, double z) { - if(_model.nodes[node].scale.size() == 0) { - _model.nodes[node].scale.resize(3); + + if(node.translation.empty() && node.rotation.empty() && node.scale.empty()) { + return parentMatrix; } - _model.nodes[node].scale[0] = x; - _model.nodes[node].scale[1] = y; - _model.nodes[node].scale[2] = z; + + Vec3f translation = (node.translation.empty()) ? Vec3f() : Vec3f(node.translation); + Vec4f rotation = (node.rotation.empty()) ? Vec4f() : Vec4f(node.rotation); + Vec3f scale = (node.scale.empty()) ? Vec3f(1, 1, 1) : Vec3f(node.scale); + + return parentMatrix * Mat4x4f(translation, rotation, scale); } -void WriteableGltfFile::set_node_rotation(int node, double x, double y, double z, double w) { - if(_model.nodes[node].rotation.size() == 0) { - _model.nodes[node].rotation.resize(4); +/***************************************************************************************************************/ + +template +void GltfHelper::set_extra(tinygltf::Value& extras, std::string jsonPointer, T value) { + if (!extras.IsObject()) { + extras = tinygltf::Value(tinygltf::Value::Object()); } - _model.nodes[node].rotation[0] = x; - _model.nodes[node].rotation[1] = y; - _model.nodes[node].rotation[2] = z; - _model.nodes[node].rotation[3] = w; + + tinygltf::Value& extrasValueToSet = get_value_using_json_pointer(extras, false, jsonPointer, true); + + extrasValueToSet = tinygltf::Value(value); } -void WriteableGltfFile::set_node_rotation_y(int node, double y) { - if(_model.nodes[node].rotation.size() == 0) { - _model.nodes[node].rotation.resize(4); - _model.nodes[node].rotation[3] = 1.0; +template void GltfHelper::set_extra(tinygltf::Value& extras, std::string jsonPointer, std::string value); +template void GltfHelper::set_extra(tinygltf::Value& extras, std::string jsonPointer, int value); +template void GltfHelper::set_extra(tinygltf::Value& extras, std::string jsonPointer, double value); +template void GltfHelper::set_extra(tinygltf::Value& extras, std::string jsonPointer, bool value); +template void GltfHelper::set_extra(tinygltf::Value& extras, std::string jsonPointer, tinygltf::Value::Array& value); + +template +void GltfHelper::set_extra(tinygltf::Value& extras, std::string jsonPointer, std::vector& values) { + if (!extras.IsObject()) { + extras = tinygltf::Value(tinygltf::Value::Object()); } - _model.nodes[node].rotation[1] = y; + + tinygltf::Value& extrasValueToSet = get_value_using_json_pointer(extras, false, jsonPointer, true); + + tinygltf::Value::Array tinygltfValues; + + for (T& value : values) { + tinygltfValues.push_back(tinygltf::Value(value)); + } + + extrasValueToSet = tinygltf::Value(tinygltfValues); } -void WriteableGltfFile::save(const std::string filepath) { - FileHelper::write_folder_if_it_does_not_exist(filepath); - _tinygltf.WriteGltfSceneToFile(&_model, filepath, - true, // embedImages - true, // embedBuffers - true, // pretty print - false); // write binary +template void GltfHelper::set_extra(tinygltf::Value& extras, std::string jsonPointer, std::vector& values); +template void GltfHelper::set_extra(tinygltf::Value& extras, std::string jsonPointer, std::vector& values); +template void GltfHelper::set_extra(tinygltf::Value& extras, std::string jsonPointer, std::vector& values); + +template +T GltfHelper::get_extra(tinygltf::Value& extras, std::string jsonPointer, T defaultValue) { + if (!extras.IsObject()) { + // This node does not have any extras, just return the default value. + return defaultValue; + } + + try { + tinygltf::Value& extrasValue = get_value_using_json_pointer(extras, false, jsonPointer, false); + return extrasValue.Get(); + } catch (int e) { + return defaultValue; + } } + +template std::string GltfHelper::get_extra(tinygltf::Value& extras, std::string jsonPointer, std::string defaultValue); +template int GltfHelper::get_extra(tinygltf::Value& extras, std::string jsonPointer, int defaultValue); +template double GltfHelper::get_extra(tinygltf::Value& extras, std::string jsonPointer, double defaultValue); +template bool GltfHelper::get_extra(tinygltf::Value& extras, std::string jsonPointer, bool defaultValue); +template tinygltf::Value::Array GltfHelper::get_extra(tinygltf::Value& extras, std::string jsonPointer, tinygltf::Value::Array defaultValue); +template tinygltf::Value::Object GltfHelper::get_extra(tinygltf::Value& extras, std::string jsonPointer, tinygltf::Value::Object defaultValue); + +bool GltfHelper::has_extra(tinygltf::Value& extras, std::string jsonPointer) { + if (!extras.IsObject()) { + return false; // Not an object, so it definitely doesn't have the key. + } + try { + tinygltf::Value& extrasValue = get_value_using_json_pointer(extras, false, jsonPointer, false); + return extrasValue.Type() != tinygltf::NULL_TYPE; + } catch (int e) { + return false; + } +} + +int GltfHelper::get_extra_type(tinygltf::Value& extras, std::string jsonPointer) { + if (!extras.IsObject()) { + return tinygltf::NULL_TYPE; // Not an object, so it definitely doesn't have the key. + } + try { + tinygltf::Value& extrasValue = get_value_using_json_pointer(extras, false, jsonPointer, false); + return extrasValue.Type(); + } catch (int e) { + return tinygltf::NULL_TYPE; + } +} + +std::optional> GltfHelper::get_extra_keys(tinygltf::Value& extras, std::string jsonPointer) { + try { + tinygltf::Value& extrasValue = get_value_using_json_pointer(extras, false, jsonPointer, false); + if (extrasValue.Type() != tinygltf::OBJECT_TYPE) { + return std::nullopt; // Return null if the type is not an object. + } + if (extrasValue.Size() < 1) { + return std::nullopt; // Return null if the object is empty. + } + return extrasValue.Keys(); + } catch (int e) { + return std::nullopt; // Return null if the extra value doesn't exist. + } +} + +template +void GltfHelper::get_extra_array(tinygltf::Value& extras, std::string jsonPointer, std::vector& outValues) { + outValues.clear(); + + if (!extras.IsObject()) { + // This node does not have any extras. + return; + } + + try { + tinygltf::Value& extrasValue = get_value_using_json_pointer(extras, false, jsonPointer, false); + + if (!extrasValue.IsArray()) { + DebugHelper::error("(GltfHelper::get_extra) \"", jsonPointer, "\" should be an array, but it is not!"); + return; + } + + + tinygltf::Value::Array tinygltfValues = extrasValue.Get(); + + for (tinygltf::Value& arrVal : tinygltfValues) { + outValues.push_back(arrVal.Get()); + } + } catch (int e) { + } +} + +template void GltfHelper::get_extra_array(tinygltf::Value& extras, std::string jsonPointer, std::vector& outValues); +template void GltfHelper::get_extra_array(tinygltf::Value& extras, std::string jsonPointer, std::vector& outValues); +template void GltfHelper::get_extra_array(tinygltf::Value& extras, std::string jsonPointer, std::vector& outValues); +template void GltfHelper::get_extra_array(tinygltf::Value& extras, std::string jsonPointer, std::vector& outValues); +template void GltfHelper::get_extra_array(tinygltf::Value& extras, std::string jsonPointer, std::vector& outValues); +template void GltfHelper::get_extra_array(tinygltf::Value& extras, std::string jsonPointer, std::vector& outValues); + +template +void GltfHelper::set_extension_value(tinygltf::ExtensionMap& extensions, std::string jsonPointer, T value) { + DebugHelper::assert_(!jsonPointer.empty(), "(GltfHelper::set_extension_value) jsonPointer was empty!"); + + std::string sectionName = get_current_section_name(jsonPointer); + + if (extensions.find(sectionName) == extensions.end()) { + extensions[sectionName] = tinygltf::Value(tinygltf::Value::Object()); + } + + tinygltf::Value &extensionValue = get_value_using_json_pointer(extensions[sectionName], false, + get_next_section_from_json_pointer(jsonPointer), true); + + extensionValue = tinygltf::Value(value); +} + +template void GltfHelper::set_extension_value(tinygltf::ExtensionMap& extensions, std::string jsonPointer, std::string value); +template void GltfHelper::set_extension_value(tinygltf::ExtensionMap& extensions, std::string jsonPointer, int value); +template void GltfHelper::set_extension_value(tinygltf::ExtensionMap& extensions, std::string jsonPointer, double value); +template void GltfHelper::set_extension_value(tinygltf::ExtensionMap& extensions, std::string jsonPointer, bool value); diff --git a/tools/dkr_assets_tool_src/helpers/gltf/gltfHelper.h b/tools/dkr_assets_tool_src/helpers/gltf/gltfHelper.h index a1d73357..62f07198 100644 --- a/tools/dkr_assets_tool_src/helpers/gltf/gltfHelper.h +++ b/tools/dkr_assets_tool_src/helpers/gltf/gltfHelper.h @@ -1,103 +1,139 @@ #pragma once -#include "helpers/debugHelper.h" -#include "helpers/fileHelper.h" -#include -#include "libs/tiny_gltf.h" +#include +#include + +#include "gltfFile.h" + +#include "helpers/mathHelper.h" +#include "libs/bytes_view.hpp" namespace DkrAssetsTool { -class GltfFile; +namespace GltfHelper { + // These functions are simple enough that they can be inlined. + inline float normalized_byte_to_float(int8_t value); + inline float normalized_ubyte_to_float(uint8_t value); + inline float normalized_short_to_float(int16_t value); + inline float normalized_ushort_to_float(uint16_t value); + inline int8_t float_to_normalized_byte(float value); + inline uint8_t float_to_normalized_ubyte(float value); + inline int16_t float_to_normalized_short(float value); + inline uint16_t float_to_normalized_ushort(float value); + inline uint8_t normalized_byte_to_normalized_ubyte(int8_t value); + inline uint8_t normalized_short_to_normalized_ubyte(int16_t value); + inline uint8_t normalized_ushort_to_normalized_ubyte(uint16_t value); + + Vec3f get_node_position(tinygltf::Node &objNode); -// Represents a node in the gltf file. -class GltfFileNode { -public: - GltfFileNode(GltfFile *file, tinygltf::Node *node); - ~GltfFileNode(); - - size_t get_child_count(); - GltfFileNode *get_child_node_by_index(int nodeIndex); - - void get_position(double &outX, double &outY, double &outZ); - std::string get_name(); - - template - T get_extra(std::string key, T defaultValue) { - if(!_node->extras.IsObject()) { - // This node does not have any extras, just return the default value. - return defaultValue; - } - - if(!_node->extras.Has(key)) { - // Return default value if the key is not in the map. - return defaultValue; - } - - tinygltf::Value val = _node->extras.Get(key); - - return val.Get(); + template + T* get_buffer_pointer(tinygltf::BufferView& bufferView, tinygltf::Buffer& buffer, size_t offset) { + return reinterpret_cast(&buffer.data[bufferView.byteOffset + offset]); } -private: - GltfFile *_file; - tinygltf::Node *_node; -}; - -// Read-Only .gltf file -class GltfFile { -public: - GltfFile(const fs::path &filepath); - ~GltfFile(); - size_t get_node_count(); - GltfFileNode *get_node(int nodeIndex); - bool search_for_node_by_name(const std::string nodeName, int &outIndex); + template + std::vector get_data_from_bufferview(GltfFile& gltf, int bufferViewIndex) { + tinygltf::BufferView &bufferView = gltf.get_bufferview(bufferViewIndex); + tinygltf::Buffer &buffer = gltf.get_buffer(bufferView.buffer); + + T *start = reinterpret_cast(&buffer.data[bufferView.byteOffset]); + T *end = reinterpret_cast(&buffer.data[bufferView.byteOffset + bufferView.byteLength]); + + return std::vector(start, end); + } -private: - tinygltf::Model _model; - tinygltf::TinyGLTF _tinygltf; - std::unordered_map _nodeCache; + // Returns 0 if the accessorIndex was invalid. + size_t get_accessor_count(GltfFile& gltf, int accessorIndex); -}; - -// Write-Only .gltf file -class WriteableGltfFile { -public: - WriteableGltfFile(std::string rootNode); - ~WriteableGltfFile(); + template + static void _write_modified_sparse_data(std::vector &dst, const std::vector &src, GltfFile& gltf, tinygltf::Accessor &accessor) { + size_t elementStride = tinygltf::GetNumComponentsInType(accessor.type); + std::vector indices = get_data_from_bufferview(gltf, accessor.sparse.indices.bufferView); + for(size_t i = 0; i < accessor.sparse.count; i++) { + auto srcStart = src.begin() + (i * elementStride), srcEnd = srcStart + elementStride; + auto dstStart = dst.begin() + (indices[i] * elementStride); + std::copy(srcStart, srcEnd, dstStart); + } + } - // Node stuff - static const int ROOT_NODE = 0; - - // returns index of the new node. - int new_node(const std::string nodeName, int parentNode = ROOT_NODE); - - void set_node_position(int node, double x, double y, double z); - void set_node_scale(int node, double x, double y, double z); - void set_node_rotation(int node, double x, double y, double z, double w); - void set_node_rotation_y(int node, double y); - - template - void set_node_extra(int node, std::string key, T value) { - if((int)_model.nodes[node].extras.Type() == 0) { - tinygltf::Value::Object initObj; - _model.nodes[node].extras = tinygltf::Value(initObj); + template + static void modify_data_using_sparse_accessor(std::vector &data, GltfFile& gltf, tinygltf::Accessor &accessor) { + if(!accessor.sparse.isSparse) { + return; // If the accessor doesn't have any sparse data, then just exit. } - tinygltf::Value::Object &obj = _model.nodes[node].extras.Get(); + std::vector values = get_data_from_bufferview(gltf, accessor.sparse.values.bufferView); + + switch(accessor.sparse.indices.componentType) { + case TINYGLTF_COMPONENT_TYPE_UNSIGNED_BYTE: + _write_modified_sparse_data(data, values, gltf, accessor); + break; + case TINYGLTF_COMPONENT_TYPE_UNSIGNED_SHORT: + _write_modified_sparse_data(data, values, gltf, accessor); + break; + case TINYGLTF_COMPONENT_TYPE_UNSIGNED_INT: + _write_modified_sparse_data(data, values, gltf, accessor); + break; + default: + DebugHelper::error("(GltfHelper::modify_data_using_sparse_accessor) Invalid component type for indices: ", + accessor.sparse.indices.componentType); + break; + } + } + + // Cannot return a reference, because the accessor could have no bufferView or be modified from a sparse accessor. + template + static std::vector get_data_from_accessor(GltfFile& gltf, int accessorIndex) { + tinygltf::Accessor &accessor = gltf.get_accessor(accessorIndex); + + if(accessor.bufferView == -1) { + std::vector out(accessor.count * tinygltf::GetNumComponentsInType(accessor.type)); // Empty array filled with zeroes. + modify_data_using_sparse_accessor(out, gltf, accessor); + return out; + } + + std::vector out = get_data_from_bufferview(gltf, accessor.bufferView); + modify_data_using_sparse_accessor(out, gltf, accessor); + return out; + } - obj[key] = tinygltf::Value(value); - }; + std::optional> get_indices_from_primitive(GltfFile& gltf, tinygltf::Primitive& gltfPrimitive); + std::optional> get_positions_from_primitive(GltfFile& gltf, tinygltf::Primitive& gltfPrimitive); + std::optional> get_colors_from_primitive(GltfFile& gltf, tinygltf::Primitive& gltfPrimitive); + std::optional> get_uvs_from_primitive(GltfFile& gltf, tinygltf::Primitive& gltfPrimitive); - void save(const std::string filepath); + std::optional> get_morph_targets_from_primitive(GltfFile& gltf, tinygltf::Primitive& gltfPrimitive); -private: - tinygltf::Model _model; - tinygltf::TinyGLTF _tinygltf; -}; + // Weights can be in multiple formats, not just floats, so this is necessary. + std::optional> get_weights_from_accessor(GltfFile& gltf, int accessorIndex); + + // Generic version for non-standard gltf data. + template + std::vector get_data_from_accessor(GltfFile& gltf, int accessorIndex); + + Mat4x4f calculate_node_matrix(tinygltf::Node &node, const Mat4x4f &parentMatrix); -class GltfHelper { -public: - + template + void set_extra(tinygltf::Value& extras, std::string jsonPointer, T value); + + template + void set_extra(tinygltf::Value& extras, std::string jsonPointer, std::vector& values); + + template + T get_extra(tinygltf::Value& extras, std::string jsonPointer, T defaultValue); + + bool has_extra(tinygltf::Value& extras, std::string jsonPointer); + + int get_extra_type(tinygltf::Value& extras, std::string jsonPointer); + + template + void get_extra_array(tinygltf::Value& extras, std::string jsonPointer, std::vector& outValues); + + // Returns nullopt if jsonPointer doesn't exist, is the wrong type, or the object has no keys. + std::optional> get_extra_keys(tinygltf::Value& extras, std::string jsonPointer); + + template + void set_extension_value(tinygltf::ExtensionMap &extensions, std::string jsonPointer, T value); }; } \ No newline at end of file diff --git a/tools/dkr_assets_tool_src/helpers/gltf/gltfPrimitive.cpp b/tools/dkr_assets_tool_src/helpers/gltf/gltfPrimitive.cpp deleted file mode 100644 index 1a1f30c2..00000000 --- a/tools/dkr_assets_tool_src/helpers/gltf/gltfPrimitive.cpp +++ /dev/null @@ -1,48 +0,0 @@ -#include "gltfPrimitive.h" - -using namespace DkrAssetsTool; - -GltfPrimitive::GltfPrimitive(DkrVertex *vertices, DkrTriangle *triangles, size_t numTris, TextureHeader *texture, int materialIndex) { - int texW = 0; - int texH = 0; - if(texture != nullptr) { - texW = texture->width; - texH = texture->height; - } - - _materialIndex = materialIndex; - - for(size_t i = 0; i < numTris; i++) { - DkrTriangle *tri = &triangles[i]; - - int index0 = _get_or_set_index(&vertices[tri->vi0], &tri->uv0, texW, texH); - int index1 = _get_or_set_index(&vertices[tri->vi1], &tri->uv1, texW, texH); - int index2 = _get_or_set_index(&vertices[tri->vi2], &tri->uv2, texW, texH); - - _triangles.push_back(new GltfDkrTriangle(index0, index1, index2)); - - if(tri->flags & 0x40) { // Draw backface - _triangles.push_back(new GltfDkrTriangle(index0, index2, index1)); - } - } -} - -GltfPrimitive::~GltfPrimitive() { - // _triangles and _vertices should be deleted automatically from std::vector's deconstructor. -} - -int GltfPrimitive::_get_or_set_index(DkrVertex *vertex, DkrTexCoords *uv, int texWidth, int texHeight) { - GltfDkrVertex *newVertex = new GltfDkrVertex(vertex, uv, texWidth, texHeight); - int i = 0; - while(i < (int)_vertices.size()) { - if(newVertex == _vertices[i]) { - // Found vertex. - delete newVertex; // Was not needed. - return i; - } - i++; - } - _vertices.push_back(newVertex); - return i; -} - diff --git a/tools/dkr_assets_tool_src/helpers/gltf/gltfPrimitive.h b/tools/dkr_assets_tool_src/helpers/gltf/gltfPrimitive.h deleted file mode 100644 index a85cbc2b..00000000 --- a/tools/dkr_assets_tool_src/helpers/gltf/gltfPrimitive.h +++ /dev/null @@ -1,24 +0,0 @@ -#pragma once - - -#include - -#include "gltfDkrVertex.h" -#include "gltfDkrTriangle.h" -#include "fileTypes/texture.hpp" - -namespace DkrAssetsTool { - -class GltfPrimitive { -public: - GltfPrimitive(DkrVertex *vertices, DkrTriangle *triangles, size_t numTris, TextureHeader *texture, int materialIndex); - ~GltfPrimitive(); -private: - std::vector _triangles; - std::vector _vertices; - int _materialIndex; - - int _get_or_set_index(DkrVertex *vertex, DkrTexCoords *uv, int texWidth, int texHeight); -}; - -} \ No newline at end of file diff --git a/tools/dkr_assets_tool_src/helpers/imageHelper.cpp b/tools/dkr_assets_tool_src/helpers/imageHelper.cpp index 54c4b19e..2df778f1 100644 --- a/tools/dkr_assets_tool_src/helpers/imageHelper.cpp +++ b/tools/dkr_assets_tool_src/helpers/imageHelper.cpp @@ -19,19 +19,64 @@ using namespace DkrAssetsTool; N64Image::N64Image(fs::path filepath, std::string format, bool alignData) : _format(format) { bool isColourImage = (format == "RGBA16") || (format == "RGBA32") || (format == "CI4") || (format == "CI8"); - DebugHelper::assert(StringHelper::ends_with(filepath, ".png"), + DebugHelper::assert_(StringHelper::ends_with(filepath, ".png"), "Only .png images are supported! Invalid image path: ", filepath); void *imgData; // Can be either rgba* or ia* n64graphics types. + int width, height; // Load image from a .png file, getting the width and height as well. if(isColourImage) { - imgData = (void *)png2rgba(filepath.c_str(), &_width, &_height); + imgData = (void *)png2rgba(filepath.c_str(), &width, &height); } else { - imgData = (void *)png2ia(filepath.c_str(), &_width, &_height); + imgData = (void *)png2ia(filepath.c_str(), &width, &height); } - DebugHelper::assert(imgData != nullptr, "(N64Image::N64Image) Could not load image: ", filepath); + DebugHelper::assert_(imgData != nullptr, + "(N64Image::N64Image) Could not load image: ", filepath); + + _complete_load_init(imgData, width, height, format, alignData, isColourImage); + + free(imgData); +} + +N64Image::N64Image(uint8_t *data, size_t dataLength, std::string mimeType, std::string format, bool alignData) { + bool isColourImage = (format == "RGBA16") || (format == "RGBA32") || (format == "CI4") || (format == "CI8"); + + DebugHelper::assert_(mimeType == "image/png", + "Only .png embedded images are supported!"); + + void *imgData; // Can be either rgba* or ia* n64graphics types. + int width, height; + + // Load image from .png data, getting the width and height as well. + if(isColourImage) { + imgData = (void *)pngdata2rgba(data, (int)dataLength, &width, &height); + } else { + imgData = (void *)pngdata2ia(data, (int)dataLength, &width, &height); + } + + DebugHelper::assert_(imgData != nullptr, + "(N64Image::N64Image) Could not load image from \"", mimeType, "\" data."); + + _complete_load_init(imgData, width, height, format, alignData, isColourImage); + + free(imgData); +} + +N64Image::N64Image(void *imgData, size_t width, size_t height, std::string format, bool alignData) { + DebugHelper::assert_(imgData != nullptr, + "(N64Image::N64Image) Passed in null image data"); + + bool isColourImage = (format == "RGBA16") || (format == "RGBA32") || (format == "CI4") || (format == "CI8"); + + _complete_load_init(imgData, width, height, format, alignData, isColourImage); +} + +void N64Image::_complete_load_init(void *imgData, int width, int height, std::string format, bool alignData, bool isColourImage) { + _width = width; + _height = height; + _format = format; size_t size = ImageHelper::image_size(_width, _height, _format); @@ -50,7 +95,6 @@ N64Image::N64Image(fs::path filepath, std::string format, bool alignData) : _for } else { ia2raw(&_data[0], (ia*)imgData, _width, _height, get_bit_depth()); } - } N64Image::N64Image(int width, int height, std::string format, bool alignData) : _width(width), _height(height), _format(format) { @@ -367,14 +411,14 @@ void N64Image::save(fs::path finalFilepath) { imgData = get_img_as_rgba32(); int result = rgba2png(finalFilepath.c_str(), reinterpret_cast(imgData), _width, _height); // Save RGBA data as a .png file. - DebugHelper::assert(result == 1, "(N64Image::save) stbi_write_png returned ", result, ". Could not save image ", finalFilepath); - DebugHelper::assert(FileHelper::path_exists(finalFilepath), "(N64Image::save) Image ", finalFilepath, " could not be saved."); + DebugHelper::assert_(result == 1, "(N64Image::save) stbi_write_png returned ", result, ". Could not save image ", finalFilepath); + DebugHelper::assert_(FileHelper::path_exists(finalFilepath), "(N64Image::save) Image ", finalFilepath, " could not be saved."); } else { // Save IA image. imgData = get_img_as_ia16(); int result = ia2png(finalFilepath.c_str(), reinterpret_cast(imgData), _width, _height); // Save IA data as a .png file. - DebugHelper::assert(result == 1, "(N64Image::save) stbi_write_png returned ", result, ". Could not save image ", finalFilepath); - DebugHelper::assert(FileHelper::path_exists(finalFilepath), "(N64Image::save) Image ", finalFilepath, " could not be saved."); + DebugHelper::assert_(result == 1, "(N64Image::save) stbi_write_png returned ", result, ". Could not save image ", finalFilepath); + DebugHelper::assert_(FileHelper::path_exists(finalFilepath), "(N64Image::save) Image ", finalFilepath, " could not be saved."); } delete[] imgData; } @@ -548,8 +592,24 @@ void ImageHelper::guess_texture_format_and_render_mode(fs::path filepath, std::s // Load the image and take a guess from the pixels. int imgWidth, imgHeight; - rgba *data = (rgba*)png2rgba(filepath.c_str(), &imgWidth, &imgHeight); + std::string guessFormat, guessRenderMode; + + guess_texture_format_and_render_mode(data, imgWidth, imgHeight, guessFormat, guessRenderMode, ignoreTextureSize); + + if(outFormat.empty()) { + outFormat = guessFormat; + } + if(outRenderMode.empty()) { + outRenderMode = guessRenderMode; + } +} + +void ImageHelper::guess_texture_format_and_render_mode(rgba *data, int imgWidth, int imgHeight, std::string &outFormat, std::string &outRenderMode, bool ignoreTextureSize) { + outFormat = ""; + outRenderMode = ""; + + // Take a guess from the pixels. int numberOfPixels = imgWidth * imgHeight; @@ -568,44 +628,41 @@ void ImageHelper::guess_texture_format_and_render_mode(fs::path filepath, std::s } } - if(outFormat.empty()) { - if(hasColor) { - // Color image (RGBA16, RGBA32) - if(!ignoreTextureSize && (numberOfPixels > 32*32)) { - // Can't use 32-bit formats at this point - if(numberOfPixels > 64*32) { - DebugHelper::error("(ImageHelper::guess_texture_format_and_render_mode) ", - filepath.filename(), " is too large for a colored image!"); - } else { - outFormat = "RGBA16"; - } + if(hasColor) { + // Color image (RGBA16, RGBA32) + if(!ignoreTextureSize && (numberOfPixels > 32*32)) { + // Can't use 32-bit formats at this point + if(numberOfPixels > 64*32) { + DebugHelper::error("(ImageHelper::guess_texture_format_and_render_mode) Image was too large for a colored image! Size was ", + imgWidth, " by ", imgHeight, ". Cannot have more than 64*32 pixels!"); } else { - outFormat = (isSemiTransparent ? "RGBA32" : "RGBA16"); + outFormat = "RGBA16"; } } else { - // Grayscale image (I4, IA4, I8, IA8, IA16) - if(!ignoreTextureSize && (numberOfPixels > 64*32)) { - // Can't use 16-bit formats at this point - if(numberOfPixels > 64*64) { - // Can't use 8-bit formats at this point. - if(numberOfPixels > 128*64) { - DebugHelper::error("(ImageHelper::guess_texture_format_and_render_mode) ", - filepath.filename(), " is too large for a grayscale image!"); - } else { - outFormat = (isTransparent ? "IA4" : "I4"); - } + outFormat = (isSemiTransparent ? "RGBA32" : "RGBA16"); + } + } else { + // Grayscale image (I4, IA4, I8, IA8, IA16) + if(!ignoreTextureSize && (numberOfPixels > 64*32)) { + // Can't use 16-bit formats at this point + if(numberOfPixels > 64*64) { + // Can't use 8-bit formats at this point. + if(numberOfPixels > 128*64) { + DebugHelper::error("(ImageHelper::guess_texture_format_and_render_mode) Image was too large for a grayscale image! Size was ", + imgWidth, " by ", imgHeight, ". Cannot have more than 128*64 pixels!"); } else { - outFormat = (isTransparent ? "IA8" : "I8"); + outFormat = (isTransparent ? "IA4" : "I4"); } } else { - outFormat = (isTransparent ? "IA16" : "I8"); + outFormat = (isTransparent ? "IA8" : "I8"); } + } else { + outFormat = (isTransparent ? "IA16" : "I8"); } } + + outRenderMode = (isTransparent ? "TRANSPARENT" : "OPAQUE"); - if(outRenderMode.empty()) { - outRenderMode = (isTransparent ? "TRANSPARENT" : "OPAQUE"); - } } bool ImageHelper::guess_if_texture_is_animated(fs::path imgFilepath) { diff --git a/tools/dkr_assets_tool_src/helpers/imageHelper.h b/tools/dkr_assets_tool_src/helpers/imageHelper.h index cd843faa..55bf535f 100644 --- a/tools/dkr_assets_tool_src/helpers/imageHelper.h +++ b/tools/dkr_assets_tool_src/helpers/imageHelper.h @@ -1,6 +1,7 @@ #pragma once #include "helpers/fileHelper.h" +#include "libs/n64graphics/n64graphics.h" #include #include @@ -15,8 +16,15 @@ public: int x0, y0, x1, y1; }; - N64Image(fs::path filepath, std::string format, bool alignData=false); // Load image from filepath (Currently only .png is supported) - N64Image(int width=0, int height=0, std::string format="I4", bool alignData=false); // Create new image + // Load image from filepath (Currently only .png is supported) + N64Image(fs::path filepath, std::string format, bool alignData=false); + // Load image from memory (Currently only mimeType "image/png" is supported) + N64Image(uint8_t *data, size_t dataLength, std::string mimeType, std::string format, bool alignData=false); + // Load image from memory (n64graphics rgba/ia data type) + N64Image(void *imgData, size_t width, size_t height, std::string format, bool alignData=false); + + // Create new image + N64Image(int width=0, int height=0, std::string format="I4", bool alignData=false); ~N64Image(); void write_from(const uint8_t *inData, size_t dataLength); @@ -48,6 +56,7 @@ private: std::vector _data; + void _complete_load_init(void *imgData, int width, int height, std::string format, bool alignData, bool isColourImage); void _prep_region_copy(size_t &srcWidth, size_t &bytesToCopyPerLine, size_t &imgWidthInBytes, size_t &xOffsetInBytes, N64Image::Region ®ion); }; @@ -62,8 +71,12 @@ namespace ImageHelper { void get_width_and_height(fs::path filepath, int &outWidth, int &outHeight); // Get width and height of a .png image std::optional guess_associated_json_file(fs::path imgFilepath); + void guess_texture_wrap_mode(fs::path filepath, std::string &outWrapS, std::string &outWrapT); - void guess_texture_format_and_render_mode(fs::path filepath, std::string &outFormat, std::string &outRenderMode, bool ignoreTextureSize=false); + void guess_texture_format_and_render_mode(fs::path filepath, std::string &outFormat, std::string &outRenderMode, + bool ignoreTextureSize=false); + void guess_texture_format_and_render_mode(rgba *data, int imgWidth, int imgHeight, std::string &outFormat, + std::string &outRenderMode, bool ignoreTextureSize=false); bool guess_if_texture_is_animated(fs::path imgFilepath); bool guess_if_texture_double_sided(fs::path filepath); float guess_animated_texture_timing(fs::path imgFilepath); // Returns time to advance texture in seconds. diff --git a/tools/dkr_assets_tool_src/helpers/mathHelper.cpp b/tools/dkr_assets_tool_src/helpers/mathHelper.cpp index dbd94c81..4245206d 100644 --- a/tools/dkr_assets_tool_src/helpers/mathHelper.cpp +++ b/tools/dkr_assets_tool_src/helpers/mathHelper.cpp @@ -1,5 +1,7 @@ #include "mathHelper.h" +#include + using namespace DkrAssetsTool; /***************************************************************************/ @@ -15,6 +17,9 @@ Vec2f::Vec2f(float x, float y) : x(x), y(y){ Vec2f::Vec2f(std::vector &floats, size_t offset) : x(floats[offset]), y(floats[offset+1]) { } +Vec2f::Vec2f(std::vector& doubles, size_t offset) : x(doubles[offset]), y(doubles[offset + 1]) { +} + float Vec2f::operator[](int index) const { return values[index]; } @@ -121,6 +126,9 @@ Vec3f::Vec3f(float x, float y, float z) : x(x), y(y), z(z) { Vec3f::Vec3f(std::vector& floats, size_t offset) : x(floats[offset]), y(floats[offset + 1]), z(floats[offset + 2]) { } +Vec3f::Vec3f(std::vector& doubles, size_t offset) : x(doubles[offset]), y(doubles[offset + 1]), z(doubles[offset + 2]) { +} + float Vec3f::operator[](int index) const { return values[index]; } @@ -137,27 +145,34 @@ bool Vec3f::operator!=(const Vec3f& other) const { return !operator==(other); } -Vec3f Vec3f::operator+(const Vec3f& other) { +Vec3f Vec3f::operator+(const Vec3f& other) const { return Vec3f(x + other.x, y + other.y, z + other.z); } -Vec3f Vec3f::operator-(const Vec3f& other) { +Vec3f &Vec3f::operator+=(const Vec3f& other) { + x += other.x; + y += other.y; + z += other.z; + return *this; +} + +Vec3f Vec3f::operator-(const Vec3f& other) const { return Vec3f(x - other.x, y - other.y, z - other.z); } -Vec3f Vec3f::operator*(const Vec3f& other) { +Vec3f Vec3f::operator*(const Vec3f& other) const { return Vec3f(x * other.x, y * other.y, z * other.z); } -Vec3f Vec3f::operator*(const float other) { +Vec3f Vec3f::operator*(const float other) const { return Vec3f(x * other, y * other, z * other); } -Vec3f Vec3f::operator/(const Vec3f& other) { +Vec3f Vec3f::operator/(const Vec3f& other) const { return Vec3f(x / other.x, y / other.y, z / other.z); } -Vec3f Vec3f::operator/(const float other) { +Vec3f Vec3f::operator/(const float other) const { return Vec3f(x / other, y / other, z / other); } @@ -253,6 +268,9 @@ Vec4f::Vec4f(uint32_t rgba) { Vec4f::Vec4f(std::vector& floats, size_t offset) : x(floats[offset]), y(floats[offset + 1]), z(floats[offset + 2]), w(floats[offset + 3]) {} +Vec4f::Vec4f(std::vector& doubles, size_t offset) : x(doubles[offset]), y(doubles[offset + 1]), + z(doubles[offset + 2]), w(doubles[offset + 3]) {} + Vec4f::Vec4f(std::vector& bytes, size_t offset) : Vec4f(bytes[offset], bytes[offset+1], bytes[offset+2], bytes[offset+3]) { } @@ -378,3 +396,194 @@ std::vector Vec4f::to_vector() { std::vector Vec4f::to_double_vector() { return { x, y, z, w }; } + +/***************************************************************************/ + +Mat4x4f::Mat4x4f() { + // Should set this to be an identity matrix + for(size_t i = 0; i < 16; i++) { + values[i] = ((i % 5) == 0) ? 1.0f : 0.0f; + } +} + +Mat4x4f::Mat4x4f(const std::vector &src, size_t offset) { + for(size_t i = 0; i < 16; i++) { + values[i] = src[offset + i]; + } +} + +Mat4x4f::Mat4x4f(const std::vector &src, size_t offset) { + for(size_t i = 0; i < 16; i++) { + values[i] = static_cast(src[offset + i]); + } +} + +// Constructor using Euler angles (rotEuler in radians, XYZ order) +Mat4x4f::Mat4x4f(const Vec3f &translation, const Vec3f &rotEuler, const Vec3f &scale) { + // Extract angles + float cx = cosf(rotEuler.x), sx = sinf(rotEuler.x); + float cy = cosf(rotEuler.y), sy = sinf(rotEuler.y); + float cz = cosf(rotEuler.z), sz = sinf(rotEuler.z); + + // Rotation matrix from Euler angles (XYZ order) + float m00 = cy * cz; + float m01 = -cy * sz; + float m02 = sy; + + float m10 = sx * sy * cz + cx * sz; + float m11 = -sx * sy * sz + cx * cz; + float m12 = -sx * cy; + + float m20 = -cx * sy * cz + sx * sz; + float m21 = cx * sy * sz + sx * cz; + float m22 = cx * cy; + + // Apply scale + m00 *= scale.x; m01 *= scale.x; m02 *= scale.x; + m10 *= scale.y; m11 *= scale.y; m12 *= scale.y; + m20 *= scale.z; m21 *= scale.z; m22 *= scale.z; + + // Set matrix rows + rows[0] = { m00, m01, m02, 0.0f }; + rows[1] = { m10, m11, m12, 0.0f }; + rows[2] = { m20, m21, m22, 0.0f }; + rows[3] = { translation.x, translation.y, translation.z, 1.0f }; +} + +// Constructor using quaternion (x, y, z, w format) +Mat4x4f::Mat4x4f(const Vec3f &translation, const Vec4f &quat, const Vec3f &scale) { + float x = quat.x, y = quat.y, z = quat.z, w = quat.w; + + float xx = x * x, yy = y * y, zz = z * z; + float xy = x * y, xz = x * z, yz = y * z; + float wx = w * x, wy = w * y, wz = w * z; + + // Rotation matrix from quaternion + float m00 = 1.0f - 2.0f * (yy + zz); + float m01 = 2.0f * (xy - wz); + float m02 = 2.0f * (xz + wy); + + float m10 = 2.0f * (xy + wz); + float m11 = 1.0f - 2.0f * (xx + zz); + float m12 = 2.0f * (yz - wx); + + float m20 = 2.0f * (xz - wy); + float m21 = 2.0f * (yz + wx); + float m22 = 1.0f - 2.0f * (xx + yy); + + // Apply scale + m00 *= scale.x; m01 *= scale.x; m02 *= scale.x; + m10 *= scale.y; m11 *= scale.y; m12 *= scale.y; + m20 *= scale.z; m21 *= scale.z; m22 *= scale.z; + + // Set matrix rows + rows[0] = { m00, m01, m02, 0.0f }; + rows[1] = { m10, m11, m12, 0.0f }; + rows[2] = { m20, m21, m22, 0.0f }; + rows[3] = { translation.x, translation.y, translation.z, 1.0f }; +} + +Mat4x4f Mat4x4f::operator*(const Mat4x4f& other) const { + Mat4x4f result; + + // Row-major multiplication: result[i][j] = dot(row_i of a, column_j of b) + for (int row = 0; row < 4; ++row) { + const Vec4f& r = rows[row]; + for (int col = 0; col < 4; ++col) { + result.values[row * 4 + col] = + r.x * other.values[0 * 4 + col] + + r.y * other.values[1 * 4 + col] + + r.z * other.values[2 * 4 + col] + + r.w * other.values[3 * 4 + col]; + } + } + + return result; +} + +Vec3f Mat4x4f::get_translation() const { + return Vec3f(values[3], values[7], values[11]); +} + +// Utility: Normalize a 3D vector +inline Vec3f normalize3(const Vec3f& v) { + float len = std::sqrt(v.x * v.x + v.y * v.y + v.z * v.z); + return (len > 0.0f) ? Vec3f{v.x / len, v.y / len, v.z / len} : Vec3f{0.0f, 0.0f, 0.0f}; +} + +Vec3f Mat4x4f::get_rotation_euler() const { + // Extract and normalize axes to remove scale + Vec3f xAxis = normalize3(Vec3f(rows[0].x, rows[0].y, rows[0].z)); + Vec3f yAxis = normalize3(Vec3f(rows[1].x, rows[1].y, rows[1].z)); + Vec3f zAxis = normalize3(Vec3f(rows[2].x, rows[2].y, rows[2].z)); + + // Reconstruct rotation matrix + float m00 = xAxis.x, m01 = xAxis.y, m02 = xAxis.z; + float m10 = yAxis.x, m11 = yAxis.y, m12 = yAxis.z; + float /*m20 = zAxis.x, m21 = zAxis.y,*/ m22 = zAxis.z; + + Vec3f euler; + + // XYZ rotation extraction + if (std::abs(m02) < 1.0f) { + euler.y = std::asin(m02); // pitch + euler.x = std::atan2(-m12, m22); // yaw + euler.z = std::atan2(-m01, m00); // roll + } else { + // Gimbal lock + euler.y = (m02 > 0.0f) ? +M_PI / 2.0f : -M_PI / 2.0f; + euler.x = std::atan2(m10, m11); + euler.z = 0.0f; + } + + return euler; +} + +Vec4f Mat4x4f::get_rotation_quaternion() const { + // Normalize rotation axes (remove scale) + Vec3f xAxis = normalize3(Vec3f(rows[0].x, rows[0].y, rows[0].z)); + Vec3f yAxis = normalize3(Vec3f(rows[1].x, rows[1].y, rows[1].z)); + Vec3f zAxis = normalize3(Vec3f(rows[2].x, rows[2].y, rows[2].z)); + + // Reconstruct rotation matrix + float m00 = xAxis.x, m01 = xAxis.y, m02 = xAxis.z; + float m10 = yAxis.x, m11 = yAxis.y, m12 = yAxis.z; + float m20 = zAxis.x, m21 = zAxis.y, m22 = zAxis.z; + + Vec4f q; // x, y, z, w + float trace = m00 + m11 + m22; + + if (trace > 0.0f) { + float s = std::sqrt(trace + 1.0f) * 2.0f; + q.w = 0.25f * s; + q.x = (m21 - m12) / s; + q.y = (m02 - m20) / s; + q.z = (m10 - m01) / s; + } else if ((m00 > m11) && (m00 > m22)) { + float s = std::sqrt(1.0f + m00 - m11 - m22) * 2.0f; + q.w = (m21 - m12) / s; + q.x = 0.25f * s; + q.y = (m01 + m10) / s; + q.z = (m02 + m20) / s; + } else if (m11 > m22) { + float s = std::sqrt(1.0f + m11 - m00 - m22) * 2.0f; + q.w = (m02 - m20) / s; + q.x = (m01 + m10) / s; + q.y = 0.25f * s; + q.z = (m12 + m21) / s; + } else { + float s = std::sqrt(1.0f + m22 - m00 - m11) * 2.0f; + q.w = (m10 - m01) / s; + q.x = (m02 + m20) / s; + q.y = (m12 + m21) / s; + q.z = 0.25f * s; + } + + return q; +} + +Vec3f Mat4x4f::get_scale() const { + return Vec3f(values[0], values[5], values[10]); +} + +/***************************************************************************/ diff --git a/tools/dkr_assets_tool_src/helpers/mathHelper.h b/tools/dkr_assets_tool_src/helpers/mathHelper.h index e75f855e..96bd9f64 100644 --- a/tools/dkr_assets_tool_src/helpers/mathHelper.h +++ b/tools/dkr_assets_tool_src/helpers/mathHelper.h @@ -5,7 +5,9 @@ #include namespace DkrAssetsTool { - + +// TODO: Make a generic Vecf class, and then define Vec2f,Vec3f,Vec4f off of that. + struct Vec2f { union { struct { @@ -23,6 +25,7 @@ struct Vec2f { Vec2f(float s); Vec2f(float x, float y); Vec2f(std::vector &floats, size_t offset = 0); + Vec2f(std::vector &doubles, size_t offset = 0); float operator[](int index) const; float& operator[](int index); @@ -74,17 +77,19 @@ struct Vec3f { Vec3f(float s); Vec3f(float x, float y, float z); Vec3f(std::vector &floats, size_t offset = 0); + Vec3f(std::vector &doubles, size_t offset = 0); float operator[](int index) const; float& operator[](int index); bool operator==(const Vec3f& other) const; bool operator!=(const Vec3f& other) const; - Vec3f operator+(const Vec3f& other); - Vec3f operator-(const Vec3f& other); - Vec3f operator*(const Vec3f& other); - Vec3f operator*(const float other); - Vec3f operator/(const Vec3f& other); - Vec3f operator/(const float other); + Vec3f operator+(const Vec3f& other) const; + Vec3f &operator+=(const Vec3f& other); + Vec3f operator-(const Vec3f& other) const; + Vec3f operator*(const Vec3f& other) const; + Vec3f operator*(const float other) const; + Vec3f operator/(const Vec3f& other) const; + Vec3f operator/(const float other) const; friend std::ostream& operator<<(std::ostream& os, const Vec3f& vec); Vec3f min(const Vec3f& other) const; // Returns the minimum XYZ values of both vectors @@ -132,6 +137,7 @@ struct Vec4f { Vec4f(uint8_t r, uint8_t g, uint8_t b, uint8_t a); Vec4f(uint32_t rgba); Vec4f(std::vector &floats, size_t offset = 0); + Vec4f(std::vector &doubles, size_t offset = 0); Vec4f(std::vector &bytes, size_t offset = 0); float operator[](int index) const; @@ -172,6 +178,37 @@ struct Vec4f { return Vec4f(std::numeric_limits::max()); } }; + +struct Mat4x4f { + union { + Vec4f rows[4]; + float values[16]; + }; + + Mat4x4f(); // Identity matrix + Mat4x4f(const std::vector &src, size_t offset = 0); + Mat4x4f(const std::vector &src, size_t offset = 0); + Mat4x4f(const Vec3f &translation, const Vec3f &rotEuler, const Vec3f &scale); + Mat4x4f(const Vec3f &translation, const Vec4f &rotQuaternion, const Vec3f &scale); + + Mat4x4f operator*(const Mat4x4f& other) const; + + static Mat4x4f identity() { + return Mat4x4f(std::vector{ + 1, 0, 0, 0, + 0, 1, 0, 0, + 0, 0, 1, 0, + 0, 0, 0, 0 + }); + } + + Vec3f get_translation() const; + + Vec3f get_rotation_euler() const; + Vec4f get_rotation_quaternion() const; + + Vec3f get_scale() const; +}; } diff --git a/tools/dkr_assets_tool_src/helpers/stringHelper.cpp b/tools/dkr_assets_tool_src/helpers/stringHelper.cpp index 2d77f525..416d2b8b 100644 --- a/tools/dkr_assets_tool_src/helpers/stringHelper.cpp +++ b/tools/dkr_assets_tool_src/helpers/stringHelper.cpp @@ -28,6 +28,16 @@ void StringHelper::make_uppercase(std::string& input) { character = std::toupper(character); } } + +std::string StringHelper::lowercase(std::string input) { + make_lowercase(input); + return input; +} + +std::string StringHelper::uppercase(std::string input) { + make_uppercase(input); + return input; +} bool StringHelper::starts_with(const std::string &input, const std::string &sub) { return input.rfind(sub, 0) == 0; @@ -82,7 +92,7 @@ void StringHelper::replace(std::string &input, std::string oldPattern, std::stri } } -void StringHelper::split(std::string &input, const char delim, std::vector &out) { +void StringHelper::split(const std::string &input, const char delim, std::vector &out) { std::stringstream ss(input); std::string s; while (std::getline(ss, s, delim)) { @@ -90,7 +100,7 @@ void StringHelper::split(std::string &input, const char delim, std::vector &out) { +void StringHelper::split_and_trim(const std::string &input, const char delim, std::vector &out) { std::stringstream ss(input); std::string s; while (std::getline(ss, s, delim)) { @@ -99,6 +109,44 @@ void StringHelper::split_and_trim(std::string &input, const char delim, std::vec } } +void StringHelper::split(const std::string &input, const std::vector delims, std::vector &out) { + if(delims.empty()) { + return; + } + + size_t startIndex = 0; + for(size_t i = 0; i < input.size(); i++) { + for(char delim : delims) { + if(input[i] == delim) { + // Output string. + out.emplace_back(input.substr(startIndex, i - startIndex)); + startIndex = i + 1; + break; + } + } + } +} + +void StringHelper::split_and_trim(const std::string &input, const std::vector delims, std::vector &out) { + if(delims.empty()) { + return; + } + + size_t startIndex = 0; + for(size_t i = 0; i < input.size(); i++) { + for(char delim : delims) { + if(input[i] == delim) { + // Output string. + std::string outEntry = input.substr(startIndex, i - startIndex); + trim(outEntry); + out.emplace_back(outEntry); + startIndex = i + 1; + break; + } + } + } +} + std::string StringHelper::join(const std::vector &input, std::string delim, size_t startIndex) { std::stringstream ss; for(size_t i = startIndex; i < input.size(); i++) { @@ -138,6 +186,71 @@ int StringHelper::find_closing_brace(const std::string &input, int inputOffset) return -1; } +std::string StringHelper::remove_comments_from_c_code(std::string code) { + std::stringstream ss; // Use stringstream to build the result + bool inMultilineComment = false; + bool inSinglelineComment = false; + bool inString = false; // To handle comments inside string literals + + for (size_t i = 0; i < code.length(); ++i) { + // Check for beginning of string literal + if (!inMultilineComment && !inSinglelineComment && code[i] == '"') { + inString = !inString; + ss << code[i]; // Append to stringstream + continue; + } + + // If we are inside a string, just append the character + if (inString) { + ss << code[i]; // Append to stringstream + // Handle escaped quotes inside string + if (code[i] == '\\' && i + 1 < code.length() && code[i + 1] == '"') { + ss << code[i + 1]; + i++; + } + continue; + } + + // Check for start of multi-line comment /* + if (!inSinglelineComment && i + 1 < code.length() && + code[i] == '/' && code[i + 1] == '*') { + inMultilineComment = true; + i++; // Skip the '*' + continue; + } + + // Check for end of multi-line comment */ + if (inMultilineComment && i + 1 < code.length() && + code[i] == '*' && code[i + 1] == '/') { + inMultilineComment = false; + i++; // Skip the '/' + continue; + } + + // Check for start of single-line comment // + if (!inMultilineComment && i + 1 < code.length() && + code[i] == '/' && code[i + 1] == '/') { + inSinglelineComment = true; + i++; // Skip the second '/' + continue; + } + + // Check for end of single-line comment (newline) + if (inSinglelineComment && code[i] == '\n') { + inSinglelineComment = false; + ss << code[i]; // Keep the newline character, append to stringstream + continue; + } + + // If not in any comment, append the character to the result + if (!inMultilineComment && !inSinglelineComment) { + ss << code[i]; + } + } + + return ss.str(); +} + std::string StringHelper::upper_snake_case_to_pascal_case(const std::string &input) { std::stringstream out; bool isFirst = true; diff --git a/tools/dkr_assets_tool_src/helpers/stringHelper.h b/tools/dkr_assets_tool_src/helpers/stringHelper.h index cdd4acd4..849cbc8a 100644 --- a/tools/dkr_assets_tool_src/helpers/stringHelper.h +++ b/tools/dkr_assets_tool_src/helpers/stringHelper.h @@ -14,6 +14,9 @@ namespace StringHelper { void make_lowercase(std::string &input); void make_uppercase(std::string &input); + std::string lowercase(std::string input); + std::string uppercase(std::string input); + bool starts_with(const std::string &input, const std::string &sub); bool ends_with(const std::string &input, const std::string &sub); @@ -31,15 +34,22 @@ namespace StringHelper { // Returns a string that is made lowercase, whitespace removed, and all dashes `-` and underscores `_` removed. std::string simplify(const std::string& input); - // Splits the input string by a single character into out, so "A,B,C" would be put into [A,B,C]. input is NOT modified! - void split(std::string &input, const char delim, std::vector &out); - void split_and_trim(std::string &input, const char delim, std::vector &out); + // Splits the input string by a single character into out, so "A,B,C" would be put into [A,B,C]. + void split(const std::string &input, const char delim, std::vector &out); + void split_and_trim(const std::string &input, const char delim, std::vector &out); + + // Splits input based on multiple delimater characters. So you can have a string based on `,` OR `;` OR `|`, etc. + void split(const std::string &input, const std::vector delims, std::vector &out); + void split_and_trim(const std::string &input, const std::vector delims, std::vector &out); std::string join(const std::vector &input, std::string delim, size_t startIndex = 0); // Returns the index of the closing brace of the first opening brace encountered. Or -1 if none was encountered. int find_closing_brace(const std::string &input, int inputOffset=0); + // Removes all single-line (//) and multi-line comments (/**/) from a block of C code. + std::string remove_comments_from_c_code(std::string code); + // Converts `FOO_BAR` to `FooBar`. std::string upper_snake_case_to_pascal_case(const std::string &input); // Converts `FooBar` to `FOO_BAR` diff --git a/tools/dkr_assets_tool_src/libs/backward.hpp b/tools/dkr_assets_tool_src/libs/backward.hpp new file mode 100644 index 00000000..263cd1b4 --- /dev/null +++ b/tools/dkr_assets_tool_src/libs/backward.hpp @@ -0,0 +1,4530 @@ +/* + * backward.hpp + * Copyright 2013 Google Inc. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef H_6B9572DA_A64B_49E6_B234_051480991C89 +#define H_6B9572DA_A64B_49E6_B234_051480991C89 + +#ifndef __cplusplus +#error "It's not going to compile without a C++ compiler..." +#endif + +#if defined(BACKWARD_CXX11) +#elif defined(BACKWARD_CXX98) +#else +#if __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1800) +#define BACKWARD_CXX11 +#define BACKWARD_ATLEAST_CXX11 +#define BACKWARD_ATLEAST_CXX98 +#if __cplusplus >= 201703L || (defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) +#define BACKWARD_ATLEAST_CXX17 +#endif +#else +#define BACKWARD_CXX98 +#define BACKWARD_ATLEAST_CXX98 +#endif +#endif + +// You can define one of the following (or leave it to the auto-detection): +// +// #define BACKWARD_SYSTEM_LINUX +// - specialization for linux +// +// #define BACKWARD_SYSTEM_DARWIN +// - specialization for Mac OS X 10.5 and later. +// +// #define BACKWARD_SYSTEM_WINDOWS +// - specialization for Windows (Clang 9 and MSVC2017) +// +// #define BACKWARD_SYSTEM_UNKNOWN +// - placebo implementation, does nothing. +// +#if defined(BACKWARD_SYSTEM_LINUX) +#elif defined(BACKWARD_SYSTEM_DARWIN) +#elif defined(BACKWARD_SYSTEM_UNKNOWN) +#elif defined(BACKWARD_SYSTEM_WINDOWS) +#else +#if defined(__linux) || defined(__linux__) +#define BACKWARD_SYSTEM_LINUX +#elif defined(__APPLE__) +#define BACKWARD_SYSTEM_DARWIN +#elif defined(_WIN32) +#define BACKWARD_SYSTEM_WINDOWS +#else +#define BACKWARD_SYSTEM_UNKNOWN +#endif +#endif + +#define NOINLINE __attribute__((noinline)) + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(BACKWARD_SYSTEM_LINUX) + +// On linux, backtrace can back-trace or "walk" the stack using the following +// libraries: +// +// #define BACKWARD_HAS_UNWIND 1 +// - unwind comes from libgcc, but I saw an equivalent inside clang itself. +// - with unwind, the stacktrace is as accurate as it can possibly be, since +// this is used by the C++ runtime in gcc/clang for stack unwinding on +// exception. +// - normally libgcc is already linked to your program by default. +// +// #define BACKWARD_HAS_LIBUNWIND 1 +// - libunwind provides, in some cases, a more accurate stacktrace as it knows +// to decode signal handler frames and lets us edit the context registers when +// unwinding, allowing stack traces over bad function references. +// +// #define BACKWARD_HAS_BACKTRACE == 1 +// - backtrace seems to be a little bit more portable than libunwind, but on +// linux, it uses unwind anyway, but abstract away a tiny information that is +// sadly really important in order to get perfectly accurate stack traces. +// - backtrace is part of the (e)glib library. +// +// The default is: +// #define BACKWARD_HAS_UNWIND == 1 +// +// Note that only one of the define should be set to 1 at a time. +// +#if BACKWARD_HAS_UNWIND == 1 +#elif BACKWARD_HAS_LIBUNWIND == 1 +#elif BACKWARD_HAS_BACKTRACE == 1 +#else +#undef BACKWARD_HAS_UNWIND +#define BACKWARD_HAS_UNWIND 1 +#undef BACKWARD_HAS_LIBUNWIND +#define BACKWARD_HAS_LIBUNWIND 0 +#undef BACKWARD_HAS_BACKTRACE +#define BACKWARD_HAS_BACKTRACE 0 +#endif + +// On linux, backward can extract detailed information about a stack trace +// using one of the following libraries: +// +// #define BACKWARD_HAS_DW 1 +// - libdw gives you the most juicy details out of your stack traces: +// - object filename +// - function name +// - source filename +// - line and column numbers +// - source code snippet (assuming the file is accessible) +// - variable names (if not optimized out) +// - variable values (not supported by backward-cpp) +// - You need to link with the lib "dw": +// - apt-get install libdw-dev +// - g++/clang++ -ldw ... +// +// #define BACKWARD_HAS_BFD 1 +// - With libbfd, you get a fair amount of details: +// - object filename +// - function name +// - source filename +// - line numbers +// - source code snippet (assuming the file is accessible) +// - You need to link with the lib "bfd": +// - apt-get install binutils-dev +// - g++/clang++ -lbfd ... +// +// #define BACKWARD_HAS_DWARF 1 +// - libdwarf gives you the most juicy details out of your stack traces: +// - object filename +// - function name +// - source filename +// - line and column numbers +// - source code snippet (assuming the file is accessible) +// - variable names (if not optimized out) +// - variable values (not supported by backward-cpp) +// - You need to link with the lib "dwarf": +// - apt-get install libdwarf-dev +// - g++/clang++ -ldwarf ... +// +// #define BACKWARD_HAS_BACKTRACE_SYMBOL 1 +// - backtrace provides minimal details for a stack trace: +// - object filename +// - function name +// - backtrace is part of the (e)glib library. +// +// The default is: +// #define BACKWARD_HAS_BACKTRACE_SYMBOL == 1 +// +// Note that only one of the define should be set to 1 at a time. +// +#if BACKWARD_HAS_DW == 1 +#elif BACKWARD_HAS_BFD == 1 +#elif BACKWARD_HAS_DWARF == 1 +#elif BACKWARD_HAS_BACKTRACE_SYMBOL == 1 +#else +#undef BACKWARD_HAS_DW +#define BACKWARD_HAS_DW 0 +#undef BACKWARD_HAS_BFD +#define BACKWARD_HAS_BFD 0 +#undef BACKWARD_HAS_DWARF +#define BACKWARD_HAS_DWARF 0 +#undef BACKWARD_HAS_BACKTRACE_SYMBOL +#define BACKWARD_HAS_BACKTRACE_SYMBOL 1 +#endif + +#include +#include +#ifdef __ANDROID__ +// Old Android API levels define _Unwind_Ptr in both link.h and +// unwind.h Rename the one in link.h as we are not going to be using +// it +#define _Unwind_Ptr _Unwind_Ptr_Custom +#include +#undef _Unwind_Ptr +#else +#include +#endif +#if defined(__ppc__) || defined(__powerpc) || defined(__powerpc__) || \ + defined(__POWERPC__) +// Linux kernel header required for the struct pt_regs definition +// to access the NIP (Next Instruction Pointer) register value +#include +#endif +#include +#include +#include +#include +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#include +#undef _GNU_SOURCE +#else +#include +#endif + +#if BACKWARD_HAS_BFD == 1 +// NOTE: defining PACKAGE{,_VERSION} is required before including +// bfd.h on some platforms, see also: +// https://sourceware.org/bugzilla/show_bug.cgi?id=14243 +#ifndef PACKAGE +#define PACKAGE +#endif +#ifndef PACKAGE_VERSION +#define PACKAGE_VERSION +#endif +#include +#endif + +#if BACKWARD_HAS_DW == 1 +#include +#include +#include +#endif + +#if BACKWARD_HAS_DWARF == 1 +#include +#include +#include +#include +#include +#endif + +#if (BACKWARD_HAS_BACKTRACE == 1) || (BACKWARD_HAS_BACKTRACE_SYMBOL == 1) +// then we shall rely on backtrace +#include +#endif + +#endif // defined(BACKWARD_SYSTEM_LINUX) + +#if defined(BACKWARD_SYSTEM_DARWIN) +// On Darwin, backtrace can back-trace or "walk" the stack using the following +// libraries: +// +// #define BACKWARD_HAS_UNWIND 1 +// - unwind comes from libgcc, but I saw an equivalent inside clang itself. +// - with unwind, the stacktrace is as accurate as it can possibly be, since +// this is used by the C++ runtime in gcc/clang for stack unwinding on +// exception. +// - normally libgcc is already linked to your program by default. +// +// #define BACKWARD_HAS_LIBUNWIND 1 +// - libunwind comes from clang, which implements an API compatible version. +// - libunwind provides, in some cases, a more accurate stacktrace as it knows +// to decode signal handler frames and lets us edit the context registers when +// unwinding, allowing stack traces over bad function references. +// +// #define BACKWARD_HAS_BACKTRACE == 1 +// - backtrace is available by default, though it does not produce as much +// information as another library might. +// +// The default is: +// #define BACKWARD_HAS_UNWIND == 1 +// +// Note that only one of the define should be set to 1 at a time. +// +#if BACKWARD_HAS_UNWIND == 1 +#elif BACKWARD_HAS_BACKTRACE == 1 +#elif BACKWARD_HAS_LIBUNWIND == 1 +#else +#undef BACKWARD_HAS_UNWIND +#define BACKWARD_HAS_UNWIND 1 +#undef BACKWARD_HAS_BACKTRACE +#define BACKWARD_HAS_BACKTRACE 0 +#undef BACKWARD_HAS_LIBUNWIND +#define BACKWARD_HAS_LIBUNWIND 0 +#endif + +// On Darwin, backward can extract detailed information about a stack trace +// using one of the following libraries: +// +// #define BACKWARD_HAS_BACKTRACE_SYMBOL 1 +// - backtrace provides minimal details for a stack trace: +// - object filename +// - function name +// +// The default is: +// #define BACKWARD_HAS_BACKTRACE_SYMBOL == 1 +// +#if BACKWARD_HAS_BACKTRACE_SYMBOL == 1 +#else +#undef BACKWARD_HAS_BACKTRACE_SYMBOL +#define BACKWARD_HAS_BACKTRACE_SYMBOL 1 +#endif + +#include +#include +#include +#include +#include +#include + +#if (BACKWARD_HAS_BACKTRACE == 1) || (BACKWARD_HAS_BACKTRACE_SYMBOL == 1) +#include +#endif +#endif // defined(BACKWARD_SYSTEM_DARWIN) + +#if defined(BACKWARD_SYSTEM_WINDOWS) + +#include +#include +#include + +#include + +#ifdef _WIN64 +typedef SSIZE_T ssize_t; +#else +typedef int ssize_t; +#endif + +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#include + +#include +#include + +#ifndef __clang__ +#undef NOINLINE +#define NOINLINE __declspec(noinline) +#endif + +#ifdef _MSC_VER +#pragma comment(lib, "psapi.lib") +#pragma comment(lib, "dbghelp.lib") +#endif + +// Comment / packing is from stackoverflow: +// https://stackoverflow.com/questions/6205981/windows-c-stack-trace-from-a-running-app/28276227#28276227 +// Some versions of imagehlp.dll lack the proper packing directives themselves +// so we need to do it. +#pragma pack(push, before_imagehlp, 8) +#include +#pragma pack(pop, before_imagehlp) + +// TODO maybe these should be undefined somewhere else? +#undef BACKWARD_HAS_UNWIND +#undef BACKWARD_HAS_BACKTRACE +#if BACKWARD_HAS_PDB_SYMBOL == 1 +#else +#undef BACKWARD_HAS_PDB_SYMBOL +#define BACKWARD_HAS_PDB_SYMBOL 1 +#endif + +#endif + +#if BACKWARD_HAS_UNWIND == 1 + +#include +// while gcc's unwind.h defines something like that: +// extern _Unwind_Ptr _Unwind_GetIP (struct _Unwind_Context *); +// extern _Unwind_Ptr _Unwind_GetIPInfo (struct _Unwind_Context *, int *); +// +// clang's unwind.h defines something like this: +// uintptr_t _Unwind_GetIP(struct _Unwind_Context* __context); +// +// Even if the _Unwind_GetIPInfo can be linked to, it is not declared, worse we +// cannot just redeclare it because clang's unwind.h doesn't define _Unwind_Ptr +// anyway. +// +// Luckily we can play on the fact that the guard macros have a different name: +#ifdef __CLANG_UNWIND_H +// In fact, this function still comes from libgcc (on my different linux boxes, +// clang links against libgcc). +#include +extern "C" uintptr_t _Unwind_GetIPInfo(_Unwind_Context *, int *); +#endif + +#endif // BACKWARD_HAS_UNWIND == 1 + +#if BACKWARD_HAS_LIBUNWIND == 1 +#define UNW_LOCAL_ONLY +#include +#endif // BACKWARD_HAS_LIBUNWIND == 1 + +#ifdef BACKWARD_ATLEAST_CXX11 +#include +#include // for std::swap +namespace backward { +namespace details { +template struct hashtable { + typedef std::unordered_map type; +}; +using std::move; +} // namespace details +} // namespace backward +#else // NOT BACKWARD_ATLEAST_CXX11 +#define nullptr NULL +#define override +#include +namespace backward { +namespace details { +template struct hashtable { + typedef std::map type; +}; +template const T &move(const T &v) { return v; } +template T &move(T &v) { return v; } +} // namespace details +} // namespace backward +#endif // BACKWARD_ATLEAST_CXX11 + +namespace backward { +namespace details { +#if defined(BACKWARD_SYSTEM_WINDOWS) +const char kBackwardPathDelimiter[] = ";"; +#else +const char kBackwardPathDelimiter[] = ":"; +#endif +} // namespace details +} // namespace backward + +namespace backward { + +namespace system_tag { +struct linux_tag; // seems that I cannot call that "linux" because the name +// is already defined... so I am adding _tag everywhere. +struct darwin_tag; +struct windows_tag; +struct unknown_tag; + +#if defined(BACKWARD_SYSTEM_LINUX) +typedef linux_tag current_tag; +#elif defined(BACKWARD_SYSTEM_DARWIN) +typedef darwin_tag current_tag; +#elif defined(BACKWARD_SYSTEM_WINDOWS) +typedef windows_tag current_tag; +#elif defined(BACKWARD_SYSTEM_UNKNOWN) +typedef unknown_tag current_tag; +#else +#error "May I please get my system defines?" +#endif +} // namespace system_tag + +namespace trace_resolver_tag { +#if defined(BACKWARD_SYSTEM_LINUX) +struct libdw; +struct libbfd; +struct libdwarf; +struct backtrace_symbol; + +#if BACKWARD_HAS_DW == 1 +typedef libdw current; +#elif BACKWARD_HAS_BFD == 1 +typedef libbfd current; +#elif BACKWARD_HAS_DWARF == 1 +typedef libdwarf current; +#elif BACKWARD_HAS_BACKTRACE_SYMBOL == 1 +typedef backtrace_symbol current; +#else +#error "You shall not pass, until you know what you want." +#endif +#elif defined(BACKWARD_SYSTEM_DARWIN) +struct backtrace_symbol; + +#if BACKWARD_HAS_BACKTRACE_SYMBOL == 1 +typedef backtrace_symbol current; +#else +#error "You shall not pass, until you know what you want." +#endif +#elif defined(BACKWARD_SYSTEM_WINDOWS) +struct pdb_symbol; +#if BACKWARD_HAS_PDB_SYMBOL == 1 +typedef pdb_symbol current; +#else +#error "You shall not pass, until you know what you want." +#endif +#endif +} // namespace trace_resolver_tag + +namespace details { + +template struct rm_ptr { typedef T type; }; + +template struct rm_ptr { typedef T type; }; + +template struct rm_ptr { typedef const T type; }; + +template struct deleter { + template void operator()(U &ptr) const { (*F)(ptr); } +}; + +template struct default_delete { + void operator()(T &ptr) const { delete ptr; } +}; + +template > +class handle { + struct dummy; + T _val; + bool _empty; + +#ifdef BACKWARD_ATLEAST_CXX11 + handle(const handle &) = delete; + handle &operator=(const handle &) = delete; +#endif + +public: + ~handle() { + if (!_empty) { + Deleter()(_val); + } + } + + explicit handle() : _val(), _empty(true) {} + explicit handle(T val) : _val(val), _empty(false) { + if (!_val) + _empty = true; + } + +#ifdef BACKWARD_ATLEAST_CXX11 + handle(handle &&from) : _empty(true) { swap(from); } + handle &operator=(handle &&from) { + swap(from); + return *this; + } +#else + explicit handle(const handle &from) : _empty(true) { + // some sort of poor man's move semantic. + swap(const_cast(from)); + } + handle &operator=(const handle &from) { + // some sort of poor man's move semantic. + swap(const_cast(from)); + return *this; + } +#endif + + void reset(T new_val) { + handle tmp(new_val); + swap(tmp); + } + + void update(T new_val) { + _val = new_val; + _empty = !static_cast(new_val); + } + + operator const dummy *() const { + if (_empty) { + return nullptr; + } + return reinterpret_cast(_val); + } + T get() { return _val; } + T release() { + _empty = true; + return _val; + } + void swap(handle &b) { + using std::swap; + swap(b._val, _val); // can throw, we are safe here. + swap(b._empty, _empty); // should not throw: if you cannot swap two + // bools without throwing... It's a lost cause anyway! + } + + T &operator->() { return _val; } + const T &operator->() const { return _val; } + + typedef typename rm_ptr::type &ref_t; + typedef const typename rm_ptr::type &const_ref_t; + ref_t operator*() { return *_val; } + const_ref_t operator*() const { return *_val; } + ref_t operator[](size_t idx) { return _val[idx]; } + + // Watch out, we've got a badass over here + T *operator&() { + _empty = false; + return &_val; + } +}; + +// Default demangler implementation (do nothing). +template struct demangler_impl { + static std::string demangle(const char *funcname) { return funcname; } +}; + +#if defined(BACKWARD_SYSTEM_LINUX) || defined(BACKWARD_SYSTEM_DARWIN) + +template <> struct demangler_impl { + demangler_impl() : _demangle_buffer_length(0) {} + + std::string demangle(const char *funcname) { + using namespace details; + char *result = abi::__cxa_demangle(funcname, _demangle_buffer.get(), + &_demangle_buffer_length, nullptr); + if (result) { + _demangle_buffer.update(result); + return result; + } + return funcname; + } + +private: + details::handle _demangle_buffer; + size_t _demangle_buffer_length; +}; + +#endif // BACKWARD_SYSTEM_LINUX || BACKWARD_SYSTEM_DARWIN + +struct demangler : public demangler_impl {}; + +// Split a string on the platform's PATH delimiter. Example: if delimiter +// is ":" then: +// "" --> [] +// ":" --> ["",""] +// "::" --> ["","",""] +// "/a/b/c" --> ["/a/b/c"] +// "/a/b/c:/d/e/f" --> ["/a/b/c","/d/e/f"] +// etc. +inline std::vector split_source_prefixes(const std::string &s) { + std::vector out; + size_t last = 0; + size_t next = 0; + size_t delimiter_size = sizeof(kBackwardPathDelimiter) - 1; + while ((next = s.find(kBackwardPathDelimiter, last)) != std::string::npos) { + out.push_back(s.substr(last, next - last)); + last = next + delimiter_size; + } + if (last <= s.length()) { + out.push_back(s.substr(last)); + } + return out; +} + +} // namespace details + +/*************** A TRACE ***************/ + +struct Trace { + void *addr; + size_t idx; + + Trace() : addr(nullptr), idx(0) {} + + explicit Trace(void *_addr, size_t _idx) : addr(_addr), idx(_idx) {} +}; + +struct ResolvedTrace : public Trace { + + struct SourceLoc { + std::string function; + std::string filename; + unsigned line; + unsigned col; + + SourceLoc() : line(0), col(0) {} + + bool operator==(const SourceLoc &b) const { + return function == b.function && filename == b.filename && + line == b.line && col == b.col; + } + + bool operator!=(const SourceLoc &b) const { return !(*this == b); } + }; + + // In which binary object this trace is located. + std::string object_filename; + + // The function in the object that contain the trace. This is not the same + // as source.function which can be an function inlined in object_function. + std::string object_function; + + // The source location of this trace. It is possible for filename to be + // empty and for line/col to be invalid (value 0) if this information + // couldn't be deduced, for example if there is no debug information in the + // binary object. + SourceLoc source; + + // An optionals list of "inliners". All the successive sources location + // from where the source location of the trace (the attribute right above) + // is inlined. It is especially useful when you compiled with optimization. + typedef std::vector source_locs_t; + source_locs_t inliners; + + ResolvedTrace() : Trace() {} + ResolvedTrace(const Trace &mini_trace) : Trace(mini_trace) {} +}; + +/*************** STACK TRACE ***************/ + +// default implemention. +template class StackTraceImpl { +public: + size_t size() const { return 0; } + Trace operator[](size_t) const { return Trace(); } + size_t load_here(size_t = 0) { return 0; } + size_t load_from(void *, size_t = 0, void * = nullptr, void * = nullptr) { + return 0; + } + size_t thread_id() const { return 0; } + void skip_n_firsts(size_t) {} + void *const *begin() const { return nullptr; } +}; + +class StackTraceImplBase { +public: + StackTraceImplBase() + : _thread_id(0), _skip(0), _context(nullptr), _error_addr(nullptr) {} + + size_t thread_id() const { return _thread_id; } + + void skip_n_firsts(size_t n) { _skip = n; } + +protected: + void load_thread_info() { +#ifdef BACKWARD_SYSTEM_LINUX +#ifndef __ANDROID__ + _thread_id = static_cast(syscall(SYS_gettid)); +#else + _thread_id = static_cast(gettid()); +#endif + if (_thread_id == static_cast(getpid())) { + // If the thread is the main one, let's hide that. + // I like to keep little secret sometimes. + _thread_id = 0; + } +#elif defined(BACKWARD_SYSTEM_DARWIN) + _thread_id = reinterpret_cast(pthread_self()); + if (pthread_main_np() == 1) { + // If the thread is the main one, let's hide that. + _thread_id = 0; + } +#endif + } + + void set_context(void *context) { _context = context; } + void *context() const { return _context; } + + void set_error_addr(void *error_addr) { _error_addr = error_addr; } + void *error_addr() const { return _error_addr; } + + size_t skip_n_firsts() const { return _skip; } + +private: + size_t _thread_id; + size_t _skip; + void *_context; + void *_error_addr; +}; + +class StackTraceImplHolder : public StackTraceImplBase { +public: + size_t size() const { + return (_stacktrace.size() >= skip_n_firsts()) + ? _stacktrace.size() - skip_n_firsts() + : 0; + } + Trace operator[](size_t idx) const { + if (idx >= size()) { + return Trace(); + } + return Trace(_stacktrace[idx + skip_n_firsts()], idx); + } + void *const *begin() const { + if (size()) { + return &_stacktrace[skip_n_firsts()]; + } + return nullptr; + } + +protected: + std::vector _stacktrace; +}; + +#if BACKWARD_HAS_UNWIND == 1 + +namespace details { + +template class Unwinder { +public: + size_t operator()(F &f, size_t depth) { + _f = &f; + _index = -1; + _depth = depth; + _Unwind_Backtrace(&this->backtrace_trampoline, this); + if (_index == -1) { + // _Unwind_Backtrace has failed to obtain any backtraces + return 0; + } else { + return static_cast(_index); + } + } + +private: + F *_f; + ssize_t _index; + size_t _depth; + + static _Unwind_Reason_Code backtrace_trampoline(_Unwind_Context *ctx, + void *self) { + return (static_cast(self))->backtrace(ctx); + } + + _Unwind_Reason_Code backtrace(_Unwind_Context *ctx) { + if (_index >= 0 && static_cast(_index) >= _depth) + return _URC_END_OF_STACK; + + int ip_before_instruction = 0; + uintptr_t ip = _Unwind_GetIPInfo(ctx, &ip_before_instruction); + + if (!ip_before_instruction) { + // calculating 0-1 for unsigned, looks like a possible bug to sanitizers, + // so let's do it explicitly: + if (ip == 0) { + ip = std::numeric_limits::max(); // set it to 0xffff... (as + // from casting 0-1) + } else { + ip -= 1; // else just normally decrement it (no overflow/underflow will + // happen) + } + } + + if (_index >= 0) { // ignore first frame. + (*_f)(static_cast(_index), reinterpret_cast(ip)); + } + _index += 1; + return _URC_NO_REASON; + } +}; + +template size_t unwind(F f, size_t depth) { + Unwinder unwinder; + return unwinder(f, depth); +} + +} // namespace details + +template <> +class StackTraceImpl : public StackTraceImplHolder { +public: + NOINLINE + size_t load_here(size_t depth = 32, void *context = nullptr, + void *error_addr = nullptr) { + load_thread_info(); + set_context(context); + set_error_addr(error_addr); + if (depth == 0) { + return 0; + } + _stacktrace.resize(depth); + size_t trace_cnt = details::unwind(callback(*this), depth); + _stacktrace.resize(trace_cnt); + skip_n_firsts(0); + return size(); + } + size_t load_from(void *addr, size_t depth = 32, void *context = nullptr, + void *error_addr = nullptr) { + load_here(depth + 8, context, error_addr); + + for (size_t i = 0; i < _stacktrace.size(); ++i) { + if (_stacktrace[i] == addr) { + skip_n_firsts(i); + break; + } + } + + _stacktrace.resize(std::min(_stacktrace.size(), skip_n_firsts() + depth)); + return size(); + } + +private: + struct callback { + StackTraceImpl &self; + callback(StackTraceImpl &_self) : self(_self) {} + + void operator()(size_t idx, void *addr) { self._stacktrace[idx] = addr; } + }; +}; + +#elif BACKWARD_HAS_LIBUNWIND == 1 + +template <> +class StackTraceImpl : public StackTraceImplHolder { +public: + __attribute__((noinline)) size_t load_here(size_t depth = 32, + void *_context = nullptr, + void *_error_addr = nullptr) { + set_context(_context); + set_error_addr(_error_addr); + load_thread_info(); + if (depth == 0) { + return 0; + } + _stacktrace.resize(depth + 1); + + int result = 0; + + unw_context_t ctx; + size_t index = 0; + + // Add the tail call. If the Instruction Pointer is the crash address it + // means we got a bad function pointer dereference, so we "unwind" the + // bad pointer manually by using the return address pointed to by the + // Stack Pointer as the Instruction Pointer and letting libunwind do + // the rest + + if (context()) { + ucontext_t *uctx = reinterpret_cast(context()); +#ifdef REG_RIP // x86_64 + if (uctx->uc_mcontext.gregs[REG_RIP] == + reinterpret_cast(error_addr())) { + uctx->uc_mcontext.gregs[REG_RIP] = + *reinterpret_cast(uctx->uc_mcontext.gregs[REG_RSP]); + } + _stacktrace[index] = + reinterpret_cast(uctx->uc_mcontext.gregs[REG_RIP]); + ++index; + ctx = *reinterpret_cast(uctx); +#elif defined(REG_EIP) // x86_32 + if (uctx->uc_mcontext.gregs[REG_EIP] == + reinterpret_cast(error_addr())) { + uctx->uc_mcontext.gregs[REG_EIP] = + *reinterpret_cast(uctx->uc_mcontext.gregs[REG_ESP]); + } + _stacktrace[index] = + reinterpret_cast(uctx->uc_mcontext.gregs[REG_EIP]); + ++index; + ctx = *reinterpret_cast(uctx); +#elif defined(__arm__) // clang libunwind/arm + // libunwind uses its own context type for ARM unwinding. + // Copy the registers from the signal handler's context so we can + // unwind + unw_getcontext(&ctx); + ctx.regs[UNW_ARM_R0] = uctx->uc_mcontext.arm_r0; + ctx.regs[UNW_ARM_R1] = uctx->uc_mcontext.arm_r1; + ctx.regs[UNW_ARM_R2] = uctx->uc_mcontext.arm_r2; + ctx.regs[UNW_ARM_R3] = uctx->uc_mcontext.arm_r3; + ctx.regs[UNW_ARM_R4] = uctx->uc_mcontext.arm_r4; + ctx.regs[UNW_ARM_R5] = uctx->uc_mcontext.arm_r5; + ctx.regs[UNW_ARM_R6] = uctx->uc_mcontext.arm_r6; + ctx.regs[UNW_ARM_R7] = uctx->uc_mcontext.arm_r7; + ctx.regs[UNW_ARM_R8] = uctx->uc_mcontext.arm_r8; + ctx.regs[UNW_ARM_R9] = uctx->uc_mcontext.arm_r9; + ctx.regs[UNW_ARM_R10] = uctx->uc_mcontext.arm_r10; + ctx.regs[UNW_ARM_R11] = uctx->uc_mcontext.arm_fp; + ctx.regs[UNW_ARM_R12] = uctx->uc_mcontext.arm_ip; + ctx.regs[UNW_ARM_R13] = uctx->uc_mcontext.arm_sp; + ctx.regs[UNW_ARM_R14] = uctx->uc_mcontext.arm_lr; + ctx.regs[UNW_ARM_R15] = uctx->uc_mcontext.arm_pc; + + // If we have crashed in the PC use the LR instead, as this was + // a bad function dereference + if (reinterpret_cast(error_addr()) == + uctx->uc_mcontext.arm_pc) { + ctx.regs[UNW_ARM_R15] = + uctx->uc_mcontext.arm_lr - sizeof(unsigned long); + } + _stacktrace[index] = reinterpret_cast(ctx.regs[UNW_ARM_R15]); + ++index; +#elif defined(__aarch64__) // gcc libunwind/arm64 + unw_getcontext(&ctx); + // If the IP is the same as the crash address we have a bad function + // dereference The caller's address is pointed to by the link pointer, so + // we dereference that value and set it to be the next frame's IP. + if (uctx->uc_mcontext.pc == reinterpret_cast<__uint64_t>(error_addr())) { + uctx->uc_mcontext.pc = uctx->uc_mcontext.regs[UNW_TDEP_IP]; + } + + // 29 general purpose registers + for (int i = UNW_AARCH64_X0; i <= UNW_AARCH64_X28; i++) { + ctx.uc_mcontext.regs[i] = uctx->uc_mcontext.regs[i]; + } + ctx.uc_mcontext.sp = uctx->uc_mcontext.sp; + ctx.uc_mcontext.pc = uctx->uc_mcontext.pc; + ctx.uc_mcontext.fault_address = uctx->uc_mcontext.fault_address; + _stacktrace[index] = reinterpret_cast(ctx.uc_mcontext.pc); + ++index; +#elif defined(__APPLE__) && defined(__x86_64__) + unw_getcontext(&ctx); + // OS X's implementation of libunwind uses its own context object + // so we need to convert the passed context to libunwind's format + // (information about the data layout taken from unw_getcontext.s + // in Apple's libunwind source + ctx.data[0] = uctx->uc_mcontext->__ss.__rax; + ctx.data[1] = uctx->uc_mcontext->__ss.__rbx; + ctx.data[2] = uctx->uc_mcontext->__ss.__rcx; + ctx.data[3] = uctx->uc_mcontext->__ss.__rdx; + ctx.data[4] = uctx->uc_mcontext->__ss.__rdi; + ctx.data[5] = uctx->uc_mcontext->__ss.__rsi; + ctx.data[6] = uctx->uc_mcontext->__ss.__rbp; + ctx.data[7] = uctx->uc_mcontext->__ss.__rsp; + ctx.data[8] = uctx->uc_mcontext->__ss.__r8; + ctx.data[9] = uctx->uc_mcontext->__ss.__r9; + ctx.data[10] = uctx->uc_mcontext->__ss.__r10; + ctx.data[11] = uctx->uc_mcontext->__ss.__r11; + ctx.data[12] = uctx->uc_mcontext->__ss.__r12; + ctx.data[13] = uctx->uc_mcontext->__ss.__r13; + ctx.data[14] = uctx->uc_mcontext->__ss.__r14; + ctx.data[15] = uctx->uc_mcontext->__ss.__r15; + ctx.data[16] = uctx->uc_mcontext->__ss.__rip; + + // If the IP is the same as the crash address we have a bad function + // dereference The caller's address is pointed to by %rsp, so we + // dereference that value and set it to be the next frame's IP. + if (uctx->uc_mcontext->__ss.__rip == + reinterpret_cast<__uint64_t>(error_addr())) { + ctx.data[16] = + *reinterpret_cast<__uint64_t *>(uctx->uc_mcontext->__ss.__rsp); + } + _stacktrace[index] = reinterpret_cast(ctx.data[16]); + ++index; +#elif defined(__APPLE__) + unw_getcontext(&ctx) + // TODO: Convert the ucontext_t to libunwind's unw_context_t like + // we do in 64 bits + if (ctx.uc_mcontext->__ss.__eip == + reinterpret_cast(error_addr())) { + ctx.uc_mcontext->__ss.__eip = ctx.uc_mcontext->__ss.__esp; + } + _stacktrace[index] = + reinterpret_cast(ctx.uc_mcontext->__ss.__eip); + ++index; +#endif + } + + unw_cursor_t cursor; + if (context()) { +#if defined(UNW_INIT_SIGNAL_FRAME) + result = unw_init_local2(&cursor, &ctx, UNW_INIT_SIGNAL_FRAME); +#else + result = unw_init_local(&cursor, &ctx); +#endif + } else { + unw_getcontext(&ctx); + ; + result = unw_init_local(&cursor, &ctx); + } + + if (result != 0) + return 1; + + unw_word_t ip = 0; + + while (index <= depth && unw_step(&cursor) > 0) { + result = unw_get_reg(&cursor, UNW_REG_IP, &ip); + if (result == 0) { + _stacktrace[index] = reinterpret_cast(--ip); + ++index; + } + } + --index; + + _stacktrace.resize(index + 1); + skip_n_firsts(0); + return size(); + } + + size_t load_from(void *addr, size_t depth = 32, void *context = nullptr, + void *error_addr = nullptr) { + load_here(depth + 8, context, error_addr); + + for (size_t i = 0; i < _stacktrace.size(); ++i) { + if (_stacktrace[i] == addr) { + skip_n_firsts(i); + _stacktrace[i] = (void *)((uintptr_t)_stacktrace[i]); + break; + } + } + + _stacktrace.resize(std::min(_stacktrace.size(), skip_n_firsts() + depth)); + return size(); + } +}; + +#elif defined(BACKWARD_HAS_BACKTRACE) + +template <> +class StackTraceImpl : public StackTraceImplHolder { +public: + NOINLINE + size_t load_here(size_t depth = 32, void *context = nullptr, + void *error_addr = nullptr) { + set_context(context); + set_error_addr(error_addr); + load_thread_info(); + if (depth == 0) { + return 0; + } + _stacktrace.resize(depth + 1); + size_t trace_cnt = backtrace(&_stacktrace[0], _stacktrace.size()); + _stacktrace.resize(trace_cnt); + skip_n_firsts(1); + return size(); + } + + size_t load_from(void *addr, size_t depth = 32, void *context = nullptr, + void *error_addr = nullptr) { + load_here(depth + 8, context, error_addr); + + for (size_t i = 0; i < _stacktrace.size(); ++i) { + if (_stacktrace[i] == addr) { + skip_n_firsts(i); + _stacktrace[i] = (void *)((uintptr_t)_stacktrace[i] + 1); + break; + } + } + + _stacktrace.resize(std::min(_stacktrace.size(), skip_n_firsts() + depth)); + return size(); + } +}; + +#elif defined(BACKWARD_SYSTEM_WINDOWS) + +template <> +class StackTraceImpl : public StackTraceImplHolder { +public: + // We have to load the machine type from the image info + // So we first initialize the resolver, and it tells us this info + void set_machine_type(DWORD machine_type) { machine_type_ = machine_type; } + void set_context(CONTEXT *ctx) { ctx_ = ctx; } + void set_thread_handle(HANDLE handle) { thd_ = handle; } + + NOINLINE + size_t load_here(size_t depth = 32, void *context = nullptr, + void *error_addr = nullptr) { + set_context(static_cast(context)); + set_error_addr(error_addr); + CONTEXT localCtx; // used when no context is provided + + if (depth == 0) { + return 0; + } + + if (!ctx_) { + ctx_ = &localCtx; + RtlCaptureContext(ctx_); + } + + if (!thd_) { + thd_ = GetCurrentThread(); + } + + HANDLE process = GetCurrentProcess(); + + STACKFRAME64 s; + memset(&s, 0, sizeof(STACKFRAME64)); + + // TODO: 32 bit context capture + s.AddrStack.Mode = AddrModeFlat; + s.AddrFrame.Mode = AddrModeFlat; + s.AddrPC.Mode = AddrModeFlat; +#if defined(_M_X64) + s.AddrPC.Offset = ctx_->Rip; + s.AddrStack.Offset = ctx_->Rsp; + s.AddrFrame.Offset = ctx_->Rbp; +#elif defined(_M_ARM64) + s.AddrPC.Offset = ctx_->Pc; + s.AddrStack.Offset = ctx_->Sp; + s.AddrFrame.Offset = ctx_->Fp; +#elif defined(_M_ARM) + s.AddrPC.Offset = ctx_->Pc; + s.AddrStack.Offset = ctx_->Sp; + s.AddrFrame.Offset = ctx_->R11; +#else + s.AddrPC.Offset = ctx_->Eip; + s.AddrStack.Offset = ctx_->Esp; + s.AddrFrame.Offset = ctx_->Ebp; +#endif + + if (!machine_type_) { +#if defined(_M_X64) + machine_type_ = IMAGE_FILE_MACHINE_AMD64; +#elif defined(_M_ARM64) + machine_type_ = IMAGE_FILE_MACHINE_ARM64; +#elif defined(_M_ARM) + machine_type_ = IMAGE_FILE_MACHINE_ARMNT; +#else + machine_type_ = IMAGE_FILE_MACHINE_I386; +#endif + } + + for (;;) { + // NOTE: this only works if PDBs are already loaded! + SetLastError(0); + if (!StackWalk64(machine_type_, process, thd_, &s, ctx_, NULL, + SymFunctionTableAccess64, SymGetModuleBase64, NULL)) + break; + + if (s.AddrReturn.Offset == 0) + break; + + _stacktrace.push_back(reinterpret_cast(s.AddrPC.Offset)); + + if (size() >= depth) + break; + } + + return size(); + } + + size_t load_from(void *addr, size_t depth = 32, void *context = nullptr, + void *error_addr = nullptr) { + load_here(depth + 8, context, error_addr); + + for (size_t i = 0; i < _stacktrace.size(); ++i) { + if (_stacktrace[i] == addr) { + skip_n_firsts(i); + break; + } + } + + _stacktrace.resize(std::min(_stacktrace.size(), skip_n_firsts() + depth)); + return size(); + } + +private: + DWORD machine_type_ = 0; + HANDLE thd_ = 0; + CONTEXT *ctx_ = nullptr; +}; + +#endif + +class StackTrace : public StackTraceImpl {}; + +/*************** TRACE RESOLVER ***************/ + +class TraceResolverImplBase { +public: + virtual ~TraceResolverImplBase() {} + + virtual void load_addresses(void *const*addresses, int address_count) { + (void)addresses; + (void)address_count; + } + + template void load_stacktrace(ST &st) { + load_addresses(st.begin(), static_cast(st.size())); + } + + virtual ResolvedTrace resolve(ResolvedTrace t) { return t; } + +protected: + std::string demangle(const char *funcname) { + return _demangler.demangle(funcname); + } + +private: + details::demangler _demangler; +}; + +template class TraceResolverImpl; + +#ifdef BACKWARD_SYSTEM_UNKNOWN + +template <> class TraceResolverImpl + : public TraceResolverImplBase {}; + +#endif + +#ifdef BACKWARD_SYSTEM_LINUX + +class TraceResolverLinuxBase : public TraceResolverImplBase { +public: + TraceResolverLinuxBase() + : argv0_(get_argv0()), exec_path_(read_symlink("/proc/self/exe")) {} + std::string resolve_exec_path(Dl_info &symbol_info) const { + // mutates symbol_info.dli_fname to be filename to open and returns filename + // to display + if (symbol_info.dli_fname == argv0_) { + // dladdr returns argv[0] in dli_fname for symbols contained in + // the main executable, which is not a valid path if the + // executable was found by a search of the PATH environment + // variable; In that case, we actually open /proc/self/exe, which + // is always the actual executable (even if it was deleted/replaced!) + // but display the path that /proc/self/exe links to. + // However, this right away reduces probability of successful symbol + // resolution, because libbfd may try to find *.debug files in the + // same dir, in case symbols are stripped. As a result, it may try + // to find a file /proc/self/.debug, which obviously does + // not exist. /proc/self/exe is a last resort. First load attempt + // should go for the original executable file path. + symbol_info.dli_fname = "/proc/self/exe"; + return exec_path_; + } else { + return symbol_info.dli_fname; + } + } + +private: + std::string argv0_; + std::string exec_path_; + + static std::string get_argv0() { + std::string argv0; + std::ifstream ifs("/proc/self/cmdline"); + std::getline(ifs, argv0, '\0'); + return argv0; + } + + static std::string read_symlink(std::string const &symlink_path) { + std::string path; + path.resize(100); + + while (true) { + ssize_t len = + ::readlink(symlink_path.c_str(), &*path.begin(), path.size()); + if (len < 0) { + return ""; + } + if (static_cast(len) == path.size()) { + path.resize(path.size() * 2); + } else { + path.resize(static_cast(len)); + break; + } + } + + return path; + } +}; + +template class TraceResolverLinuxImpl; + +#if BACKWARD_HAS_BACKTRACE_SYMBOL == 1 + +template <> +class TraceResolverLinuxImpl + : public TraceResolverLinuxBase { +public: + void load_addresses(void *const*addresses, int address_count) override { + if (address_count == 0) { + return; + } + _symbols.reset(backtrace_symbols(addresses, address_count)); + } + + ResolvedTrace resolve(ResolvedTrace trace) override { + char *filename = _symbols[trace.idx]; + char *funcname = filename; + while (*funcname && *funcname != '(') { + funcname += 1; + } + trace.object_filename.assign(filename, + funcname); // ok even if funcname is the ending + // \0 (then we assign entire string) + + if (*funcname) { // if it's not end of string (e.g. from last frame ip==0) + funcname += 1; + char *funcname_end = funcname; + while (*funcname_end && *funcname_end != ')' && *funcname_end != '+') { + funcname_end += 1; + } + *funcname_end = '\0'; + trace.object_function = this->demangle(funcname); + trace.source.function = trace.object_function; // we cannot do better. + } + return trace; + } + +private: + details::handle _symbols; +}; + +#endif // BACKWARD_HAS_BACKTRACE_SYMBOL == 1 + +#if BACKWARD_HAS_BFD == 1 + +template <> +class TraceResolverLinuxImpl + : public TraceResolverLinuxBase { +public: + TraceResolverLinuxImpl() : _bfd_loaded(false) {} + + ResolvedTrace resolve(ResolvedTrace trace) override { + Dl_info symbol_info; + + // trace.addr is a virtual address in memory pointing to some code. + // Let's try to find from which loaded object it comes from. + // The loaded object can be yourself btw. + if (!dladdr(trace.addr, &symbol_info)) { + return trace; // dat broken trace... + } + + // Now we get in symbol_info: + // .dli_fname: + // pathname of the shared object that contains the address. + // .dli_fbase: + // where the object is loaded in memory. + // .dli_sname: + // the name of the nearest symbol to trace.addr, we expect a + // function name. + // .dli_saddr: + // the exact address corresponding to .dli_sname. + + if (symbol_info.dli_sname) { + trace.object_function = demangle(symbol_info.dli_sname); + } + + if (!symbol_info.dli_fname) { + return trace; + } + + trace.object_filename = resolve_exec_path(symbol_info); + bfd_fileobject *fobj; + // Before rushing to resolution need to ensure the executable + // file still can be used. For that compare inode numbers of + // what is stored by the executable's file path, and in the + // dli_fname, which not necessarily equals to the executable. + // It can be a shared library, or /proc/self/exe, and in the + // latter case has drawbacks. See the exec path resolution for + // details. In short - the dli object should be used only as + // the last resort. + // If inode numbers are equal, it is known dli_fname and the + // executable file are the same. This is guaranteed by Linux, + // because if the executable file is changed/deleted, it will + // be done in a new inode. The old file will be preserved in + // /proc/self/exe, and may even have inode 0. The latter can + // happen if the inode was actually reused, and the file was + // kept only in the main memory. + // + struct stat obj_stat; + struct stat dli_stat; + if (stat(trace.object_filename.c_str(), &obj_stat) == 0 && + stat(symbol_info.dli_fname, &dli_stat) == 0 && + obj_stat.st_ino == dli_stat.st_ino) { + // The executable file, and the shared object containing the + // address are the same file. Safe to use the original path. + // this is preferable. Libbfd will search for stripped debug + // symbols in the same directory. + fobj = load_object_with_bfd(trace.object_filename); + } else{ + // The original object file was *deleted*! The only hope is + // that the debug symbols are either inside the shared + // object file, or are in the same directory, and this is + // not /proc/self/exe. + fobj = nullptr; + } + if (fobj == nullptr || !fobj->handle) { + fobj = load_object_with_bfd(symbol_info.dli_fname); + if (!fobj->handle) { + return trace; + } + } + + find_sym_result *details_selected; // to be filled. + + // trace.addr is the next instruction to be executed after returning + // from the nested stack frame. In C++ this usually relate to the next + // statement right after the function call that leaded to a new stack + // frame. This is not usually what you want to see when printing out a + // stacktrace... + find_sym_result details_call_site = + find_symbol_details(fobj, trace.addr, symbol_info.dli_fbase); + details_selected = &details_call_site; + +#if BACKWARD_HAS_UNWIND == 0 + // ...this is why we also try to resolve the symbol that is right + // before the return address. If we are lucky enough, we will get the + // line of the function that was called. But if the code is optimized, + // we might get something absolutely not related since the compiler + // can reschedule the return address with inline functions and + // tail-call optimization (among other things that I don't even know + // or cannot even dream about with my tiny limited brain). + find_sym_result details_adjusted_call_site = find_symbol_details( + fobj, (void *)(uintptr_t(trace.addr) - 1), symbol_info.dli_fbase); + + // In debug mode, we should always get the right thing(TM). + if (details_call_site.found && details_adjusted_call_site.found) { + // Ok, we assume that details_adjusted_call_site is a better estimation. + details_selected = &details_adjusted_call_site; + trace.addr = (void *)(uintptr_t(trace.addr) - 1); + } + + if (details_selected == &details_call_site && details_call_site.found) { + // we have to re-resolve the symbol in order to reset some + // internal state in BFD... so we can call backtrace_inliners + // thereafter... + details_call_site = + find_symbol_details(fobj, trace.addr, symbol_info.dli_fbase); + } +#endif // BACKWARD_HAS_UNWIND + + if (details_selected->found) { + if (details_selected->filename) { + trace.source.filename = details_selected->filename; + } + trace.source.line = details_selected->line; + + if (details_selected->funcname) { + // this time we get the name of the function where the code is + // located, instead of the function were the address is + // located. In short, if the code was inlined, we get the + // function corresponding to the code. Else we already got in + // trace.function. + trace.source.function = demangle(details_selected->funcname); + + if (!symbol_info.dli_sname) { + // for the case dladdr failed to find the symbol name of + // the function, we might as well try to put something + // here. + trace.object_function = trace.source.function; + } + } + + // Maybe the source of the trace got inlined inside the function + // (trace.source.function). Let's see if we can get all the inlined + // calls along the way up to the initial call site. + trace.inliners = backtrace_inliners(fobj, *details_selected); + +#if 0 + if (trace.inliners.size() == 0) { + // Maybe the trace was not inlined... or maybe it was and we + // are lacking the debug information. Let's try to make the + // world better and see if we can get the line number of the + // function (trace.source.function) now. + // + // We will get the location of where the function start (to be + // exact: the first instruction that really start the + // function), not where the name of the function is defined. + // This can be quite far away from the name of the function + // btw. + // + // If the source of the function is the same as the source of + // the trace, we cannot say if the trace was really inlined or + // not. However, if the filename of the source is different + // between the function and the trace... we can declare it as + // an inliner. This is not 100% accurate, but better than + // nothing. + + if (symbol_info.dli_saddr) { + find_sym_result details = find_symbol_details(fobj, + symbol_info.dli_saddr, + symbol_info.dli_fbase); + + if (details.found) { + ResolvedTrace::SourceLoc diy_inliner; + diy_inliner.line = details.line; + if (details.filename) { + diy_inliner.filename = details.filename; + } + if (details.funcname) { + diy_inliner.function = demangle(details.funcname); + } else { + diy_inliner.function = trace.source.function; + } + if (diy_inliner != trace.source) { + trace.inliners.push_back(diy_inliner); + } + } + } + } +#endif + } + + return trace; + } + +private: + bool _bfd_loaded; + + typedef details::handle > + bfd_handle_t; + + typedef details::handle bfd_symtab_t; + + struct bfd_fileobject { + bfd_handle_t handle; + bfd_vma base_addr; + bfd_symtab_t symtab; + bfd_symtab_t dynamic_symtab; + }; + + typedef details::hashtable::type fobj_bfd_map_t; + fobj_bfd_map_t _fobj_bfd_map; + + bfd_fileobject *load_object_with_bfd(const std::string &filename_object) { + using namespace details; + + if (!_bfd_loaded) { + using namespace details; + bfd_init(); + _bfd_loaded = true; + } + + fobj_bfd_map_t::iterator it = _fobj_bfd_map.find(filename_object); + if (it != _fobj_bfd_map.end()) { + return &it->second; + } + + // this new object is empty for now. + bfd_fileobject *r = &_fobj_bfd_map[filename_object]; + + // we do the work temporary in this one; + bfd_handle_t bfd_handle; + + int fd = open(filename_object.c_str(), O_RDONLY); + bfd_handle.reset(bfd_fdopenr(filename_object.c_str(), "default", fd)); + if (!bfd_handle) { + close(fd); + return r; + } + + if (!bfd_check_format(bfd_handle.get(), bfd_object)) { + return r; // not an object? You lose. + } + + if ((bfd_get_file_flags(bfd_handle.get()) & HAS_SYMS) == 0) { + return r; // that's what happen when you forget to compile in debug. + } + + ssize_t symtab_storage_size = bfd_get_symtab_upper_bound(bfd_handle.get()); + + ssize_t dyn_symtab_storage_size = + bfd_get_dynamic_symtab_upper_bound(bfd_handle.get()); + + if (symtab_storage_size <= 0 && dyn_symtab_storage_size <= 0) { + return r; // weird, is the file is corrupted? + } + + bfd_symtab_t symtab, dynamic_symtab; + ssize_t symcount = 0, dyn_symcount = 0; + + if (symtab_storage_size > 0) { + symtab.reset(static_cast( + malloc(static_cast(symtab_storage_size)))); + symcount = bfd_canonicalize_symtab(bfd_handle.get(), symtab.get()); + } + + if (dyn_symtab_storage_size > 0) { + dynamic_symtab.reset(static_cast( + malloc(static_cast(dyn_symtab_storage_size)))); + dyn_symcount = bfd_canonicalize_dynamic_symtab(bfd_handle.get(), + dynamic_symtab.get()); + } + + if (symcount <= 0 && dyn_symcount <= 0) { + return r; // damned, that's a stripped file that you got there! + } + + r->handle = move(bfd_handle); + r->symtab = move(symtab); + r->dynamic_symtab = move(dynamic_symtab); + return r; + } + + struct find_sym_result { + bool found; + const char *filename; + const char *funcname; + unsigned int line; + }; + + struct find_sym_context { + TraceResolverLinuxImpl *self; + bfd_fileobject *fobj; + void *addr; + void *base_addr; + find_sym_result result; + }; + + find_sym_result find_symbol_details(bfd_fileobject *fobj, void *addr, + void *base_addr) { + find_sym_context context; + context.self = this; + context.fobj = fobj; + context.addr = addr; + context.base_addr = base_addr; + context.result.found = false; + bfd_map_over_sections(fobj->handle.get(), &find_in_section_trampoline, + static_cast(&context)); + return context.result; + } + + static void find_in_section_trampoline(bfd *, asection *section, void *data) { + find_sym_context *context = static_cast(data); + context->self->find_in_section( + reinterpret_cast(context->addr), + reinterpret_cast(context->base_addr), context->fobj, section, + context->result); + } + + void find_in_section(bfd_vma addr, bfd_vma base_addr, bfd_fileobject *fobj, + asection *section, find_sym_result &result) { + if (result.found) + return; + +#ifdef bfd_get_section_flags + if ((bfd_get_section_flags(fobj->handle.get(), section) & SEC_ALLOC) == 0) +#else + if ((bfd_section_flags(section) & SEC_ALLOC) == 0) +#endif + return; // a debug section is never loaded automatically. + +#ifdef bfd_get_section_vma + bfd_vma sec_addr = bfd_get_section_vma(fobj->handle.get(), section); +#else + bfd_vma sec_addr = bfd_section_vma(section); +#endif +#ifdef bfd_get_section_size + bfd_size_type size = bfd_get_section_size(section); +#else + bfd_size_type size = bfd_section_size(section); +#endif + + // are we in the boundaries of the section? + if (addr < sec_addr || addr >= sec_addr + size) { + addr -= base_addr; // oops, a relocated object, lets try again... + if (addr < sec_addr || addr >= sec_addr + size) { + return; + } + } + +#if defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant" +#endif + if (!result.found && fobj->symtab) { + result.found = bfd_find_nearest_line( + fobj->handle.get(), section, fobj->symtab.get(), addr - sec_addr, + &result.filename, &result.funcname, &result.line); + } + + if (!result.found && fobj->dynamic_symtab) { + result.found = bfd_find_nearest_line( + fobj->handle.get(), section, fobj->dynamic_symtab.get(), + addr - sec_addr, &result.filename, &result.funcname, &result.line); + } +#if defined(__clang__) +#pragma clang diagnostic pop +#endif + } + + ResolvedTrace::source_locs_t + backtrace_inliners(bfd_fileobject *fobj, find_sym_result previous_result) { + // This function can be called ONLY after a SUCCESSFUL call to + // find_symbol_details. The state is global to the bfd_handle. + ResolvedTrace::source_locs_t results; + while (previous_result.found) { + find_sym_result result; + result.found = bfd_find_inliner_info(fobj->handle.get(), &result.filename, + &result.funcname, &result.line); + + if (result + .found) /* and not ( + cstrings_eq(previous_result.filename, + result.filename) and + cstrings_eq(previous_result.funcname, result.funcname) + and result.line == previous_result.line + )) */ + { + ResolvedTrace::SourceLoc src_loc; + src_loc.line = result.line; + if (result.filename) { + src_loc.filename = result.filename; + } + if (result.funcname) { + src_loc.function = demangle(result.funcname); + } + results.push_back(src_loc); + } + previous_result = result; + } + return results; + } + + bool cstrings_eq(const char *a, const char *b) { + if (!a || !b) { + return false; + } + return strcmp(a, b) == 0; + } +}; +#endif // BACKWARD_HAS_BFD == 1 + +#if BACKWARD_HAS_DW == 1 + +template <> +class TraceResolverLinuxImpl + : public TraceResolverLinuxBase { +public: + TraceResolverLinuxImpl() : _dwfl_handle_initialized(false) {} + + ResolvedTrace resolve(ResolvedTrace trace) override { + using namespace details; + + Dwarf_Addr trace_addr = reinterpret_cast(trace.addr); + + if (!_dwfl_handle_initialized) { + // initialize dwfl... + _dwfl_cb.reset(new Dwfl_Callbacks); + _dwfl_cb->find_elf = &dwfl_linux_proc_find_elf; + _dwfl_cb->find_debuginfo = &dwfl_standard_find_debuginfo; + _dwfl_cb->debuginfo_path = 0; + + _dwfl_handle.reset(dwfl_begin(_dwfl_cb.get())); + _dwfl_handle_initialized = true; + + if (!_dwfl_handle) { + return trace; + } + + // ...from the current process. + dwfl_report_begin(_dwfl_handle.get()); + int r = dwfl_linux_proc_report(_dwfl_handle.get(), getpid()); + dwfl_report_end(_dwfl_handle.get(), NULL, NULL); + if (r < 0) { + return trace; + } + } + + if (!_dwfl_handle) { + return trace; + } + + // find the module (binary object) that contains the trace's address. + // This is not using any debug information, but the addresses ranges of + // all the currently loaded binary object. + Dwfl_Module *mod = dwfl_addrmodule(_dwfl_handle.get(), trace_addr); + if (mod) { + // now that we found it, lets get the name of it, this will be the + // full path to the running binary or one of the loaded library. + const char *module_name = dwfl_module_info(mod, 0, 0, 0, 0, 0, 0, 0); + if (module_name) { + trace.object_filename = module_name; + } + // We also look after the name of the symbol, equal or before this + // address. This is found by walking the symtab. We should get the + // symbol corresponding to the function (mangled) containing the + // address. If the code corresponding to the address was inlined, + // this is the name of the out-most inliner function. + const char *sym_name = dwfl_module_addrname(mod, trace_addr); + if (sym_name) { + trace.object_function = demangle(sym_name); + } + } + + // now let's get serious, and find out the source location (file and + // line number) of the address. + + // This function will look in .debug_aranges for the address and map it + // to the location of the compilation unit DIE in .debug_info and + // return it. + Dwarf_Addr mod_bias = 0; + Dwarf_Die *cudie = dwfl_module_addrdie(mod, trace_addr, &mod_bias); + +#if 1 + if (!cudie) { + // Sadly clang does not generate the section .debug_aranges, thus + // dwfl_module_addrdie will fail early. Clang doesn't either set + // the lowpc/highpc/range info for every compilation unit. + // + // So in order to save the world: + // for every compilation unit, we will iterate over every single + // DIEs. Normally functions should have a lowpc/highpc/range, which + // we will use to infer the compilation unit. + + // note that this is probably badly inefficient. + while ((cudie = dwfl_module_nextcu(mod, cudie, &mod_bias))) { + Dwarf_Die die_mem; + Dwarf_Die *fundie = + find_fundie_by_pc(cudie, trace_addr - mod_bias, &die_mem); + if (fundie) { + break; + } + } + } +#endif + +//#define BACKWARD_I_DO_NOT_RECOMMEND_TO_ENABLE_THIS_HORRIBLE_PIECE_OF_CODE +#ifdef BACKWARD_I_DO_NOT_RECOMMEND_TO_ENABLE_THIS_HORRIBLE_PIECE_OF_CODE + if (!cudie) { + // If it's still not enough, lets dive deeper in the shit, and try + // to save the world again: for every compilation unit, we will + // load the corresponding .debug_line section, and see if we can + // find our address in it. + + Dwarf_Addr cfi_bias; + Dwarf_CFI *cfi_cache = dwfl_module_eh_cfi(mod, &cfi_bias); + + Dwarf_Addr bias; + while ((cudie = dwfl_module_nextcu(mod, cudie, &bias))) { + if (dwarf_getsrc_die(cudie, trace_addr - bias)) { + + // ...but if we get a match, it might be a false positive + // because our (address - bias) might as well be valid in a + // different compilation unit. So we throw our last card on + // the table and lookup for the address into the .eh_frame + // section. + + handle frame; + dwarf_cfi_addrframe(cfi_cache, trace_addr - cfi_bias, &frame); + if (frame) { + break; + } + } + } + } +#endif + + if (!cudie) { + return trace; // this time we lost the game :/ + } + + // Now that we have a compilation unit DIE, this function will be able + // to load the corresponding section in .debug_line (if not already + // loaded) and hopefully find the source location mapped to our + // address. + Dwarf_Line *srcloc = dwarf_getsrc_die(cudie, trace_addr - mod_bias); + + if (srcloc) { + const char *srcfile = dwarf_linesrc(srcloc, 0, 0); + if (srcfile) { + trace.source.filename = srcfile; + } + int line = 0, col = 0; + dwarf_lineno(srcloc, &line); + dwarf_linecol(srcloc, &col); + trace.source.line = static_cast(line); + trace.source.col = static_cast(col); + } + + deep_first_search_by_pc(cudie, trace_addr - mod_bias, + inliners_search_cb(trace)); + if (trace.source.function.size() == 0) { + // fallback. + trace.source.function = trace.object_function; + } + + return trace; + } + +private: + typedef details::handle > + dwfl_handle_t; + details::handle > + _dwfl_cb; + dwfl_handle_t _dwfl_handle; + bool _dwfl_handle_initialized; + + // defined here because in C++98, template function cannot take locally + // defined types... grrr. + struct inliners_search_cb { + void operator()(Dwarf_Die *die) { + switch (dwarf_tag(die)) { + const char *name; + case DW_TAG_subprogram: + if ((name = dwarf_diename(die))) { + trace.source.function = name; + } + break; + + case DW_TAG_inlined_subroutine: + ResolvedTrace::SourceLoc sloc; + Dwarf_Attribute attr_mem; + + if ((name = dwarf_diename(die))) { + sloc.function = name; + } + if ((name = die_call_file(die))) { + sloc.filename = name; + } + + Dwarf_Word line = 0, col = 0; + dwarf_formudata(dwarf_attr(die, DW_AT_call_line, &attr_mem), &line); + dwarf_formudata(dwarf_attr(die, DW_AT_call_column, &attr_mem), &col); + sloc.line = static_cast(line); + sloc.col = static_cast(col); + + trace.inliners.push_back(sloc); + break; + }; + } + ResolvedTrace &trace; + inliners_search_cb(ResolvedTrace &t) : trace(t) {} + }; + + static bool die_has_pc(Dwarf_Die *die, Dwarf_Addr pc) { + Dwarf_Addr low, high; + + // continuous range + if (dwarf_hasattr(die, DW_AT_low_pc) && dwarf_hasattr(die, DW_AT_high_pc)) { + if (dwarf_lowpc(die, &low) != 0) { + return false; + } + if (dwarf_highpc(die, &high) != 0) { + Dwarf_Attribute attr_mem; + Dwarf_Attribute *attr = dwarf_attr(die, DW_AT_high_pc, &attr_mem); + Dwarf_Word value; + if (dwarf_formudata(attr, &value) != 0) { + return false; + } + high = low + value; + } + return pc >= low && pc < high; + } + + // non-continuous range. + Dwarf_Addr base; + ptrdiff_t offset = 0; + while ((offset = dwarf_ranges(die, offset, &base, &low, &high)) > 0) { + if (pc >= low && pc < high) { + return true; + } + } + return false; + } + + static Dwarf_Die *find_fundie_by_pc(Dwarf_Die *parent_die, Dwarf_Addr pc, + Dwarf_Die *result) { + if (dwarf_child(parent_die, result) != 0) { + return 0; + } + + Dwarf_Die *die = result; + do { + switch (dwarf_tag(die)) { + case DW_TAG_subprogram: + case DW_TAG_inlined_subroutine: + if (die_has_pc(die, pc)) { + return result; + } + }; + bool declaration = false; + Dwarf_Attribute attr_mem; + dwarf_formflag(dwarf_attr(die, DW_AT_declaration, &attr_mem), + &declaration); + if (!declaration) { + // let's be curious and look deeper in the tree, + // function are not necessarily at the first level, but + // might be nested inside a namespace, structure etc. + Dwarf_Die die_mem; + Dwarf_Die *indie = find_fundie_by_pc(die, pc, &die_mem); + if (indie) { + *result = die_mem; + return result; + } + } + } while (dwarf_siblingof(die, result) == 0); + return 0; + } + + template + static bool deep_first_search_by_pc(Dwarf_Die *parent_die, Dwarf_Addr pc, + CB cb) { + Dwarf_Die die_mem; + if (dwarf_child(parent_die, &die_mem) != 0) { + return false; + } + + bool branch_has_pc = false; + Dwarf_Die *die = &die_mem; + do { + bool declaration = false; + Dwarf_Attribute attr_mem; + dwarf_formflag(dwarf_attr(die, DW_AT_declaration, &attr_mem), + &declaration); + if (!declaration) { + // let's be curious and look deeper in the tree, function are + // not necessarily at the first level, but might be nested + // inside a namespace, structure, a function, an inlined + // function etc. + branch_has_pc = deep_first_search_by_pc(die, pc, cb); + } + if (!branch_has_pc) { + branch_has_pc = die_has_pc(die, pc); + } + if (branch_has_pc) { + cb(die); + } + } while (dwarf_siblingof(die, &die_mem) == 0); + return branch_has_pc; + } + + static const char *die_call_file(Dwarf_Die *die) { + Dwarf_Attribute attr_mem; + Dwarf_Word file_idx = 0; + + dwarf_formudata(dwarf_attr(die, DW_AT_call_file, &attr_mem), &file_idx); + + if (file_idx == 0) { + return 0; + } + + Dwarf_Die die_mem; + Dwarf_Die *cudie = dwarf_diecu(die, &die_mem, 0, 0); + if (!cudie) { + return 0; + } + + Dwarf_Files *files = 0; + size_t nfiles; + dwarf_getsrcfiles(cudie, &files, &nfiles); + if (!files) { + return 0; + } + + return dwarf_filesrc(files, file_idx, 0, 0); + } +}; +#endif // BACKWARD_HAS_DW == 1 + +#if BACKWARD_HAS_DWARF == 1 + +template <> +class TraceResolverLinuxImpl + : public TraceResolverLinuxBase { +public: + TraceResolverLinuxImpl() : _dwarf_loaded(false) {} + + ResolvedTrace resolve(ResolvedTrace trace) override { + // trace.addr is a virtual address in memory pointing to some code. + // Let's try to find from which loaded object it comes from. + // The loaded object can be yourself btw. + + Dl_info symbol_info; + int dladdr_result = 0; +#if defined(__GLIBC__) + link_map *link_map; + // We request the link map so we can get information about offsets + dladdr_result = + dladdr1(trace.addr, &symbol_info, reinterpret_cast(&link_map), + RTLD_DL_LINKMAP); +#else + // Android doesn't have dladdr1. Don't use the linker map. + dladdr_result = dladdr(trace.addr, &symbol_info); +#endif + if (!dladdr_result) { + return trace; // dat broken trace... + } + + // Now we get in symbol_info: + // .dli_fname: + // pathname of the shared object that contains the address. + // .dli_fbase: + // where the object is loaded in memory. + // .dli_sname: + // the name of the nearest symbol to trace.addr, we expect a + // function name. + // .dli_saddr: + // the exact address corresponding to .dli_sname. + // + // And in link_map: + // .l_addr: + // difference between the address in the ELF file and the address + // in memory + // l_name: + // absolute pathname where the object was found + + if (symbol_info.dli_sname) { + trace.object_function = demangle(symbol_info.dli_sname); + } + + if (!symbol_info.dli_fname) { + return trace; + } + + trace.object_filename = resolve_exec_path(symbol_info); + dwarf_fileobject &fobj = load_object_with_dwarf(symbol_info.dli_fname); + if (!fobj.dwarf_handle) { + return trace; // sad, we couldn't load the object :( + } + +#if defined(__GLIBC__) + // Convert the address to a module relative one by looking at + // the module's loading address in the link map + Dwarf_Addr address = reinterpret_cast(trace.addr) - + reinterpret_cast(link_map->l_addr); +#else + Dwarf_Addr address = reinterpret_cast(trace.addr); +#endif + + if (trace.object_function.empty()) { + symbol_cache_t::iterator it = fobj.symbol_cache.lower_bound(address); + + if (it != fobj.symbol_cache.end()) { + if (it->first != address) { + if (it != fobj.symbol_cache.begin()) { + --it; + } + } + trace.object_function = demangle(it->second.c_str()); + } + } + + // Get the Compilation Unit DIE for the address + Dwarf_Die die = find_die(fobj, address); + + if (!die) { + return trace; // this time we lost the game :/ + } + + // libdwarf doesn't give us direct access to its objects, it always + // allocates a copy for the caller. We keep that copy alive in a cache + // and we deallocate it later when it's no longer required. + die_cache_entry &die_object = get_die_cache(fobj, die); + if (die_object.isEmpty()) + return trace; // We have no line section for this DIE + + die_linemap_t::iterator it = die_object.line_section.lower_bound(address); + + if (it != die_object.line_section.end()) { + if (it->first != address) { + if (it == die_object.line_section.begin()) { + // If we are on the first item of the line section + // but the address does not match it means that + // the address is below the range of the DIE. Give up. + return trace; + } else { + --it; + } + } + } else { + return trace; // We didn't find the address. + } + + // Get the Dwarf_Line that the address points to and call libdwarf + // to get source file, line and column info. + Dwarf_Line line = die_object.line_buffer[it->second]; + Dwarf_Error error = DW_DLE_NE; + + char *filename; + if (dwarf_linesrc(line, &filename, &error) == DW_DLV_OK) { + trace.source.filename = std::string(filename); + dwarf_dealloc(fobj.dwarf_handle.get(), filename, DW_DLA_STRING); + } + + Dwarf_Unsigned number = 0; + if (dwarf_lineno(line, &number, &error) == DW_DLV_OK) { + trace.source.line = number; + } else { + trace.source.line = 0; + } + + if (dwarf_lineoff_b(line, &number, &error) == DW_DLV_OK) { + trace.source.col = number; + } else { + trace.source.col = 0; + } + + std::vector namespace_stack; + deep_first_search_by_pc(fobj, die, address, namespace_stack, + inliners_search_cb(trace, fobj, die)); + + dwarf_dealloc(fobj.dwarf_handle.get(), die, DW_DLA_DIE); + + return trace; + } + +public: + static int close_dwarf(Dwarf_Debug dwarf) { + return dwarf_finish(dwarf, NULL); + } + +private: + bool _dwarf_loaded; + + typedef details::handle > + dwarf_file_t; + + typedef details::handle > + dwarf_elf_t; + + typedef details::handle > + dwarf_handle_t; + + typedef std::map die_linemap_t; + + typedef std::map die_specmap_t; + + struct die_cache_entry { + die_specmap_t spec_section; + die_linemap_t line_section; + Dwarf_Line *line_buffer; + Dwarf_Signed line_count; + Dwarf_Line_Context line_context; + + inline bool isEmpty() { + return line_buffer == NULL || line_count == 0 || line_context == NULL || + line_section.empty(); + } + + die_cache_entry() : line_buffer(0), line_count(0), line_context(0) {} + + ~die_cache_entry() { + if (line_context) { + dwarf_srclines_dealloc_b(line_context); + } + } + }; + + typedef std::map die_cache_t; + + typedef std::map symbol_cache_t; + + struct dwarf_fileobject { + dwarf_file_t file_handle; + dwarf_elf_t elf_handle; + dwarf_handle_t dwarf_handle; + symbol_cache_t symbol_cache; + + // Die cache + die_cache_t die_cache; + die_cache_entry *current_cu; + }; + + typedef details::hashtable::type + fobj_dwarf_map_t; + fobj_dwarf_map_t _fobj_dwarf_map; + + static bool cstrings_eq(const char *a, const char *b) { + if (!a || !b) { + return false; + } + return strcmp(a, b) == 0; + } + + dwarf_fileobject &load_object_with_dwarf(const std::string &filename_object) { + + if (!_dwarf_loaded) { + // Set the ELF library operating version + // If that fails there's nothing we can do + _dwarf_loaded = elf_version(EV_CURRENT) != EV_NONE; + } + + fobj_dwarf_map_t::iterator it = _fobj_dwarf_map.find(filename_object); + if (it != _fobj_dwarf_map.end()) { + return it->second; + } + + // this new object is empty for now + dwarf_fileobject &r = _fobj_dwarf_map[filename_object]; + + dwarf_file_t file_handle; + file_handle.reset(open(filename_object.c_str(), O_RDONLY)); + if (file_handle.get() < 0) { + return r; + } + + // Try to get an ELF handle. We need to read the ELF sections + // because we want to see if there is a .gnu_debuglink section + // that points to a split debug file + dwarf_elf_t elf_handle; + elf_handle.reset(elf_begin(file_handle.get(), ELF_C_READ, NULL)); + if (!elf_handle) { + return r; + } + + const char *e_ident = elf_getident(elf_handle.get(), 0); + if (!e_ident) { + return r; + } + + // Get the number of sections + // We use the new APIs as elf_getshnum is deprecated + size_t shdrnum = 0; + if (elf_getshdrnum(elf_handle.get(), &shdrnum) == -1) { + return r; + } + + // Get the index to the string section + size_t shdrstrndx = 0; + if (elf_getshdrstrndx(elf_handle.get(), &shdrstrndx) == -1) { + return r; + } + + std::string debuglink; + // Iterate through the ELF sections to try to get a gnu_debuglink + // note and also to cache the symbol table. + // We go the preprocessor way to avoid having to create templated + // classes or using gelf (which might throw a compiler error if 64 bit + // is not supported +#define ELF_GET_DATA(ARCH) \ + Elf_Scn *elf_section = 0; \ + Elf_Data *elf_data = 0; \ + Elf##ARCH##_Shdr *section_header = 0; \ + Elf_Scn *symbol_section = 0; \ + size_t symbol_count = 0; \ + size_t symbol_strings = 0; \ + Elf##ARCH##_Sym *symbol = 0; \ + const char *section_name = 0; \ + \ + while ((elf_section = elf_nextscn(elf_handle.get(), elf_section)) != NULL) { \ + section_header = elf##ARCH##_getshdr(elf_section); \ + if (section_header == NULL) { \ + return r; \ + } \ + \ + if ((section_name = elf_strptr(elf_handle.get(), shdrstrndx, \ + section_header->sh_name)) == NULL) { \ + return r; \ + } \ + \ + if (cstrings_eq(section_name, ".gnu_debuglink")) { \ + elf_data = elf_getdata(elf_section, NULL); \ + if (elf_data && elf_data->d_size > 0) { \ + debuglink = \ + std::string(reinterpret_cast(elf_data->d_buf)); \ + } \ + } \ + \ + switch (section_header->sh_type) { \ + case SHT_SYMTAB: \ + symbol_section = elf_section; \ + symbol_count = section_header->sh_size / section_header->sh_entsize; \ + symbol_strings = section_header->sh_link; \ + break; \ + \ + /* We use .dynsyms as a last resort, we prefer .symtab */ \ + case SHT_DYNSYM: \ + if (!symbol_section) { \ + symbol_section = elf_section; \ + symbol_count = section_header->sh_size / section_header->sh_entsize; \ + symbol_strings = section_header->sh_link; \ + } \ + break; \ + } \ + } \ + \ + if (symbol_section && symbol_count && symbol_strings) { \ + elf_data = elf_getdata(symbol_section, NULL); \ + symbol = reinterpret_cast(elf_data->d_buf); \ + for (size_t i = 0; i < symbol_count; ++i) { \ + int type = ELF##ARCH##_ST_TYPE(symbol->st_info); \ + if (type == STT_FUNC && symbol->st_value > 0) { \ + r.symbol_cache[symbol->st_value] = std::string( \ + elf_strptr(elf_handle.get(), symbol_strings, symbol->st_name)); \ + } \ + ++symbol; \ + } \ + } + + if (e_ident[EI_CLASS] == ELFCLASS32) { + ELF_GET_DATA(32) + } else if (e_ident[EI_CLASS] == ELFCLASS64) { + // libelf might have been built without 64 bit support +#if __LIBELF64 + ELF_GET_DATA(64) +#endif + } + + if (!debuglink.empty()) { + // We have a debuglink section! Open an elf instance on that + // file instead. If we can't open the file, then return + // the elf handle we had already opened. + dwarf_file_t debuglink_file; + debuglink_file.reset(open(debuglink.c_str(), O_RDONLY)); + if (debuglink_file.get() > 0) { + dwarf_elf_t debuglink_elf; + debuglink_elf.reset(elf_begin(debuglink_file.get(), ELF_C_READ, NULL)); + + // If we have a valid elf handle, return the new elf handle + // and file handle and discard the original ones + if (debuglink_elf) { + elf_handle = move(debuglink_elf); + file_handle = move(debuglink_file); + } + } + } + + // Ok, we have a valid ELF handle, let's try to get debug symbols + Dwarf_Debug dwarf_debug; + Dwarf_Error error = DW_DLE_NE; + dwarf_handle_t dwarf_handle; + + int dwarf_result = dwarf_elf_init(elf_handle.get(), DW_DLC_READ, NULL, NULL, + &dwarf_debug, &error); + + // We don't do any special handling for DW_DLV_NO_ENTRY specially. + // If we get an error, or the file doesn't have debug information + // we just return. + if (dwarf_result != DW_DLV_OK) { + return r; + } + + dwarf_handle.reset(dwarf_debug); + + r.file_handle = move(file_handle); + r.elf_handle = move(elf_handle); + r.dwarf_handle = move(dwarf_handle); + + return r; + } + + die_cache_entry &get_die_cache(dwarf_fileobject &fobj, Dwarf_Die die) { + Dwarf_Error error = DW_DLE_NE; + + // Get the die offset, we use it as the cache key + Dwarf_Off die_offset; + if (dwarf_dieoffset(die, &die_offset, &error) != DW_DLV_OK) { + die_offset = 0; + } + + die_cache_t::iterator it = fobj.die_cache.find(die_offset); + + if (it != fobj.die_cache.end()) { + fobj.current_cu = &it->second; + return it->second; + } + + die_cache_entry &de = fobj.die_cache[die_offset]; + fobj.current_cu = &de; + + Dwarf_Addr line_addr; + Dwarf_Small table_count; + + // The addresses in the line section are not fully sorted (they might + // be sorted by block of code belonging to the same file), which makes + // it necessary to do so before searching is possible. + // + // As libdwarf allocates a copy of everything, let's get the contents + // of the line section and keep it around. We also create a map of + // program counter to line table indices so we can search by address + // and get the line buffer index. + // + // To make things more difficult, the same address can span more than + // one line, so we need to keep the index pointing to the first line + // by using insert instead of the map's [ operator. + + // Get the line context for the DIE + if (dwarf_srclines_b(die, 0, &table_count, &de.line_context, &error) == + DW_DLV_OK) { + // Get the source lines for this line context, to be deallocated + // later + if (dwarf_srclines_from_linecontext(de.line_context, &de.line_buffer, + &de.line_count, + &error) == DW_DLV_OK) { + + // Add all the addresses to our map + for (int i = 0; i < de.line_count; i++) { + if (dwarf_lineaddr(de.line_buffer[i], &line_addr, &error) != + DW_DLV_OK) { + line_addr = 0; + } + de.line_section.insert(std::pair(line_addr, i)); + } + } + } + + // For each CU, cache the function DIEs that contain the + // DW_AT_specification attribute. When building with -g3 the function + // DIEs are separated in declaration and specification, with the + // declaration containing only the name and parameters and the + // specification the low/high pc and other compiler attributes. + // + // We cache those specifications so we don't skip over the declarations, + // because they have no pc, and we can do namespace resolution for + // DWARF function names. + Dwarf_Debug dwarf = fobj.dwarf_handle.get(); + Dwarf_Die current_die = 0; + if (dwarf_child(die, ¤t_die, &error) == DW_DLV_OK) { + for (;;) { + Dwarf_Die sibling_die = 0; + + Dwarf_Half tag_value; + dwarf_tag(current_die, &tag_value, &error); + + if (tag_value == DW_TAG_subprogram || + tag_value == DW_TAG_inlined_subroutine) { + + Dwarf_Bool has_attr = 0; + if (dwarf_hasattr(current_die, DW_AT_specification, &has_attr, + &error) == DW_DLV_OK) { + if (has_attr) { + Dwarf_Attribute attr_mem; + if (dwarf_attr(current_die, DW_AT_specification, &attr_mem, + &error) == DW_DLV_OK) { + Dwarf_Off spec_offset = 0; + if (dwarf_formref(attr_mem, &spec_offset, &error) == + DW_DLV_OK) { + Dwarf_Off spec_die_offset; + if (dwarf_dieoffset(current_die, &spec_die_offset, &error) == + DW_DLV_OK) { + de.spec_section[spec_offset] = spec_die_offset; + } + } + } + dwarf_dealloc(dwarf, attr_mem, DW_DLA_ATTR); + } + } + } + + int result = dwarf_siblingof(dwarf, current_die, &sibling_die, &error); + if (result == DW_DLV_ERROR) { + break; + } else if (result == DW_DLV_NO_ENTRY) { + break; + } + + if (current_die != die) { + dwarf_dealloc(dwarf, current_die, DW_DLA_DIE); + current_die = 0; + } + + current_die = sibling_die; + } + } + return de; + } + + static Dwarf_Die get_referenced_die(Dwarf_Debug dwarf, Dwarf_Die die, + Dwarf_Half attr, bool global) { + Dwarf_Error error = DW_DLE_NE; + Dwarf_Attribute attr_mem; + + Dwarf_Die found_die = NULL; + if (dwarf_attr(die, attr, &attr_mem, &error) == DW_DLV_OK) { + Dwarf_Off offset; + int result = 0; + if (global) { + result = dwarf_global_formref(attr_mem, &offset, &error); + } else { + result = dwarf_formref(attr_mem, &offset, &error); + } + + if (result == DW_DLV_OK) { + if (dwarf_offdie(dwarf, offset, &found_die, &error) != DW_DLV_OK) { + found_die = NULL; + } + } + dwarf_dealloc(dwarf, attr_mem, DW_DLA_ATTR); + } + return found_die; + } + + static std::string get_referenced_die_name(Dwarf_Debug dwarf, Dwarf_Die die, + Dwarf_Half attr, bool global) { + Dwarf_Error error = DW_DLE_NE; + std::string value; + + Dwarf_Die found_die = get_referenced_die(dwarf, die, attr, global); + + if (found_die) { + char *name; + if (dwarf_diename(found_die, &name, &error) == DW_DLV_OK) { + if (name) { + value = std::string(name); + } + dwarf_dealloc(dwarf, name, DW_DLA_STRING); + } + dwarf_dealloc(dwarf, found_die, DW_DLA_DIE); + } + + return value; + } + + // Returns a spec DIE linked to the passed one. The caller should + // deallocate the DIE + static Dwarf_Die get_spec_die(dwarf_fileobject &fobj, Dwarf_Die die) { + Dwarf_Debug dwarf = fobj.dwarf_handle.get(); + Dwarf_Error error = DW_DLE_NE; + Dwarf_Off die_offset; + if (fobj.current_cu && + dwarf_die_CU_offset(die, &die_offset, &error) == DW_DLV_OK) { + die_specmap_t::iterator it = + fobj.current_cu->spec_section.find(die_offset); + + // If we have a DIE that completes the current one, check if + // that one has the pc we are looking for + if (it != fobj.current_cu->spec_section.end()) { + Dwarf_Die spec_die = 0; + if (dwarf_offdie(dwarf, it->second, &spec_die, &error) == DW_DLV_OK) { + return spec_die; + } + } + } + + // Maybe we have an abstract origin DIE with the function information? + return get_referenced_die(fobj.dwarf_handle.get(), die, + DW_AT_abstract_origin, true); + } + + static bool die_has_pc(dwarf_fileobject &fobj, Dwarf_Die die, Dwarf_Addr pc) { + Dwarf_Addr low_pc = 0, high_pc = 0; + Dwarf_Half high_pc_form = 0; + Dwarf_Form_Class return_class; + Dwarf_Error error = DW_DLE_NE; + Dwarf_Debug dwarf = fobj.dwarf_handle.get(); + bool has_lowpc = false; + bool has_highpc = false; + bool has_ranges = false; + + if (dwarf_lowpc(die, &low_pc, &error) == DW_DLV_OK) { + // If we have a low_pc check if there is a high pc. + // If we don't have a high pc this might mean we have a base + // address for the ranges list or just an address. + has_lowpc = true; + + if (dwarf_highpc_b(die, &high_pc, &high_pc_form, &return_class, &error) == + DW_DLV_OK) { + // We do have a high pc. In DWARF 4+ this is an offset from the + // low pc, but in earlier versions it's an absolute address. + + has_highpc = true; + // In DWARF 2/3 this would be a DW_FORM_CLASS_ADDRESS + if (return_class == DW_FORM_CLASS_CONSTANT) { + high_pc = low_pc + high_pc; + } + + // We have low and high pc, check if our address + // is in that range + return pc >= low_pc && pc < high_pc; + } + } else { + // Reset the low_pc, in case dwarf_lowpc failing set it to some + // undefined value. + low_pc = 0; + } + + // Check if DW_AT_ranges is present and search for the PC in the + // returned ranges list. We always add the low_pc, as it not set it will + // be 0, in case we had a DW_AT_low_pc and DW_AT_ranges pair + bool result = false; + + Dwarf_Attribute attr; + if (dwarf_attr(die, DW_AT_ranges, &attr, &error) == DW_DLV_OK) { + + Dwarf_Off offset; + if (dwarf_global_formref(attr, &offset, &error) == DW_DLV_OK) { + Dwarf_Ranges *ranges; + Dwarf_Signed ranges_count = 0; + Dwarf_Unsigned byte_count = 0; + + if (dwarf_get_ranges_a(dwarf, offset, die, &ranges, &ranges_count, + &byte_count, &error) == DW_DLV_OK) { + has_ranges = ranges_count != 0; + for (int i = 0; i < ranges_count; i++) { + if (ranges[i].dwr_addr1 != 0 && + pc >= ranges[i].dwr_addr1 + low_pc && + pc < ranges[i].dwr_addr2 + low_pc) { + result = true; + break; + } + } + dwarf_ranges_dealloc(dwarf, ranges, ranges_count); + } + } + } + + // Last attempt. We might have a single address set as low_pc. + if (!result && low_pc != 0 && pc == low_pc) { + result = true; + } + + // If we don't have lowpc, highpc and ranges maybe this DIE is a + // declaration that relies on a DW_AT_specification DIE that happens + // later. Use the specification cache we filled when we loaded this CU. + if (!result && (!has_lowpc && !has_highpc && !has_ranges)) { + Dwarf_Die spec_die = get_spec_die(fobj, die); + if (spec_die) { + result = die_has_pc(fobj, spec_die, pc); + dwarf_dealloc(dwarf, spec_die, DW_DLA_DIE); + } + } + + return result; + } + + static void get_type(Dwarf_Debug dwarf, Dwarf_Die die, std::string &type) { + Dwarf_Error error = DW_DLE_NE; + + Dwarf_Die child = 0; + if (dwarf_child(die, &child, &error) == DW_DLV_OK) { + get_type(dwarf, child, type); + } + + if (child) { + type.insert(0, "::"); + dwarf_dealloc(dwarf, child, DW_DLA_DIE); + } + + char *name; + if (dwarf_diename(die, &name, &error) == DW_DLV_OK) { + type.insert(0, std::string(name)); + dwarf_dealloc(dwarf, name, DW_DLA_STRING); + } else { + type.insert(0, ""); + } + } + + static std::string get_type_by_signature(Dwarf_Debug dwarf, Dwarf_Die die) { + Dwarf_Error error = DW_DLE_NE; + + Dwarf_Sig8 signature; + Dwarf_Bool has_attr = 0; + if (dwarf_hasattr(die, DW_AT_signature, &has_attr, &error) == DW_DLV_OK) { + if (has_attr) { + Dwarf_Attribute attr_mem; + if (dwarf_attr(die, DW_AT_signature, &attr_mem, &error) == DW_DLV_OK) { + if (dwarf_formsig8(attr_mem, &signature, &error) != DW_DLV_OK) { + return std::string(""); + } + } + dwarf_dealloc(dwarf, attr_mem, DW_DLA_ATTR); + } + } + + Dwarf_Unsigned next_cu_header; + Dwarf_Sig8 tu_signature; + std::string result; + bool found = false; + + while (dwarf_next_cu_header_d(dwarf, 0, 0, 0, 0, 0, 0, 0, &tu_signature, 0, + &next_cu_header, 0, &error) == DW_DLV_OK) { + + if (strncmp(signature.signature, tu_signature.signature, 8) == 0) { + Dwarf_Die type_cu_die = 0; + if (dwarf_siblingof_b(dwarf, 0, 0, &type_cu_die, &error) == DW_DLV_OK) { + Dwarf_Die child_die = 0; + if (dwarf_child(type_cu_die, &child_die, &error) == DW_DLV_OK) { + get_type(dwarf, child_die, result); + found = !result.empty(); + dwarf_dealloc(dwarf, child_die, DW_DLA_DIE); + } + dwarf_dealloc(dwarf, type_cu_die, DW_DLA_DIE); + } + } + } + + if (found) { + while (dwarf_next_cu_header_d(dwarf, 0, 0, 0, 0, 0, 0, 0, 0, 0, + &next_cu_header, 0, &error) == DW_DLV_OK) { + // Reset the cu header state. Unfortunately, libdwarf's + // next_cu_header API keeps its own iterator per Dwarf_Debug + // that can't be reset. We need to keep fetching elements until + // the end. + } + } else { + // If we couldn't resolve the type just print out the signature + std::ostringstream string_stream; + string_stream << "<0x" << std::hex << std::setfill('0'); + for (int i = 0; i < 8; ++i) { + string_stream << std::setw(2) << std::hex + << (int)(unsigned char)(signature.signature[i]); + } + string_stream << ">"; + result = string_stream.str(); + } + return result; + } + + struct type_context_t { + bool is_const; + bool is_typedef; + bool has_type; + bool has_name; + std::string text; + + type_context_t() + : is_const(false), is_typedef(false), has_type(false), has_name(false) { + } + }; + + // Types are resolved from right to left: we get the variable name first + // and then all specifiers (like const or pointer) in a chain of DW_AT_type + // DIEs. Call this function recursively until we get a complete type + // string. + static void set_parameter_string(dwarf_fileobject &fobj, Dwarf_Die die, + type_context_t &context) { + char *name; + Dwarf_Error error = DW_DLE_NE; + + // typedefs contain also the base type, so we skip it and only + // print the typedef name + if (!context.is_typedef) { + if (dwarf_diename(die, &name, &error) == DW_DLV_OK) { + if (!context.text.empty()) { + context.text.insert(0, " "); + } + context.text.insert(0, std::string(name)); + dwarf_dealloc(fobj.dwarf_handle.get(), name, DW_DLA_STRING); + } + } else { + context.is_typedef = false; + context.has_type = true; + if (context.is_const) { + context.text.insert(0, "const "); + context.is_const = false; + } + } + + bool next_type_is_const = false; + bool is_keyword = true; + + Dwarf_Half tag = 0; + Dwarf_Bool has_attr = 0; + if (dwarf_tag(die, &tag, &error) == DW_DLV_OK) { + switch (tag) { + case DW_TAG_structure_type: + case DW_TAG_union_type: + case DW_TAG_class_type: + case DW_TAG_enumeration_type: + context.has_type = true; + if (dwarf_hasattr(die, DW_AT_signature, &has_attr, &error) == + DW_DLV_OK) { + // If we have a signature it means the type is defined + // in .debug_types, so we need to load the DIE pointed + // at by the signature and resolve it + if (has_attr) { + std::string type = + get_type_by_signature(fobj.dwarf_handle.get(), die); + if (context.is_const) + type.insert(0, "const "); + + if (!context.text.empty()) + context.text.insert(0, " "); + context.text.insert(0, type); + } + + // Treat enums like typedefs, and skip printing its + // base type + context.is_typedef = (tag == DW_TAG_enumeration_type); + } + break; + case DW_TAG_const_type: + next_type_is_const = true; + break; + case DW_TAG_pointer_type: + context.text.insert(0, "*"); + break; + case DW_TAG_reference_type: + context.text.insert(0, "&"); + break; + case DW_TAG_restrict_type: + context.text.insert(0, "restrict "); + break; + case DW_TAG_rvalue_reference_type: + context.text.insert(0, "&&"); + break; + case DW_TAG_volatile_type: + context.text.insert(0, "volatile "); + break; + case DW_TAG_typedef: + // Propagate the const-ness to the next type + // as typedefs are linked to its base type + next_type_is_const = context.is_const; + context.is_typedef = true; + context.has_type = true; + break; + case DW_TAG_base_type: + context.has_type = true; + break; + case DW_TAG_formal_parameter: + context.has_name = true; + break; + default: + is_keyword = false; + break; + } + } + + if (!is_keyword && context.is_const) { + context.text.insert(0, "const "); + } + + context.is_const = next_type_is_const; + + Dwarf_Die ref = + get_referenced_die(fobj.dwarf_handle.get(), die, DW_AT_type, true); + if (ref) { + set_parameter_string(fobj, ref, context); + dwarf_dealloc(fobj.dwarf_handle.get(), ref, DW_DLA_DIE); + } + + if (!context.has_type && context.has_name) { + context.text.insert(0, "void "); + context.has_type = true; + } + } + + // Resolve the function return type and parameters + static void set_function_parameters(std::string &function_name, + std::vector &ns, + dwarf_fileobject &fobj, Dwarf_Die die) { + Dwarf_Debug dwarf = fobj.dwarf_handle.get(); + Dwarf_Error error = DW_DLE_NE; + Dwarf_Die current_die = 0; + std::string parameters; + bool has_spec = true; + // Check if we have a spec DIE. If we do we use it as it contains + // more information, like parameter names. + Dwarf_Die spec_die = get_spec_die(fobj, die); + if (!spec_die) { + has_spec = false; + spec_die = die; + } + + std::vector::const_iterator it = ns.begin(); + std::string ns_name; + for (it = ns.begin(); it < ns.end(); ++it) { + ns_name.append(*it).append("::"); + } + + if (!ns_name.empty()) { + function_name.insert(0, ns_name); + } + + // See if we have a function return type. It can be either on the + // current die or in its spec one (usually true for inlined functions) + std::string return_type = + get_referenced_die_name(dwarf, die, DW_AT_type, true); + if (return_type.empty()) { + return_type = get_referenced_die_name(dwarf, spec_die, DW_AT_type, true); + } + if (!return_type.empty()) { + return_type.append(" "); + function_name.insert(0, return_type); + } + + if (dwarf_child(spec_die, ¤t_die, &error) == DW_DLV_OK) { + for (;;) { + Dwarf_Die sibling_die = 0; + + Dwarf_Half tag_value; + dwarf_tag(current_die, &tag_value, &error); + + if (tag_value == DW_TAG_formal_parameter) { + // Ignore artificial (ie, compiler generated) parameters + bool is_artificial = false; + Dwarf_Attribute attr_mem; + if (dwarf_attr(current_die, DW_AT_artificial, &attr_mem, &error) == + DW_DLV_OK) { + Dwarf_Bool flag = 0; + if (dwarf_formflag(attr_mem, &flag, &error) == DW_DLV_OK) { + is_artificial = flag != 0; + } + dwarf_dealloc(dwarf, attr_mem, DW_DLA_ATTR); + } + + if (!is_artificial) { + type_context_t context; + set_parameter_string(fobj, current_die, context); + + if (parameters.empty()) { + parameters.append("("); + } else { + parameters.append(", "); + } + parameters.append(context.text); + } + } + + int result = dwarf_siblingof(dwarf, current_die, &sibling_die, &error); + if (result == DW_DLV_ERROR) { + break; + } else if (result == DW_DLV_NO_ENTRY) { + break; + } + + if (current_die != die) { + dwarf_dealloc(dwarf, current_die, DW_DLA_DIE); + current_die = 0; + } + + current_die = sibling_die; + } + } + if (parameters.empty()) + parameters = "("; + parameters.append(")"); + + // If we got a spec DIE we need to deallocate it + if (has_spec) + dwarf_dealloc(dwarf, spec_die, DW_DLA_DIE); + + function_name.append(parameters); + } + + // defined here because in C++98, template function cannot take locally + // defined types... grrr. + struct inliners_search_cb { + void operator()(Dwarf_Die die, std::vector &ns) { + Dwarf_Error error = DW_DLE_NE; + Dwarf_Half tag_value; + Dwarf_Attribute attr_mem; + Dwarf_Debug dwarf = fobj.dwarf_handle.get(); + + dwarf_tag(die, &tag_value, &error); + + switch (tag_value) { + char *name; + case DW_TAG_subprogram: + if (!trace.source.function.empty()) + break; + if (dwarf_diename(die, &name, &error) == DW_DLV_OK) { + trace.source.function = std::string(name); + dwarf_dealloc(dwarf, name, DW_DLA_STRING); + } else { + // We don't have a function name in this DIE. + // Check if there is a referenced non-defining + // declaration. + trace.source.function = + get_referenced_die_name(dwarf, die, DW_AT_abstract_origin, true); + if (trace.source.function.empty()) { + trace.source.function = + get_referenced_die_name(dwarf, die, DW_AT_specification, true); + } + } + + // Append the function parameters, if available + set_function_parameters(trace.source.function, ns, fobj, die); + + // If the object function name is empty, it's possible that + // there is no dynamic symbol table (maybe the executable + // was stripped or not built with -rdynamic). See if we have + // a DWARF linkage name to use instead. We try both + // linkage_name and MIPS_linkage_name because the MIPS tag + // was the unofficial one until it was adopted in DWARF4. + // Old gcc versions generate MIPS_linkage_name + if (trace.object_function.empty()) { + details::demangler demangler; + + if (dwarf_attr(die, DW_AT_linkage_name, &attr_mem, &error) != + DW_DLV_OK) { + if (dwarf_attr(die, DW_AT_MIPS_linkage_name, &attr_mem, &error) != + DW_DLV_OK) { + break; + } + } + + char *linkage; + if (dwarf_formstring(attr_mem, &linkage, &error) == DW_DLV_OK) { + trace.object_function = demangler.demangle(linkage); + dwarf_dealloc(dwarf, linkage, DW_DLA_STRING); + } + dwarf_dealloc(dwarf, attr_mem, DW_DLA_ATTR); + } + break; + + case DW_TAG_inlined_subroutine: + ResolvedTrace::SourceLoc sloc; + + if (dwarf_diename(die, &name, &error) == DW_DLV_OK) { + sloc.function = std::string(name); + dwarf_dealloc(dwarf, name, DW_DLA_STRING); + } else { + // We don't have a name for this inlined DIE, it could + // be that there is an abstract origin instead. + // Get the DW_AT_abstract_origin value, which is a + // reference to the source DIE and try to get its name + sloc.function = + get_referenced_die_name(dwarf, die, DW_AT_abstract_origin, true); + } + + set_function_parameters(sloc.function, ns, fobj, die); + + std::string file = die_call_file(dwarf, die, cu_die); + if (!file.empty()) + sloc.filename = file; + + Dwarf_Unsigned number = 0; + if (dwarf_attr(die, DW_AT_call_line, &attr_mem, &error) == DW_DLV_OK) { + if (dwarf_formudata(attr_mem, &number, &error) == DW_DLV_OK) { + sloc.line = number; + } + dwarf_dealloc(dwarf, attr_mem, DW_DLA_ATTR); + } + + if (dwarf_attr(die, DW_AT_call_column, &attr_mem, &error) == + DW_DLV_OK) { + if (dwarf_formudata(attr_mem, &number, &error) == DW_DLV_OK) { + sloc.col = number; + } + dwarf_dealloc(dwarf, attr_mem, DW_DLA_ATTR); + } + + trace.inliners.push_back(sloc); + break; + }; + } + ResolvedTrace &trace; + dwarf_fileobject &fobj; + Dwarf_Die cu_die; + inliners_search_cb(ResolvedTrace &t, dwarf_fileobject &f, Dwarf_Die c) + : trace(t), fobj(f), cu_die(c) {} + }; + + static Dwarf_Die find_fundie_by_pc(dwarf_fileobject &fobj, + Dwarf_Die parent_die, Dwarf_Addr pc, + Dwarf_Die result) { + Dwarf_Die current_die = 0; + Dwarf_Error error = DW_DLE_NE; + Dwarf_Debug dwarf = fobj.dwarf_handle.get(); + + if (dwarf_child(parent_die, ¤t_die, &error) != DW_DLV_OK) { + return NULL; + } + + for (;;) { + Dwarf_Die sibling_die = 0; + Dwarf_Half tag_value; + dwarf_tag(current_die, &tag_value, &error); + + switch (tag_value) { + case DW_TAG_subprogram: + case DW_TAG_inlined_subroutine: + if (die_has_pc(fobj, current_die, pc)) { + return current_die; + } + }; + bool declaration = false; + Dwarf_Attribute attr_mem; + if (dwarf_attr(current_die, DW_AT_declaration, &attr_mem, &error) == + DW_DLV_OK) { + Dwarf_Bool flag = 0; + if (dwarf_formflag(attr_mem, &flag, &error) == DW_DLV_OK) { + declaration = flag != 0; + } + dwarf_dealloc(dwarf, attr_mem, DW_DLA_ATTR); + } + + if (!declaration) { + // let's be curious and look deeper in the tree, functions are + // not necessarily at the first level, but might be nested + // inside a namespace, structure, a function, an inlined + // function etc. + Dwarf_Die die_mem = 0; + Dwarf_Die indie = find_fundie_by_pc(fobj, current_die, pc, die_mem); + if (indie) { + result = die_mem; + return result; + } + } + + int res = dwarf_siblingof(dwarf, current_die, &sibling_die, &error); + if (res == DW_DLV_ERROR) { + return NULL; + } else if (res == DW_DLV_NO_ENTRY) { + break; + } + + if (current_die != parent_die) { + dwarf_dealloc(dwarf, current_die, DW_DLA_DIE); + current_die = 0; + } + + current_die = sibling_die; + } + return NULL; + } + + template + static bool deep_first_search_by_pc(dwarf_fileobject &fobj, + Dwarf_Die parent_die, Dwarf_Addr pc, + std::vector &ns, CB cb) { + Dwarf_Die current_die = 0; + Dwarf_Debug dwarf = fobj.dwarf_handle.get(); + Dwarf_Error error = DW_DLE_NE; + + if (dwarf_child(parent_die, ¤t_die, &error) != DW_DLV_OK) { + return false; + } + + bool branch_has_pc = false; + bool has_namespace = false; + for (;;) { + Dwarf_Die sibling_die = 0; + + Dwarf_Half tag; + if (dwarf_tag(current_die, &tag, &error) == DW_DLV_OK) { + if (tag == DW_TAG_namespace || tag == DW_TAG_class_type) { + char *ns_name = NULL; + if (dwarf_diename(current_die, &ns_name, &error) == DW_DLV_OK) { + if (ns_name) { + ns.push_back(std::string(ns_name)); + } else { + ns.push_back(""); + } + dwarf_dealloc(dwarf, ns_name, DW_DLA_STRING); + } else { + ns.push_back(""); + } + has_namespace = true; + } + } + + bool declaration = false; + Dwarf_Attribute attr_mem; + if (tag != DW_TAG_class_type && + dwarf_attr(current_die, DW_AT_declaration, &attr_mem, &error) == + DW_DLV_OK) { + Dwarf_Bool flag = 0; + if (dwarf_formflag(attr_mem, &flag, &error) == DW_DLV_OK) { + declaration = flag != 0; + } + dwarf_dealloc(dwarf, attr_mem, DW_DLA_ATTR); + } + + if (!declaration) { + // let's be curious and look deeper in the tree, function are + // not necessarily at the first level, but might be nested + // inside a namespace, structure, a function, an inlined + // function etc. + branch_has_pc = deep_first_search_by_pc(fobj, current_die, pc, ns, cb); + } + + if (!branch_has_pc) { + branch_has_pc = die_has_pc(fobj, current_die, pc); + } + + if (branch_has_pc) { + cb(current_die, ns); + } + + int result = dwarf_siblingof(dwarf, current_die, &sibling_die, &error); + if (result == DW_DLV_ERROR) { + return false; + } else if (result == DW_DLV_NO_ENTRY) { + break; + } + + if (current_die != parent_die) { + dwarf_dealloc(dwarf, current_die, DW_DLA_DIE); + current_die = 0; + } + + if (has_namespace) { + has_namespace = false; + ns.pop_back(); + } + current_die = sibling_die; + } + + if (has_namespace) { + ns.pop_back(); + } + return branch_has_pc; + } + + static std::string die_call_file(Dwarf_Debug dwarf, Dwarf_Die die, + Dwarf_Die cu_die) { + Dwarf_Attribute attr_mem; + Dwarf_Error error = DW_DLE_NE; + Dwarf_Unsigned file_index; + + std::string file; + + if (dwarf_attr(die, DW_AT_call_file, &attr_mem, &error) == DW_DLV_OK) { + if (dwarf_formudata(attr_mem, &file_index, &error) != DW_DLV_OK) { + file_index = 0; + } + dwarf_dealloc(dwarf, attr_mem, DW_DLA_ATTR); + + if (file_index == 0) { + return file; + } + + char **srcfiles = 0; + Dwarf_Signed file_count = 0; + if (dwarf_srcfiles(cu_die, &srcfiles, &file_count, &error) == DW_DLV_OK) { + if (file_count > 0 && file_index <= static_cast(file_count)) { + file = std::string(srcfiles[file_index - 1]); + } + + // Deallocate all strings! + for (int i = 0; i < file_count; ++i) { + dwarf_dealloc(dwarf, srcfiles[i], DW_DLA_STRING); + } + dwarf_dealloc(dwarf, srcfiles, DW_DLA_LIST); + } + } + return file; + } + + Dwarf_Die find_die(dwarf_fileobject &fobj, Dwarf_Addr addr) { + // Let's get to work! First see if we have a debug_aranges section so + // we can speed up the search + + Dwarf_Debug dwarf = fobj.dwarf_handle.get(); + Dwarf_Error error = DW_DLE_NE; + Dwarf_Arange *aranges; + Dwarf_Signed arange_count; + + Dwarf_Die returnDie; + bool found = false; + if (dwarf_get_aranges(dwarf, &aranges, &arange_count, &error) != + DW_DLV_OK) { + aranges = NULL; + } + + if (aranges) { + // We have aranges. Get the one where our address is. + Dwarf_Arange arange; + if (dwarf_get_arange(aranges, arange_count, addr, &arange, &error) == + DW_DLV_OK) { + + // We found our address. Get the compilation-unit DIE offset + // represented by the given address range. + Dwarf_Off cu_die_offset; + if (dwarf_get_cu_die_offset(arange, &cu_die_offset, &error) == + DW_DLV_OK) { + // Get the DIE at the offset returned by the aranges search. + // We set is_info to 1 to specify that the offset is from + // the .debug_info section (and not .debug_types) + int dwarf_result = + dwarf_offdie_b(dwarf, cu_die_offset, 1, &returnDie, &error); + + found = dwarf_result == DW_DLV_OK; + } + dwarf_dealloc(dwarf, arange, DW_DLA_ARANGE); + } + } + + if (found) + return returnDie; // The caller is responsible for freeing the die + + // The search for aranges failed. Try to find our address by scanning + // all compilation units. + Dwarf_Unsigned next_cu_header; + Dwarf_Half tag = 0; + returnDie = 0; + + while (!found && + dwarf_next_cu_header_d(dwarf, 1, 0, 0, 0, 0, 0, 0, 0, 0, + &next_cu_header, 0, &error) == DW_DLV_OK) { + + if (returnDie) + dwarf_dealloc(dwarf, returnDie, DW_DLA_DIE); + + if (dwarf_siblingof(dwarf, 0, &returnDie, &error) == DW_DLV_OK) { + if ((dwarf_tag(returnDie, &tag, &error) == DW_DLV_OK) && + tag == DW_TAG_compile_unit) { + if (die_has_pc(fobj, returnDie, addr)) { + found = true; + } + } + } + } + + if (found) { + while (dwarf_next_cu_header_d(dwarf, 1, 0, 0, 0, 0, 0, 0, 0, 0, + &next_cu_header, 0, &error) == DW_DLV_OK) { + // Reset the cu header state. Libdwarf's next_cu_header API + // keeps its own iterator per Dwarf_Debug that can't be reset. + // We need to keep fetching elements until the end. + } + } + + if (found) + return returnDie; + + // We couldn't find any compilation units with ranges or a high/low pc. + // Try again by looking at all DIEs in all compilation units. + Dwarf_Die cudie; + while (dwarf_next_cu_header_d(dwarf, 1, 0, 0, 0, 0, 0, 0, 0, 0, + &next_cu_header, 0, &error) == DW_DLV_OK) { + if (dwarf_siblingof(dwarf, 0, &cudie, &error) == DW_DLV_OK) { + Dwarf_Die die_mem = 0; + Dwarf_Die resultDie = find_fundie_by_pc(fobj, cudie, addr, die_mem); + + if (resultDie) { + found = true; + break; + } + } + } + + if (found) { + while (dwarf_next_cu_header_d(dwarf, 1, 0, 0, 0, 0, 0, 0, 0, 0, + &next_cu_header, 0, &error) == DW_DLV_OK) { + // Reset the cu header state. Libdwarf's next_cu_header API + // keeps its own iterator per Dwarf_Debug that can't be reset. + // We need to keep fetching elements until the end. + } + } + + if (found) + return cudie; + + // We failed. + return NULL; + } +}; +#endif // BACKWARD_HAS_DWARF == 1 + +template <> +class TraceResolverImpl + : public TraceResolverLinuxImpl {}; + +#endif // BACKWARD_SYSTEM_LINUX + +#ifdef BACKWARD_SYSTEM_DARWIN + +template class TraceResolverDarwinImpl; + +template <> +class TraceResolverDarwinImpl + : public TraceResolverImplBase { +public: + void load_addresses(void *const*addresses, int address_count) override { + if (address_count == 0) { + return; + } + _symbols.reset(backtrace_symbols(addresses, address_count)); + } + + ResolvedTrace resolve(ResolvedTrace trace) override { + // parse: + // + + char *filename = _symbols[trace.idx]; + + // skip " " + while (*filename && *filename != ' ') + filename++; + while (*filename == ' ') + filename++; + + // find start of from end ( may contain a space) + char *p = filename + strlen(filename) - 1; + // skip to start of " + " + while (p > filename && *p != ' ') + p--; + while (p > filename && *p == ' ') + p--; + while (p > filename && *p != ' ') + p--; + while (p > filename && *p == ' ') + p--; + char *funcname_end = p + 1; + + // skip to start of "" + while (p > filename && *p != ' ') + p--; + char *funcname = p + 1; + + // skip to start of " " + while (p > filename && *p == ' ') + p--; + while (p > filename && *p != ' ') + p--; + while (p > filename && *p == ' ') + p--; + + // skip "", handling the case where it contains a + char *filename_end = p + 1; + if (p == filename) { + // something went wrong, give up + filename_end = filename + strlen(filename); + funcname = filename_end; + } + trace.object_filename.assign( + filename, filename_end); // ok even if filename_end is the ending \0 + // (then we assign entire string) + + if (*funcname) { // if it's not end of string + *funcname_end = '\0'; + + trace.object_function = this->demangle(funcname); + trace.object_function += " "; + trace.object_function += (funcname_end + 1); + trace.source.function = trace.object_function; // we cannot do better. + } + return trace; + } + +private: + details::handle _symbols; +}; + +template <> +class TraceResolverImpl + : public TraceResolverDarwinImpl {}; + +#endif // BACKWARD_SYSTEM_DARWIN + +#ifdef BACKWARD_SYSTEM_WINDOWS + +// Load all symbol info +// Based on: +// https://stackoverflow.com/questions/6205981/windows-c-stack-trace-from-a-running-app/28276227#28276227 + +struct module_data { + std::string image_name; + std::string module_name; + void *base_address; + DWORD load_size; +}; + +class get_mod_info { + HANDLE process; + static const int buffer_length = 4096; + +public: + get_mod_info(HANDLE h) : process(h) {} + + module_data operator()(HMODULE module) { + module_data ret; + char temp[buffer_length]; + MODULEINFO mi; + + GetModuleInformation(process, module, &mi, sizeof(mi)); + ret.base_address = mi.lpBaseOfDll; + ret.load_size = mi.SizeOfImage; + + GetModuleFileNameExA(process, module, temp, sizeof(temp)); + ret.image_name = temp; + GetModuleBaseNameA(process, module, temp, sizeof(temp)); + ret.module_name = temp; + std::vector img(ret.image_name.begin(), ret.image_name.end()); + std::vector mod(ret.module_name.begin(), ret.module_name.end()); + SymLoadModule64(process, 0, &img[0], &mod[0], (DWORD64)ret.base_address, + ret.load_size); + return ret; + } +}; + +template <> class TraceResolverImpl + : public TraceResolverImplBase { +public: + TraceResolverImpl() { + + HANDLE process = GetCurrentProcess(); + + std::vector modules; + DWORD cbNeeded; + std::vector module_handles(1); + SymInitialize(process, NULL, false); + DWORD symOptions = SymGetOptions(); + symOptions |= SYMOPT_LOAD_LINES | SYMOPT_UNDNAME; + SymSetOptions(symOptions); + EnumProcessModules(process, &module_handles[0], + static_cast(module_handles.size() * sizeof(HMODULE)), + &cbNeeded); + module_handles.resize(cbNeeded / sizeof(HMODULE)); + EnumProcessModules(process, &module_handles[0], + static_cast(module_handles.size() * sizeof(HMODULE)), + &cbNeeded); + std::transform(module_handles.begin(), module_handles.end(), + std::back_inserter(modules), get_mod_info(process)); + void *base = modules[0].base_address; + IMAGE_NT_HEADERS *h = ImageNtHeader(base); + image_type = h->FileHeader.Machine; + } + + static const int max_sym_len = 255; + struct symbol_t { + SYMBOL_INFO sym; + char buffer[max_sym_len]; + } sym; + + DWORD64 displacement; + + ResolvedTrace resolve(ResolvedTrace t) override { + HANDLE process = GetCurrentProcess(); + + char name[256]; + + memset(&sym, 0, sizeof(sym)); + sym.sym.SizeOfStruct = sizeof(SYMBOL_INFO); + sym.sym.MaxNameLen = max_sym_len; + + if (!SymFromAddr(process, (ULONG64)t.addr, &displacement, &sym.sym)) { + // TODO: error handling everywhere + char* lpMsgBuf; + DWORD dw = GetLastError(); + + if (FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | + FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, dw, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + (char*)&lpMsgBuf, 0, NULL)) { + std::fprintf(stderr, "%s\n", lpMsgBuf); + LocalFree(lpMsgBuf); + } + + // abort(); + } + UnDecorateSymbolName(sym.sym.Name, (PSTR)name, 256, UNDNAME_COMPLETE); + + DWORD offset = 0; + IMAGEHLP_LINE line; + if (SymGetLineFromAddr(process, (ULONG64)t.addr, &offset, &line)) { + t.object_filename = line.FileName; + t.source.filename = line.FileName; + t.source.line = line.LineNumber; + t.source.col = offset; + } + + t.source.function = name; + t.object_filename = ""; + t.object_function = name; + + return t; + } + + DWORD machine_type() const { return image_type; } + +private: + DWORD image_type; +}; + +#endif + +class TraceResolver : public TraceResolverImpl {}; + +/*************** CODE SNIPPET ***************/ + +class SourceFile { +public: + typedef std::vector > lines_t; + + SourceFile() {} + SourceFile(const std::string &path) { + // 1. If BACKWARD_CXX_SOURCE_PREFIXES is set then assume it contains + // a colon-separated list of path prefixes. Try prepending each + // to the given path until a valid file is found. + const std::vector &prefixes = get_paths_from_env_variable(); + for (size_t i = 0; i < prefixes.size(); ++i) { + // Double slashes (//) should not be a problem. + std::string new_path = prefixes[i] + '/' + path; + _file.reset(new std::ifstream(new_path.c_str())); + if (is_open()) + break; + } + // 2. If no valid file found then fallback to opening the path as-is. + if (!_file || !is_open()) { + _file.reset(new std::ifstream(path.c_str())); + } + } + bool is_open() const { return _file->is_open(); } + + lines_t &get_lines(unsigned line_start, unsigned line_count, lines_t &lines) { + using namespace std; + // This function make uses of the dumbest algo ever: + // 1) seek(0) + // 2) read lines one by one and discard until line_start + // 3) read line one by one until line_start + line_count + // + // If you are getting snippets many time from the same file, it is + // somewhat a waste of CPU, feel free to benchmark and propose a + // better solution ;) + + _file->clear(); + _file->seekg(0); + string line; + unsigned line_idx; + + for (line_idx = 1; line_idx < line_start; ++line_idx) { + std::getline(*_file, line); + if (!*_file) { + return lines; + } + } + + // think of it like a lambda in C++98 ;) + // but look, I will reuse it two times! + // What a good boy am I. + struct isspace { + bool operator()(char c) { return std::isspace(c); } + }; + + bool started = false; + for (; line_idx < line_start + line_count; ++line_idx) { + getline(*_file, line); + if (!*_file) { + return lines; + } + if (!started) { + if (std::find_if(line.begin(), line.end(), not_isspace()) == line.end()) + continue; + started = true; + } + lines.push_back(make_pair(line_idx, line)); + } + + lines.erase( + std::find_if(lines.rbegin(), lines.rend(), not_isempty()).base(), + lines.end()); + return lines; + } + + lines_t get_lines(unsigned line_start, unsigned line_count) { + lines_t lines; + return get_lines(line_start, line_count, lines); + } + + // there is no find_if_not in C++98, lets do something crappy to + // workaround. + struct not_isspace { + bool operator()(char c) { return !std::isspace(c); } + }; + // and define this one here because C++98 is not happy with local defined + // struct passed to template functions, fuuuu. + struct not_isempty { + bool operator()(const lines_t::value_type &p) { + return !(std::find_if(p.second.begin(), p.second.end(), not_isspace()) == + p.second.end()); + } + }; + + void swap(SourceFile &b) { _file.swap(b._file); } + +#ifdef BACKWARD_ATLEAST_CXX11 + SourceFile(SourceFile &&from) : _file(nullptr) { swap(from); } + SourceFile &operator=(SourceFile &&from) { + swap(from); + return *this; + } +#else + explicit SourceFile(const SourceFile &from) { + // some sort of poor man's move semantic. + swap(const_cast(from)); + } + SourceFile &operator=(const SourceFile &from) { + // some sort of poor man's move semantic. + swap(const_cast(from)); + return *this; + } +#endif + + // Allow adding to paths gotten from BACKWARD_CXX_SOURCE_PREFIXES after loading the + // library; this can be useful when the library is loaded when the locations are unknown + // Warning: Because this edits the static paths variable, it is *not* intrinsiclly thread safe + static void add_paths_to_env_variable_impl(const std::string & to_add) { + get_mutable_paths_from_env_variable().push_back(to_add); + } + +private: + details::handle > + _file; + + static std::vector get_paths_from_env_variable_impl() { + std::vector paths; + const char *prefixes_str = std::getenv("BACKWARD_CXX_SOURCE_PREFIXES"); + if (prefixes_str && prefixes_str[0]) { + paths = details::split_source_prefixes(prefixes_str); + } + return paths; + } + + static std::vector &get_mutable_paths_from_env_variable() { + static volatile std::vector paths = get_paths_from_env_variable_impl(); + return const_cast&>(paths); + } + + static const std::vector &get_paths_from_env_variable() { + return get_mutable_paths_from_env_variable(); + } + +#ifdef BACKWARD_ATLEAST_CXX11 + SourceFile(const SourceFile &) = delete; + SourceFile &operator=(const SourceFile &) = delete; +#endif +}; + +class SnippetFactory { +public: + typedef SourceFile::lines_t lines_t; + + lines_t get_snippet(const std::string &filename, unsigned line_start, + unsigned context_size) { + + SourceFile &src_file = get_src_file(filename); + unsigned start = line_start - context_size / 2; + return src_file.get_lines(start, context_size); + } + + lines_t get_combined_snippet(const std::string &filename_a, unsigned line_a, + const std::string &filename_b, unsigned line_b, + unsigned context_size) { + SourceFile &src_file_a = get_src_file(filename_a); + SourceFile &src_file_b = get_src_file(filename_b); + + lines_t lines = + src_file_a.get_lines(line_a - context_size / 4, context_size / 2); + src_file_b.get_lines(line_b - context_size / 4, context_size / 2, lines); + return lines; + } + + lines_t get_coalesced_snippet(const std::string &filename, unsigned line_a, + unsigned line_b, unsigned context_size) { + SourceFile &src_file = get_src_file(filename); + + using std::max; + using std::min; + unsigned a = min(line_a, line_b); + unsigned b = max(line_a, line_b); + + if ((b - a) < (context_size / 3)) { + return src_file.get_lines((a + b - context_size + 1) / 2, context_size); + } + + lines_t lines = src_file.get_lines(a - context_size / 4, context_size / 2); + src_file.get_lines(b - context_size / 4, context_size / 2, lines); + return lines; + } + +private: + typedef details::hashtable::type src_files_t; + src_files_t _src_files; + + SourceFile &get_src_file(const std::string &filename) { + src_files_t::iterator it = _src_files.find(filename); + if (it != _src_files.end()) { + return it->second; + } + SourceFile &new_src_file = _src_files[filename]; + new_src_file = SourceFile(filename); + return new_src_file; + } +}; + +/*************** PRINTER ***************/ + +namespace ColorMode { +enum type { automatic, never, always }; +} + +class cfile_streambuf : public std::streambuf { +public: + cfile_streambuf(FILE *_sink) : sink(_sink) {} + int_type underflow() override { return traits_type::eof(); } + int_type overflow(int_type ch) override { + if (traits_type::not_eof(ch) && fputc(ch, sink) != EOF) { + return ch; + } + return traits_type::eof(); + } + + std::streamsize xsputn(const char_type *s, std::streamsize count) override { + return static_cast( + fwrite(s, sizeof *s, static_cast(count), sink)); + } + +#ifdef BACKWARD_ATLEAST_CXX11 +public: + cfile_streambuf(const cfile_streambuf &) = delete; + cfile_streambuf &operator=(const cfile_streambuf &) = delete; +#else +private: + cfile_streambuf(const cfile_streambuf &); + cfile_streambuf &operator=(const cfile_streambuf &); +#endif + +private: + FILE *sink; + std::vector buffer; +}; + +#ifdef BACKWARD_SYSTEM_LINUX + +namespace Color { +enum type { yellow = 33, purple = 35, reset = 39 }; +} // namespace Color + +class Colorize { +public: + Colorize(std::ostream &os) : _os(os), _reset(false), _enabled(false) {} + + void activate(ColorMode::type mode) { _enabled = mode == ColorMode::always; } + + void activate(ColorMode::type mode, FILE *fp) { activate(mode, fileno(fp)); } + + void set_color(Color::type ccode) { + if (!_enabled) + return; + + // I assume that the terminal can handle basic colors. Seriously I + // don't want to deal with all the termcap shit. + _os << "\033[" << static_cast(ccode) << "m"; + _reset = (ccode != Color::reset); + } + + ~Colorize() { + if (_reset) { + set_color(Color::reset); + } + } + +private: + void activate(ColorMode::type mode, int fd) { + activate(mode == ColorMode::automatic && isatty(fd) ? ColorMode::always + : mode); + } + + std::ostream &_os; + bool _reset; + bool _enabled; +}; + +#else // ndef BACKWARD_SYSTEM_LINUX + +namespace Color { +enum type { yellow = 0, purple = 0, reset = 0 }; +} // namespace Color + +class Colorize { +public: + Colorize(std::ostream &) {} + void activate(ColorMode::type) {} + void activate(ColorMode::type, FILE *) {} + void set_color(Color::type) {} +}; + +#endif // BACKWARD_SYSTEM_LINUX + +class Printer { +public: + bool snippet; + ColorMode::type color_mode; + bool address; + bool object; + int inliner_context_size; + int trace_context_size; + bool reverse; + + Printer() + : snippet(true), color_mode(ColorMode::automatic), address(false), + object(false), inliner_context_size(5), trace_context_size(7), + reverse(true) {} + + template FILE *print(ST &st, FILE *fp = stderr) { + cfile_streambuf obuf(fp); + std::ostream os(&obuf); + Colorize colorize(os); + colorize.activate(color_mode, fp); + print_stacktrace(st, os, colorize); + return fp; + } + + template std::ostream &print(ST &st, std::ostream &os) { + Colorize colorize(os); + colorize.activate(color_mode); + print_stacktrace(st, os, colorize); + return os; + } + + template + FILE *print(IT begin, IT end, FILE *fp = stderr, size_t thread_id = 0) { + cfile_streambuf obuf(fp); + std::ostream os(&obuf); + Colorize colorize(os); + colorize.activate(color_mode, fp); + print_stacktrace(begin, end, os, thread_id, colorize); + return fp; + } + + template + std::ostream &print(IT begin, IT end, std::ostream &os, + size_t thread_id = 0) { + Colorize colorize(os); + colorize.activate(color_mode); + print_stacktrace(begin, end, os, thread_id, colorize); + return os; + } + + TraceResolver const &resolver() const { return _resolver; } + +private: + TraceResolver _resolver; + SnippetFactory _snippets; + + template + void print_stacktrace(ST &st, std::ostream &os, Colorize &colorize) { + print_header(os, st.thread_id()); + _resolver.load_stacktrace(st); + if ( reverse ) { + for (size_t trace_idx = st.size(); trace_idx > 0; --trace_idx) { + print_trace(os, _resolver.resolve(st[trace_idx - 1]), colorize); + } + } else { + for (size_t trace_idx = 0; trace_idx < st.size(); ++trace_idx) { + print_trace(os, _resolver.resolve(st[trace_idx]), colorize); + } + } + } + + template + void print_stacktrace(IT begin, IT end, std::ostream &os, size_t thread_id, + Colorize &colorize) { + print_header(os, thread_id); + for (; begin != end; ++begin) { + print_trace(os, *begin, colorize); + } + } + + void print_header(std::ostream &os, size_t thread_id) { + os << "Stack trace (most recent call last)"; + if (thread_id) { + os << " in thread " << thread_id; + } + os << ":\n"; + } + + void print_trace(std::ostream &os, const ResolvedTrace &trace, + Colorize &colorize) { + os << "#" << std::left << std::setw(2) << trace.idx << std::right; + bool already_indented = true; + + if (!trace.source.filename.size() || object) { + os << " Object \"" << trace.object_filename << "\", at " << trace.addr + << ", in " << trace.object_function << "\n"; + already_indented = false; + } + + for (size_t inliner_idx = trace.inliners.size(); inliner_idx > 0; + --inliner_idx) { + if (!already_indented) { + os << " "; + } + const ResolvedTrace::SourceLoc &inliner_loc = + trace.inliners[inliner_idx - 1]; + print_source_loc(os, " | ", inliner_loc); + if (snippet) { + print_snippet(os, " | ", inliner_loc, colorize, Color::purple, + inliner_context_size); + } + already_indented = false; + } + + if (trace.source.filename.size()) { + if (!already_indented) { + os << " "; + } + print_source_loc(os, " ", trace.source, trace.addr); + if (snippet) { + print_snippet(os, " ", trace.source, colorize, Color::yellow, + trace_context_size); + } + } + } + + void print_snippet(std::ostream &os, const char *indent, + const ResolvedTrace::SourceLoc &source_loc, + Colorize &colorize, Color::type color_code, + int context_size) { + using namespace std; + typedef SnippetFactory::lines_t lines_t; + + lines_t lines = _snippets.get_snippet(source_loc.filename, source_loc.line, + static_cast(context_size)); + + for (lines_t::const_iterator it = lines.begin(); it != lines.end(); ++it) { + if (it->first == source_loc.line) { + colorize.set_color(color_code); + os << indent << ">"; + } else { + os << indent << " "; + } + os << std::setw(4) << it->first << ": " << it->second << "\n"; + if (it->first == source_loc.line) { + colorize.set_color(Color::reset); + } + } + } + + void print_source_loc(std::ostream &os, const char *indent, + const ResolvedTrace::SourceLoc &source_loc, + void *addr = nullptr) { + os << indent << "Source \"" << source_loc.filename << "\", line " + << source_loc.line << ", in " << source_loc.function; + + if (address && addr != nullptr) { + os << " [" << addr << "]"; + } + os << "\n"; + } +}; + +/*************** SIGNALS HANDLING ***************/ + +#if defined(BACKWARD_SYSTEM_LINUX) || defined(BACKWARD_SYSTEM_DARWIN) + +class SignalHandling { +public: + static std::vector make_default_signals() { + const int posix_signals[] = { + // Signals for which the default action is "Core". + SIGABRT, // Abort signal from abort(3) + SIGBUS, // Bus error (bad memory access) + SIGFPE, // Floating point exception + SIGILL, // Illegal Instruction + SIGIOT, // IOT trap. A synonym for SIGABRT + SIGQUIT, // Quit from keyboard + SIGSEGV, // Invalid memory reference + SIGSYS, // Bad argument to routine (SVr4) + SIGTRAP, // Trace/breakpoint trap + SIGXCPU, // CPU time limit exceeded (4.2BSD) + SIGXFSZ, // File size limit exceeded (4.2BSD) +#if defined(BACKWARD_SYSTEM_DARWIN) + SIGEMT, // emulation instruction executed +#endif + }; + return std::vector(posix_signals, + posix_signals + + sizeof posix_signals / sizeof posix_signals[0]); + } + + SignalHandling(const std::vector &posix_signals = make_default_signals()) + : _loaded(false) { + bool success = true; + + const size_t stack_size = 1024 * 1024 * 8; + _stack_content.reset(static_cast(malloc(stack_size))); + if (_stack_content) { + stack_t ss; + ss.ss_sp = _stack_content.get(); + ss.ss_size = stack_size; + ss.ss_flags = 0; + if (sigaltstack(&ss, nullptr) < 0) { + success = false; + } + } else { + success = false; + } + + for (size_t i = 0; i < posix_signals.size(); ++i) { + struct sigaction action; + memset(&action, 0, sizeof action); + action.sa_flags = + static_cast(SA_SIGINFO | SA_ONSTACK | SA_NODEFER | SA_RESETHAND); + sigfillset(&action.sa_mask); + sigdelset(&action.sa_mask, posix_signals[i]); +#if defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdisabled-macro-expansion" +#endif + action.sa_sigaction = &sig_handler; +#if defined(__clang__) +#pragma clang diagnostic pop +#endif + + int r = sigaction(posix_signals[i], &action, nullptr); + if (r < 0) + success = false; + } + + _loaded = success; + } + + bool loaded() const { return _loaded; } + + static void handleSignal(int, siginfo_t *info, void *_ctx) { + ucontext_t *uctx = static_cast(_ctx); + + StackTrace st; + void *error_addr = nullptr; +#ifdef REG_RIP // x86_64 + error_addr = reinterpret_cast(uctx->uc_mcontext.gregs[REG_RIP]); +#elif defined(REG_EIP) // x86_32 + error_addr = reinterpret_cast(uctx->uc_mcontext.gregs[REG_EIP]); +#elif defined(__arm__) + error_addr = reinterpret_cast(uctx->uc_mcontext.arm_pc); +#elif defined(__aarch64__) + #if defined(__APPLE__) + error_addr = reinterpret_cast(uctx->uc_mcontext->__ss.__pc); + #else + error_addr = reinterpret_cast(uctx->uc_mcontext.pc); + #endif +#elif defined(__mips__) + error_addr = reinterpret_cast( + reinterpret_cast(&uctx->uc_mcontext)->sc_pc); +#elif defined(__APPLE__) && defined(__POWERPC__) + error_addr = reinterpret_cast(uctx->uc_mcontext->__ss.__srr0); +#elif defined(__ppc__) || defined(__powerpc) || defined(__powerpc__) || \ + defined(__POWERPC__) + error_addr = reinterpret_cast(uctx->uc_mcontext.regs->nip); +#elif defined(__riscv) + error_addr = reinterpret_cast(uctx->uc_mcontext.__gregs[REG_PC]); +#elif defined(__s390x__) + error_addr = reinterpret_cast(uctx->uc_mcontext.psw.addr); +#elif defined(__APPLE__) && defined(__x86_64__) + error_addr = reinterpret_cast(uctx->uc_mcontext->__ss.__rip); +#elif defined(__APPLE__) + error_addr = reinterpret_cast(uctx->uc_mcontext->__ss.__eip); +#elif defined(__loongarch__) + error_addr = reinterpret_cast(uctx->uc_mcontext.__pc); +#else +#warning ":/ sorry, ain't know no nothing none not of your architecture!" +#endif + if (error_addr) { + st.load_from(error_addr, 32, reinterpret_cast(uctx), + info->si_addr); + } else { + st.load_here(32, reinterpret_cast(uctx), info->si_addr); + } + + Printer printer; + printer.address = true; + printer.print(st, stderr); + +#if (defined(_XOPEN_SOURCE) && _XOPEN_SOURCE >= 700) || \ + (defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 200809L) + psiginfo(info, nullptr); +#else + (void)info; +#endif + } + +private: + details::handle _stack_content; + bool _loaded; + +#ifdef __GNUC__ + __attribute__((noreturn)) +#endif + static void + sig_handler(int signo, siginfo_t *info, void *_ctx) { + handleSignal(signo, info, _ctx); + + // try to forward the signal. + raise(info->si_signo); + + // terminate the process immediately. + puts("watf? exit"); + _exit(EXIT_FAILURE); + } +}; + +#endif // BACKWARD_SYSTEM_LINUX || BACKWARD_SYSTEM_DARWIN + +#ifdef BACKWARD_SYSTEM_WINDOWS + +class SignalHandling { +public: + SignalHandling(const std::vector & = std::vector()) + : reporter_thread_([]() { + /* We handle crashes in a utility thread: + backward structures and some Windows functions called here + need stack space, which we do not have when we encounter a + stack overflow. + To support reporting stack traces during a stack overflow, + we create a utility thread at startup, which waits until a + crash happens or the program exits normally. */ + + { + std::unique_lock lk(mtx()); + cv().wait(lk, [] { return crashed() != crash_status::running; }); + } + if (crashed() == crash_status::crashed) { + handle_stacktrace(skip_recs()); + } + { + std::unique_lock lk(mtx()); + crashed() = crash_status::ending; + } + cv().notify_one(); + }) { + SetUnhandledExceptionFilter(crash_handler); + + signal(SIGABRT, signal_handler); + _set_abort_behavior(0, _WRITE_ABORT_MSG | _CALL_REPORTFAULT); + + std::set_terminate(&terminator); +#ifndef BACKWARD_ATLEAST_CXX17 + std::set_unexpected(&terminator); +#endif + _set_purecall_handler(&terminator); + _set_invalid_parameter_handler(&invalid_parameter_handler); + } + bool loaded() const { return true; } + + ~SignalHandling() { + { + std::unique_lock lk(mtx()); + crashed() = crash_status::normal_exit; + } + + cv().notify_one(); + + reporter_thread_.join(); + } + +private: + static CONTEXT *ctx() { + static CONTEXT data; + return &data; + } + + enum class crash_status { running, crashed, normal_exit, ending }; + + static crash_status &crashed() { + static crash_status data; + return data; + } + + static std::mutex &mtx() { + static std::mutex data; + return data; + } + + static std::condition_variable &cv() { + static std::condition_variable data; + return data; + } + + static HANDLE &thread_handle() { + static HANDLE handle; + return handle; + } + + std::thread reporter_thread_; + + // TODO: how not to hardcode these? + static const constexpr int signal_skip_recs = +#ifdef __clang__ + // With clang, RtlCaptureContext also captures the stack frame of the + // current function Below that, there are 3 internal Windows functions + 4 +#else + // With MSVC cl, RtlCaptureContext misses the stack frame of the current + // function The first entries during StackWalk are the 3 internal Windows + // functions + 3 +#endif + ; + + static int &skip_recs() { + static int data; + return data; + } + + static inline void terminator() { + crash_handler(signal_skip_recs); + abort(); + } + + static inline void signal_handler(int) { + crash_handler(signal_skip_recs); + abort(); + } + + static inline void __cdecl invalid_parameter_handler(const wchar_t *, + const wchar_t *, + const wchar_t *, + unsigned int, + uintptr_t) { + crash_handler(signal_skip_recs); + abort(); + } + + NOINLINE static LONG WINAPI crash_handler(EXCEPTION_POINTERS *info) { + // The exception info supplies a trace from exactly where the issue was, + // no need to skip records + crash_handler(0, info->ContextRecord); + return EXCEPTION_CONTINUE_SEARCH; + } + + NOINLINE static void crash_handler(int skip, CONTEXT *ct = nullptr) { + + if (ct == nullptr) { + RtlCaptureContext(ctx()); + } else { + memcpy(ctx(), ct, sizeof(CONTEXT)); + } + DuplicateHandle(GetCurrentProcess(), GetCurrentThread(), + GetCurrentProcess(), &thread_handle(), 0, FALSE, + DUPLICATE_SAME_ACCESS); + + skip_recs() = skip; + + { + std::unique_lock lk(mtx()); + crashed() = crash_status::crashed; + } + + cv().notify_one(); + + { + std::unique_lock lk(mtx()); + cv().wait(lk, [] { return crashed() != crash_status::crashed; }); + } + } + + static void handle_stacktrace(int skip_frames = 0) { + // printer creates the TraceResolver, which can supply us a machine type + // for stack walking. Without this, StackTrace can only guess using some + // macros. + // StackTrace also requires that the PDBs are already loaded, which is done + // in the constructor of TraceResolver + Printer printer; + + StackTrace st; + st.set_machine_type(printer.resolver().machine_type()); + st.set_thread_handle(thread_handle()); + st.load_here(32 + skip_frames, ctx()); + st.skip_n_firsts(skip_frames); + + printer.address = true; + printer.print(st, std::cerr); + } +}; + +#endif // BACKWARD_SYSTEM_WINDOWS + +#ifdef BACKWARD_SYSTEM_UNKNOWN + +class SignalHandling { +public: + SignalHandling(const std::vector & = std::vector()) {} + bool init() { return false; } + bool loaded() { return false; } +}; + +#endif // BACKWARD_SYSTEM_UNKNOWN + +} // namespace backward + +#endif /* H_GUARD */ diff --git a/tools/dkr_assets_tool_src/libs/n64graphics/n64graphics.c b/tools/dkr_assets_tool_src/libs/n64graphics/n64graphics.c index e3fc65a6..df8e7957 100644 --- a/tools/dkr_assets_tool_src/libs/n64graphics/n64graphics.c +++ b/tools/dkr_assets_tool_src/libs/n64graphics/n64graphics.c @@ -440,6 +440,69 @@ rgba *png2rgba(const char *png_filename, int *width, int *height) return img; } +rgba *pngdata2rgba(uint8_t *png_data, int data_size, int *width, int *height) +{ + rgba *img = NULL; + int w = 0; + int h = 0; + int channels = 0; + int img_size; + + stbi_uc *data = stbi_load_from_memory(png_data, data_size, &w, &h, &channels, STBI_default); + if (!data || w <= 0 || h <= 0) { + ERROR("Error loading png data\n"); + return NULL; + } + + img_size = w * h * sizeof(*img); + img = (rgba *)malloc(img_size); + if (!img) { + ERROR("Error allocating %u bytes\n", img_size); + return NULL; + } + + switch (channels) { + case 3: // red, green, blue + case 4: // red, green, blue, alpha + for (int j = 0; j < h; j++) { + for (int i = 0; i < w; i++) { + int idx = j*w + i; + img[idx].red = data[channels*idx]; + img[idx].green = data[channels*idx + 1]; + img[idx].blue = data[channels*idx + 2]; + if (channels == 4) { + img[idx].alpha = data[channels*idx + 3]; + } else { + img[idx].alpha = 0xFF; + } + } + } + break; + case 2: // grey, alpha + for (int j = 0; j < h; j++) { + for (int i = 0; i < w; i++) { + int idx = j*w + i; + img[idx].red = data[2*idx]; + img[idx].green = data[2*idx]; + img[idx].blue = data[2*idx]; + img[idx].alpha = data[2*idx + 1]; + } + } + break; + default: + ERROR("Don't know how to read channels: %d\n", channels); + free(img); + img = NULL; + } + + // cleanup + stbi_image_free(data); + + *width = w; + *height = h; + return img; +} + ia *png2ia(const char *png_filename, int *width, int *height) { ia *img = NULL; @@ -464,7 +527,67 @@ ia *png2ia(const char *png_filename, int *width, int *height) switch (channels) { case 3: // red, green, blue case 4: // red, green, blue, alpha - ERROR("Warning: averaging RGB PNG to create IA\n"); + //ERROR("Warning: averaging RGB PNG to create IA\n"); + for (int j = 0; j < h; j++) { + for (int i = 0; i < w; i++) { + int idx = j*w + i; + int sum = data[channels*idx] + data[channels*idx + 1] + data[channels*idx + 2]; + img[idx].intensity = (sum + 1) / 3; // add 1 to round up where appropriate + if (channels == 4) { + img[idx].alpha = data[channels*idx + 3]; + } else { + img[idx].alpha = 0xFF; + } + } + } + break; + case 2: // grey, alpha + for (int j = 0; j < h; j++) { + for (int i = 0; i < w; i++) { + int idx = j*w + i; + img[idx].intensity = data[2*idx]; + img[idx].alpha = data[2*idx + 1]; + } + } + break; + default: + ERROR("Don't know how to read channels: %d\n", channels); + free(img); + img = NULL; + } + + // cleanup + stbi_image_free(data); + + *width = w; + *height = h; + return img; +} + +ia *pngdata2ia(uint8_t *png_data, int data_size, int *width, int *height) +{ + ia *img = NULL; + int w = 0, h = 0; + int channels = 0; + int img_size; + + stbi_uc *data = stbi_load_from_memory(png_data, data_size, &w, &h, &channels, STBI_default); + if (!data || w <= 0 || h <= 0) { + ERROR("Error loading png data\n"); + return NULL; + } + + img_size = w * h * sizeof(*img); + img = (ia *)malloc(img_size); + if (!img) { + ERROR("Error allocating %d bytes\n", img_size); + return NULL; + } + + switch (channels) { + case 3: // red, green, blue + case 4: // red, green, blue, alpha + //ERROR("Warning: averaging RGB PNG to create IA\n"); for (int j = 0; j < h; j++) { for (int i = 0; i < w; i++) { int idx = j*w + i; diff --git a/tools/dkr_assets_tool_src/libs/n64graphics/n64graphics.h b/tools/dkr_assets_tool_src/libs/n64graphics/n64graphics.h index 6f73156d..52f3002f 100644 --- a/tools/dkr_assets_tool_src/libs/n64graphics/n64graphics.h +++ b/tools/dkr_assets_tool_src/libs/n64graphics/n64graphics.h @@ -87,9 +87,15 @@ int ia2png(const char *png_filename, const ia *img, int width, int height); // PNG file -> intermediate RGBA rgba *png2rgba(const char *png_filename, int *width, int *height); +// PNG data -> intermediate RGBA +rgba *pngdata2rgba(uint8_t *png_data, int data_size, int *width, int *height); + // PNG file -> intermediate IA ia *png2ia(const char *png_filename, int *width, int *height); +// PNG data -> intermediate IA +ia *pngdata2ia(uint8_t *png_data, int data_size, int *width, int *height); + //--------------------------------------------------------- // version diff --git a/tools/dkr_assets_tool_src/misc/constants.hpp b/tools/dkr_assets_tool_src/misc/constants.hpp index 22654239..674e2dda 100644 --- a/tools/dkr_assets_tool_src/misc/constants.hpp +++ b/tools/dkr_assets_tool_src/misc/constants.hpp @@ -7,7 +7,7 @@ namespace DkrAssetsTool { const std::string DKRAT_NAME = "DKR Assets Tool"; const int DKRAT_VERSION_RELEASE = 0; const int DKRAT_VERSION_MAJOR = 5; -const int DKRAT_VERSION_MINOR = 1; +const int DKRAT_VERSION_MINOR = 2; const std::string DKRAT_VERSION = "Version " + std::to_string(DKRAT_VERSION_RELEASE) + "." + std::to_string(DKRAT_VERSION_MAJOR) + "." + std::to_string(DKRAT_VERSION_MINOR); diff --git a/tools/python/score.py b/tools/python/score.py index 395b302e..3ac44583 100644 --- a/tools/python/score.py +++ b/tools/python/score.py @@ -148,6 +148,16 @@ class ScoreFile: for nonMatching in self.nonMatchings: self.nonMatchingsSizes += MAP_FILE.functionSizes[nonMatching] self.numNonMatchings = len(self.nonMatchings) +<<<<<<< HEAD + + self.nonEquivalents = re.findall(NON_EQUVIALENT_REGEX, self.text) + self.nonEquivalentsSizes = 0 + for nonEquivalent in self.nonEquivalents: + if (nonEquivalent not in NOT_FUNCTION_NAMES): + self.nonEquivalentsSizes += MAP_FILE.functionSizes[nonEquivalent] + self.numNonEquivalents = len(self.nonEquivalents) + +======= all_nonEquivalents = re.findall(NON_EQUVIALENT_REGEX, self.text) # Filter out the ones that are in NOT_FUNCTION_NAMES self.nonEquivalents = [ne for ne in all_nonEquivalents if ne not in NOT_FUNCTION_NAMES] @@ -156,6 +166,7 @@ class ScoreFile: for nonEquivalent in self.nonEquivalents: self.nonEquivalentsSizes += MAP_FILE.functionSizes[nonEquivalent] self.numNonEquivalents = len(self.nonEquivalents) +>>>>>>> master self.text = re.sub(WIP_REGEX, r"GLOBAL_ASM(\1)", self.text) def get_matches(self):