izzy/Torch
0
0
Fork 0
mirror of https://github.com/izzy2lost/Torch.git synced 2026-07-06 00:18:35 -07:00
Code Issues Packages Projects Releases Wiki Activity

Fixed dependency system and modding reimport

Browse Source
This commit is contained in:
KiritoDv
2024-05-15 19:16:26 -06:00
parent 311638a3bf
commit b6524bfb13
7 changed files with 207 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/Companion.cpp
+58 -30
View File
@@ -163,7 +163,7 @@ void Companion::ParseEnums(std::string& header) {
continue;
}
if(line.find("}") != std::string::npos) {
if(line.find('}') != std::string::npos) {
inEnum = false;
continue;
}
@@ -171,10 +171,10 @@ void Companion::ParseEnums(std::string& header) {
// Remove any comments and non-alphanumeric characters
line = std::regex_replace(line, std::regex(R"((/\*.*?\*/)|(//.*$)|([^a-zA-Z0-9=_\-\.]))"), "");
if(line.find("=") != std::string::npos) {
auto value = line.substr(line.find("=") + 1);
auto name = line.substr(0, line.find("="));
enumIndex = static_cast<int32_t>(std::stoll(value, 0, 0));
if(line.find('=') != std::string::npos) {
auto value = line.substr(line.find('=') + 1);
auto name = line.substr(0, line.find('='));
enumIndex = static_cast<int32_t>(std::stoll(value, nullptr, 0));
this->gEnums[enumName][enumIndex] = name;
} else {
enumIndex++;
@@ -196,6 +196,7 @@ std::optional<ParseResultData> Companion::ParseNode(YAML::Node& node, std::strin
SPDLOG_INFO("- [{}] Processing {}", type, name);
}
spdlog::set_pattern(line);
node["vpath"] = name;
auto factory = this->GetFactory(type);
if(!factory.has_value()){
@@ -205,27 +206,32 @@ std::optional<ParseResultData> Companion::ParseNode(YAML::Node& node, std::strin
auto impl = factory->get();
auto exporter = impl->GetExporter(this->gConfig.exporterType);
if(!exporter.has_value()){
if(!exporter.has_value() && !impl->HasModdedDependencies()){
SPDLOG_WARN("No exporter found for {}", name);
return std::nullopt;
}
bool executeDef = true;
std::optional<std::shared_ptr<IParsedData>> result;
if(this->gConfig.modding && impl->SupportModdedAssets() && this->gModdedAssetPaths.contains(name)) {
auto path = fs::path(this->gConfig.moddingPath) / this->gModdedAssetPaths[name];
if(!fs::exists(path)) {
if(!exists(path)) {
SPDLOG_ERROR("Modded asset {} not found", this->gModdedAssetPaths[name]);
return std::nullopt;
} else {
std::ifstream input(path, std::ios::binary);
std::vector<uint8_t> data = std::vector<uint8_t>( std::istreambuf_iterator( input ), {});
input.close();
result = impl->parse_modding(data, node);
executeDef = !result.has_value();
}
}
std::ifstream input(path, std::ios::binary);
std::vector<uint8_t> data = std::vector<uint8_t>( std::istreambuf_iterator( input ), {});
input.close();
result = impl->parse_modding(data, node);
} else if(this->gConfig.parseMode == ParseMode::Default) {
if(executeDef && this->gConfig.parseMode == ParseMode::Default) {
result = impl->parse(this->gRomData, node);
} else if(this->gConfig.parseMode == ParseMode::Directory) {
}
if(executeDef && this->gConfig.parseMode == ParseMode::Directory) {
auto path = GetSafeNode<std::string>(node, "path");
std::ifstream input( path, std::ios::binary );
auto data = std::vector<uint8_t>( std::istreambuf_iterator( input ), {} );
@@ -238,16 +244,16 @@ std::optional<ParseResultData> Companion::ParseNode(YAML::Node& node, std::strin
return std::nullopt;
}
for (auto [fst, snd] : this->gAssetDependencies[this->gCurrentFile]) {
if(snd.second) {
for (auto& [name, node, processed] : this->gDependencies) {
if(processed) {
continue;
}
std::string doutput = (this->gCurrentDirectory / fst).string();
std::string doutput = (this->gCurrentDirectory / name).string();
std::replace(doutput.begin(), doutput.end(), '\\', '/');
this->gAssetDependencies[this->gCurrentFile][fst].second = true;
auto result = this->ParseNode(snd.first, doutput);
if(result.has_value()) {
this->gParseResults[this->gCurrentFile].push_back(result.value());
processed = true;
auto dResult = this->ParseNode(node, doutput);
if(dResult.has_value()) {
this->gParseResults[this->gCurrentFile].push_back(dResult.value());
}
spdlog::set_pattern(regular);
SPDLOG_INFO("------------------------------------------------");
@@ -256,6 +262,7 @@ std::optional<ParseResultData> Companion::ParseNode(YAML::Node& node, std::strin
SPDLOG_INFO("Processed {}", name);
this->gDependencies.clear();
return ParseResultData {
name, type, node, result
@@ -397,6 +404,7 @@ void Companion::ParseCurrentFileConfig(YAML::Node node) {
this->gEnablePadGen = GetSafeNode<bool>(node, "autopads", true);
this->gNodeForceProcessing = GetSafeNode<bool>(node, "force", false);
this->gIndividualIncludes = GetSafeNode<bool>(node, "individual_data_incs", false);
this->gCurrentVirtualPath = GetSafeNode<std::string>(node, "path", "");
}
void Companion::ParseHash() {
@@ -545,6 +553,10 @@ void Companion::ProcessFile(YAML::Node root) {
assetNode["offset"] = (segment << 24) | assetNode["offset"].as<uint32_t>();
}
if(!gCurrentVirtualPath.empty()) {
node["path"] = gCurrentVirtualPath;
}
this->gAddrMap[this->gCurrentFile][assetNode["offset"].as<uint32_t>()] = std::make_tuple(output, assetNode);
}
} else {
@@ -570,6 +582,10 @@ void Companion::ProcessFile(YAML::Node root) {
}
}
if(!gCurrentVirtualPath.empty()) {
node["path"] = gCurrentVirtualPath;
}
this->gAddrMap[this->gCurrentFile][node["offset"].as<uint32_t>()] = std::make_tuple(output, node);
}
}
@@ -580,6 +596,7 @@ void Companion::ProcessFile(YAML::Node root) {
this->gFileHeader.clear();
this->gCurrentPad = 0;
this->gCurrentVram = std::nullopt;
this->gCurrentVirtualPath = "";
this->gCurrentSegmentNumber = 0;
this->gCurrentCompressionType = CompressionType::None;
this->gCurrentFileOffset = 0;
@@ -624,6 +641,10 @@ void Companion::ProcessFile(YAML::Node root) {
node["offset"] = (segment << 24) | node["offset"].as<uint32_t>();
}
if(!gCurrentVirtualPath.empty()) {
node["path"] = gCurrentVirtualPath;
}
auto output = (this->gCurrentDirectory / entryName / childName).string();
std::replace(output.begin(), output.end(), '\\', '/');
this->gConfig.segment.temporal.clear();
@@ -639,6 +660,11 @@ void Companion::ProcessFile(YAML::Node root) {
assetNode["offset"] = (gCurrentSegmentNumber << 24) | offset;
}
}
if(!gCurrentVirtualPath.empty()) {
assetNode["path"] = gCurrentVirtualPath;
}
std::string output = (this->gCurrentDirectory / entryName).string();
std::replace(output.begin(), output.end(), '\\', '/');
this->gConfig.segment.temporal.clear();
@@ -662,7 +688,7 @@ void Companion::ProcessFile(YAML::Node root) {
const auto impl = this->GetFactory(result.type)->get();
const auto exporter = impl->GetExporter(this->gConfig.exporterType);
if(exporter == nullptr) {
if(!exporter.has_value()) {
continue;
}
@@ -691,6 +717,7 @@ void Companion::ProcessFile(YAML::Node root) {
create_directories(fs::path(dpath).parent_path());
}
SPDLOG_INFO(ogname);
this->gModdedAssetPaths[ogname] = result.name;
std::ofstream file(dpath, std::ios::binary);
@@ -1199,7 +1226,9 @@ std::optional<std::tuple<std::string, YAML::Node>> Companion::RegisterAsset(cons
return std::nullopt;
}
this->gAssetDependencies[this->gCurrentFile][name] = std::make_pair(node, false);
this->gDependencies.push_back(Dependency {
name, node, false
});
auto output = (this->gCurrentDirectory / name).string();
std::replace(output.begin(), output.end(), '\\', '/');
@@ -1394,13 +1423,12 @@ std::string Companion::CalculateHash(const std::vector<uint8_t>& data) {
}
static std::string ConvertType(std::string type) {
int index;
int index = type.find(':');
if((index = type.find(':')) != std::string::npos) {
if(index != std::string::npos) {
type = type.substr(index + 1);
}
type = std::regex_replace(type, std::regex(R"([^_A-Za-z0-9]*)"), "");
std::transform(type.begin(), type.end(), type.begin(), ::tolower);
std::transform(type.begin(), type.end(), type.begin(), tolower);
return type;
}
@@ -1428,7 +1456,7 @@ std::optional<YAML::Node> Companion::AddAsset(YAML::Node asset) {
std::string output;
std::string typeId = ConvertType(type);
if(symbol != "") {
if(!symbol.empty()) {
output = symbol;
} else if(Decompressor::IsSegmented(offset)){
output = this->NormalizeAsset("seg" + std::to_string(SEGMENT_NUMBER(offset)) +"_" + typeId + "_" + Torch::to_hex(SEGMENT_OFFSET(offset), false));
@@ -1441,7 +1469,7 @@ std::optional<YAML::Node> Companion::AddAsset(YAML::Node asset) {
auto result = this->RegisterAsset(output, asset);
if(result.has_value()){
asset["path"] = std::get<0>(result.value());
asset["vpath"] = std::get<0>(result.value());
return std::get<1>(result.value());
}
src/Companion.h
+8 -1
View File
@@ -99,6 +99,12 @@ struct ParseResultData {
}
};
struct Dependency {
std::string name;
YAML::Node node;
bool processed;
};
class Companion {
public:
static Companion* Instance;
@@ -171,6 +177,7 @@ private:
// Temporal Variables
std::string gCurrentFile;
std::string gCurrentVirtualPath;
std::string gFileHeader;
bool gEnablePadGen = false;
uint32_t gCurrentPad = 0;
@@ -183,12 +190,12 @@ private:
std::unordered_set<std::string> gProcessedFiles;
std::unordered_map<std::string, std::vector<ParseResultData>> gParseResults;
std::vector<Dependency> gDependencies;
std::unordered_map<std::string, std::string> gModdedAssetPaths;
std::variant<std::vector<std::string>, std::string> gWriteOrder;
std::unordered_map<std::string, std::shared_ptr<BaseFactory>> gFactories;
std::unordered_map<std::string, std::map<std::string, std::vector<WriteEntry>>> gWriteMap;
std::unordered_map<std::string, std::map<std::string, std::pair<YAML::Node, bool>>> gAssetDependencies;
std::unordered_map<std::string, std::unordered_map<uint32_t, std::tuple<std::string, YAML::Node>>> gAddrMap;
void ProcessFile(YAML::Node root);
src/factories/BaseFactory.h
+3
View File
@@ -101,6 +101,9 @@ public:
virtual bool SupportModdedAssets() {
return false;
}
virtual bool HasModdedDependencies() {
return false;
}
virtual uint32_t GetAlignment() {
return 4;
}
src/factories/TextureFactory.cpp
+3 -3
View File
@@ -245,7 +245,7 @@ ExportResult TextureModdingExporter::Export(std::ostream&write, std::shared_ptr<
auto ext = GetSafeNode<std::string>(node, "format");
std::transform(ext.begin(), ext.end(), ext.begin(), tolower);
(*replacement) += "." + ext + ".png";
*replacement += "." + ext + ".png";
switch (format.type) {
case TextureType::TLUT:
@@ -269,7 +269,6 @@ ExportResult TextureModdingExporter::Export(std::ostream&write, std::shared_ptr<
}
case TextureType::Palette8bpp:
case TextureType::Palette4bpp: {
// This check needed until sf64 has tluts fixed.
if (node["tlut_symbol"]) {
auto tlut = GetSafeNode<std::string>(node,"tlut_symbol");
auto palette = Companion::Instance->GetParseDataBySymbol(tlut);
@@ -474,7 +473,8 @@ std::optional<std::shared_ptr<IParsedData>> TextureFactory::parse_modding(std::v
// } else {
// }
break;
SPDLOG_ERROR("Unsupported texture format for modding: {}", format);
return std::nullopt;
}
case TextureType::Grayscale8bpp:
case TextureType::Grayscale4bpp: {
src/factories/sf64/MessageFactory.cpp
+127 -1
View File
@@ -4,6 +4,7 @@
#include "spdlog/spdlog.h"
#include "Companion.h"
#include <regex>
#include <sstream>
#define END_CODE 0
#define NEWLINE_CODE 1
@@ -50,6 +51,21 @@ std::unordered_map<std::string, std::string> ASCIITable = {
{ "CLN", ":" }, { "PIP", "| " }
};
std::vector<std::string> gASCIIFullTable = {
"\0", "\n", "{NP:2}", "{NP:3}", "{NP:4}", "{NP:5}", "{NP:6}", "{NP:7}",
"{PRI:0}", "{PRI:1}", "{PRI:2}", "{PRI:3}", " ", "{HSP}", "{QSP}", "{NXT}",
"{C:<}", "{C:^}", "{C:>}", "{C:v}", "^", "<", "v", ">",
"A", "B", "C", "D", "E", "F", "G", "H",
"I", "J", "K", "L", "M", "N", "O", "P",
"Q", "R", "S", "T", "U", "V", "W", "X",
"Y", "Z", "a", "b", "c", "d", "e", "f",
"g", "h", "i", "j", "k", "l", "m", "n",
"o", "p", "q", "r", "s", "t", "u", "v",
"w", "x", "y", "z", "!", "?", "-", ",",
".", "0", "1", "2", "3", "4", "5", "6",
"7", "8", "9", "'", "(", ")", ":", "|",
};
ExportResult SF64::MessageHeaderExporter::Export(std::ostream &write, std::shared_ptr<IParsedData> raw, std::string& entryName, YAML::Node &node, std::string* replacement) {
const auto symbol = GetSafeNode(node, "symbol", entryName);
@@ -105,6 +121,34 @@ ExportResult SF64::MessageBinaryExporter::Export(std::ostream &write, std::share
return std::nullopt;
}
ExportResult SF64::MessageModdingExporter::Export(std::ostream&write, std::shared_ptr<IParsedData> raw, std::string&entryName, YAML::Node&node, std::string* replacement) {
const auto data = std::static_pointer_cast<MessageData>(raw);
const auto symbol = GetSafeNode(node, "symbol", entryName);
*replacement += ".yml";
auto count = data->mMessage.size();
std::stringstream stream;
YAML::Emitter out;
out << YAML::BeginMap;
out << YAML::Key << symbol;
out << YAML::Value << YAML::BeginSeq;
for(size_t i = 0; i < count; i++) {
if(data->mMessage[i] == NEWLINE_CODE){
stream << "\0";
out << stream.str();
stream.str("");
} else {
stream << gASCIIFullTable[data->mMessage[i]];
}
}
out << YAML::EndSeq << YAML::EndMap;
write.write(out.c_str(), out.size());
return std::nullopt;
}
std::optional<std::shared_ptr<IParsedData>> SF64::MessageFactory::parse(std::vector<uint8_t>& buffer, YAML::Node& node) {
std::vector<uint16_t> message;
std::ostringstream mesgStr;
@@ -113,7 +157,6 @@ std::optional<std::shared_ptr<IParsedData>> SF64::MessageFactory::parse(std::vec
reader.SetEndianness(LUS::Endianness::Big);
uint16_t c;
bool lastWasSpace = false;
std::string whitespace = "";
do {
c = reader.ReadUInt16();
@@ -138,5 +181,88 @@ std::optional<std::shared_ptr<IParsedData>> SF64::MessageFactory::parse(std::vec
} while(c != END_CODE);
return std::make_shared<MessageData>(message, mesgStr.str());
}
std::optional<uint16_t> getCharByCode(const std::string& code) {
auto it = std::find(gASCIIFullTable.begin(), gASCIIFullTable.end(), code) - gASCIIFullTable.begin();
if(it < gASCIIFullTable.size()){
return it;
}
return std::nullopt;
}
std::optional<std::shared_ptr<IParsedData>> SF64::MessageFactory::parse_modding(std::vector<uint8_t>& buffer, YAML::Node& data) {
std::vector<uint16_t> message;
std::ostringstream mesgStr;
std::string whitespace = "";
YAML::Node node;
try {
std::string text((char*) buffer.data(), buffer.size());
node = YAML::Load(text.c_str());
} catch (YAML::ParserException& e) {
SPDLOG_ERROR("Failed to parse message data: {}", e.what());
SPDLOG_ERROR("{}", (char*) buffer.data());
return std::nullopt;
}
std::regex fmt("\\(([^)]+)\\)");
std::vector<std::string> lines = node.begin()->second.as<std::vector<std::string>>();
for(auto& line : lines){
for(size_t i = 0; i < line.size(); i++){
char c = line[i];
std::string enumCode;
if(c == '{' && line.substr(i).find('}') != std::string::npos){
auto code = line.substr(i, line.substr(i).find('}') + 1);
auto opcode = getCharByCode(code);
if(opcode.has_value()){
auto x = opcode.value();
message.push_back(x);
i += line.substr(i).find('}');
enumCode = gCharCodeEnums[x];
} else {
SPDLOG_WARN("Unknown character: {}", code);
}
} else {
auto code = getCharByCode(std::string(1, c));
if(code.has_value()){
auto x = code.value();
message.push_back(x);
enumCode = gCharCodeEnums[x];
} else {
SPDLOG_WARN("Unknown character: {}", c);
}
}
if(!enumCode.empty()) {
if(enumCode.find("SP") != std::string::npos && whitespace.empty()) {
whitespace = " ";
}
if(c == NEWLINE_CODE) {
whitespace += "\n";
}
if (c >= CLF_CODE) {
mesgStr << whitespace;
whitespace = "";
}
if(enumCode.starts_with("_")){
mesgStr << enumCode.substr(1);
} else if(ASCIITable.contains(enumCode)){
mesgStr << ASCIITable[enumCode];
}
}
}
if(line != lines.back()){
message.push_back(NEWLINE_CODE);
mesgStr << "\n";
}
}
message.push_back(END_CODE);
return std::make_shared<MessageData>(message, mesgStr.str());
}
src/factories/sf64/MessageFactory.h
+7
View File
@@ -24,15 +24,22 @@ class MessageBinaryExporter : public BaseExporter {
ExportResult Export(std::ostream& write, std::shared_ptr<IParsedData> data, std::string& entryName, YAML::Node& node, std::string* replacement) override;
};
class MessageModdingExporter : public BaseExporter {
ExportResult Export(std::ostream& write, std::shared_ptr<IParsedData> data, std::string& entryName, YAML::Node& node, std::string* replacement) override;
};
class MessageFactory : public BaseFactory {
public:
std::optional<std::shared_ptr<IParsedData>> parse(std::vector<uint8_t>& buffer, YAML::Node& data) override;
std::optional<std::shared_ptr<IParsedData>> parse_modding(std::vector<uint8_t>& buffer, YAML::Node& data) override;
inline std::unordered_map<ExportType, std::shared_ptr<BaseExporter>> GetExporters() override {
return {
REGISTER(Code, MessageCodeExporter)
REGISTER(Header, MessageHeaderExporter)
REGISTER(Binary, MessageBinaryExporter)
REGISTER(Modding, MessageModdingExporter)
};
}
bool SupportModdedAssets() override { return true; }
};
}
src/factories/sf64/MessageLookupFactory.h
+1
View File
@@ -38,5 +38,6 @@ public:
REGISTER(Binary, MessageLookupBinaryExporter)
};
}
bool HasModdedDependencies() override { return true; }
};
}