From 683bb80c7252a5414967e6ed1dea20a05f941997 Mon Sep 17 00:00:00 2001 From: someone2639 Date: Tue, 29 Aug 2023 10:06:41 -0400 Subject: [PATCH] file purge + extract_assets revamp (#644) * remove gen_asset_list (frozen) * move format.sh * remove rasm2armips (decompilation helper) * remove patch_libmalloc (IDO exclusive) * remove unnecessary sha1's and makefile definitions * remove libultra math script (IDO exclusive) * remove patch_elf_32bit * actually remove bitwidth tool * remove the need for all the sha1 files * extract_assets can now find a baserom named anything * added error handling and some more graceful error handling * formatting the error message * more relevant error info * add a check for a roms folder * fix gitignore to prevent patch_elf from being re-committed * naming clarity; change to baseroms/; feedback addressed * skip rom verification if assets have been extracted * changes addressed --------- Co-authored-by: someone2639 --- Makefile | 10 - extract_assets.py | 129 ++++++---- sm64.eu.sha1 | 1 - sm64.jp.sha1 | 1 - sm64.sh.sha1 | 1 - sm64.us.sha1 | 1 - tools/Makefile | 4 +- tools/determine-endian-bitwidth.c | 11 - format.sh => tools/format.sh | 0 tools/gen_asset_list.cpp | 382 ------------------------------ tools/patch_elf_32bit.c | 161 ------------- tools/patch_libmalloc.py | 58 ----- tools/patch_libultra_math | Bin 14472 -> 0 bytes tools/rasm2armips.py | 76 ------ 14 files changed, 85 insertions(+), 750 deletions(-) delete mode 100644 sm64.eu.sha1 delete mode 100644 sm64.jp.sha1 delete mode 100644 sm64.sh.sha1 delete mode 100644 sm64.us.sha1 delete mode 100644 tools/determine-endian-bitwidth.c rename format.sh => tools/format.sh (100%) delete mode 100644 tools/gen_asset_list.cpp delete mode 100644 tools/patch_elf_32bit.c delete mode 100755 tools/patch_libmalloc.py delete mode 100755 tools/patch_libultra_math delete mode 100755 tools/rasm2armips.py diff --git a/Makefile b/Makefile index ca6b7b712..b91b09db1 100644 --- a/Makefile +++ b/Makefile @@ -488,7 +488,6 @@ CPPFLAGS := -P -Wno-trigraphs $(DEF_INC_CFLAGS) YAY0TOOL := $(TOOLS_DIR)/slienc MIO0TOOL := $(TOOLS_DIR)/mio0 RNCPACK := $(TOOLS_DIR)/rncpack -ROMALIGN := $(TOOLS_DIR)/romalign FILESIZER := $(TOOLS_DIR)/filesizer N64CKSUM := $(TOOLS_DIR)/n64cksum N64GRAPHICS := $(TOOLS_DIR)/n64graphics @@ -512,7 +511,6 @@ ifneq (,$(call find-command,armips)) else RSPASM := $(TOOLS_DIR)/armips endif -ENDIAN_BITWIDTH := $(BUILD_DIR)/endian-and-bitwidth EMULATOR = mupen64plus EMU_FLAGS = @@ -736,14 +734,6 @@ $(BUILD_DIR)/%.aifc: $(BUILD_DIR)/%.table %.aiff $(call print,Encoding ADPCM:,$(word 2,$^),$@) $(V)$(VADPCM_ENC) -c $^ $@ -$(ENDIAN_BITWIDTH): $(TOOLS_DIR)/determine-endian-bitwidth.c - @$(PRINT) "$(GREEN)Generating endian-bitwidth $(NO_COL)\n" - $(V)$(CC) -c $(CFLAGS) -o $@.dummy2 $< 2>$@.dummy1; true - $(V)grep -o 'msgbegin --endian .* --bitwidth .* msgend' $@.dummy1 > $@.dummy2 - $(V)head -n1 <$@.dummy2 | cut -d' ' -f2-5 > $@ - $(V)$(RM) $@.dummy1 - $(V)$(RM) $@.dummy2 - $(SOUND_BIN_DIR)/sound_data.ctl: sound/sound_banks/ $(SOUND_BANK_FILES) $(SOUND_SAMPLE_AIFCS) $(ENDIAN_BITWIDTH) @$(PRINT) "$(GREEN)Generating: $(BLUE)$@ $(NO_COL)\n" $(V)$(PYTHON) $(TOOLS_DIR)/assemble_sound.py $(BUILD_DIR)/sound/samples/ sound/sound_banks/ $(SOUND_BIN_DIR)/sound_data.ctl $(SOUND_BIN_DIR)/ctl_header $(SOUND_BIN_DIR)/sound_data.tbl $(SOUND_BIN_DIR)/tbl_header $(C_DEFINES) $$(cat $(ENDIAN_BITWIDTH)) diff --git a/extract_assets.py b/extract_assets.py index bde5ef216..c072a3ee9 100755 --- a/extract_assets.py +++ b/extract_assets.py @@ -2,6 +2,56 @@ import sys import os import json +import subprocess + +ROMS_DIR=os.path.expanduser("~/baseroms/") + +sha1_LUT = { + "eu": "4ac5721683d0e0b6bbb561b58a71740845dceea9", + "jp": "8a20a5c83d6ceb0f0506cfc9fa20d8f438cafe51", + "sh": "3f319ae697533a255a1003d09202379d78d5a2e0", + "us": "9bef1128717f958171a4afac3ed78ee2bb4e86ce", +} + +sha1_swapLUT = { + "eu": "d80ee9eeb6454d53a96ceb6ed0aca3ffde045091", + "jp": "1d2579dd5fb1d8263a4bcc063a651a64acc88921", + "sh": "2a2b85e94581545ca3c05b8f864b488b141a8a1f", + "us": "1002dd7b56aa0a59a9103f1fb3d57d6b161f8da7", +} + +def get_rom_candidates(): + fileArray = [f for f in os.listdir(os.getcwd()) if os.path.isfile(f)] + if os.path.exists(ROMS_DIR): + fileArray += [os.path.join(ROMS_DIR, f) for f in os.listdir(ROMS_DIR) if os.path.isfile(os.path.join(ROMS_DIR, f))] + + foundVersions = {} + + for f in fileArray: + try: + p = subprocess.Popen( + ["sha1sum", f], + stdout=subprocess.PIPE + ) + sha1sum = p.communicate()[0].decode('ascii').split()[0] + for k, v in sha1_LUT.items(): + if v == sha1sum: + foundVersions[k] = f + + for k, v in sha1_swapLUT.items(): + if v == sha1sum: # the ROM is swapped! + subprocess.run( + [ + "dd","conv=swab", + "if=%s" % f, + "of=/tmp/baserom.%s.swapped.z64" % k + ], + stderr=subprocess.PIPE, + ) + foundVersions[k] = "/tmp/baserom.%s.swapped.z64" % k + except Exception as e: + continue + return foundVersions def read_asset_map(): @@ -18,25 +68,6 @@ def read_local_asset_list(f): ret.append(line.strip()) return ret - -def asset_needs_update(asset, version): - if version <= 6 and asset in ["actors/king_bobomb/king_bob-omb_eyes.rgba16.png", "actors/king_bobomb/king_bob-omb_hand.rgba16.png"]: - return True - if version <= 5 and asset == "textures/spooky/bbh_textures.00800.rgba16.png": - return True - if version <= 4 and asset in ["textures/mountain/ttm_textures.01800.rgba16.png", "textures/mountain/ttm_textures.05800.rgba16.png"]: - return True - if version <= 3 and asset == "textures/cave/hmc_textures.01800.rgba16.png": - return True - if version <= 2 and asset == "textures/inside/inside_castle_textures.09000.rgba16.png": - return True - if version <= 1 and asset.endswith(".m64"): - return True - if version <= 0 and asset.endswith(".aiff"): - return True - return False - - def remove_file(fname): os.remove(fname) print("deleting", fname) @@ -76,13 +107,6 @@ def main(): clean_assets(local_asset_file) sys.exit(0) - all_langs = ["jp", "us", "eu", "sh"] - if not langs or not all(a in all_langs for a in langs): - langs_str = " ".join("[" + lang + "]" for lang in all_langs) - print("Usage: " + sys.argv[0] + " " + langs_str) - print("For each version, baserom..z64 must exist") - sys.exit(1) - asset_map = read_asset_map() all_assets = [] any_missing_assets = False @@ -101,8 +125,30 @@ def main(): # the list of old assets either. return + romLUT = get_rom_candidates() + + + # verify the correct rom + for lang in langs: + if lang not in romLUT: + print("[%s] Error: No %s ROM detected in this folder." + % (sys.argv[0], lang.upper()) + ) + if len(romLUT.items()) > 0: + print() + print("Detected ROMS:") + for k,v in romLUT.items(): + print(" %s ROM found at: %s" % (k.upper(), v)) + sys.exit(1) + + all_langs = ["jp", "us", "eu", "sh"] + if not langs or not all(a in all_langs for a in langs): + langs_str = " ".join("[" + lang + "]" for lang in all_langs) + print("Usage: " + sys.argv[0] + " " + langs_str) + print("For each version, its ROM file must exist in this folder") + sys.exit(1) + # Late imports (to optimize startup perf) - import subprocess import hashlib import tempfile from collections import defaultdict @@ -119,7 +165,7 @@ def main(): todo = defaultdict(lambda: []) for (asset, data, exists) in all_assets: # Leave existing assets alone if they have a compatible version. - if exists and not asset_needs_update(asset, local_version): + if exists: continue meta = data[:-2] @@ -134,25 +180,16 @@ def main(): # Load ROMs roms = {} for lang in langs: - fname = "baserom." + lang + ".z64" + romname = romLUT[lang] try: - with open(fname, "rb") as f: + with open(romname, "rb") as f: roms[lang] = f.read() except Exception as e: - print("Failed to open " + fname + "! " + str(e)) - sys.exit(1) - sha1 = hashlib.sha1(roms[lang]).hexdigest() - with open("sm64." + lang + ".sha1", "r") as f: - expected_sha1 = f.read().split()[0] - if sha1 != expected_sha1: - print( - fname - + " has the wrong hash! Found " - + sha1 - + ", expected " - + expected_sha1 - ) + print("Failed to open " + romname + "! " + str(e)) sys.exit(1) + # There used to be an SHA1 check here, + # but it's unnecessary since we detect the + # presence of the correct roms automatically # Make sure tools exist subprocess.check_call( @@ -165,14 +202,16 @@ def main(): # Import new assets for key in keys: + assets = todo[key] lang, mio0 = key + romname = romLUT[lang] if mio0 == "@sound": rom = roms[lang] args = [ "python3", "tools/disassemble_sound.py", - "baserom." + lang + ".z64", + romname, ] def append_args(key): size, locs = asset_map["@sound " + key + " " + lang] @@ -199,7 +238,7 @@ def main(): "-d", "-o", str(mio0), - "baserom." + lang + ".z64", + romname, "-", ], check=True, diff --git a/sm64.eu.sha1 b/sm64.eu.sha1 deleted file mode 100644 index 49e8d7282..000000000 --- a/sm64.eu.sha1 +++ /dev/null @@ -1 +0,0 @@ -4ac5721683d0e0b6bbb561b58a71740845dceea9 build/eu/sm64.eu.z64 diff --git a/sm64.jp.sha1 b/sm64.jp.sha1 deleted file mode 100644 index 1d43a0d56..000000000 --- a/sm64.jp.sha1 +++ /dev/null @@ -1 +0,0 @@ -8a20a5c83d6ceb0f0506cfc9fa20d8f438cafe51 build/jp/sm64.jp.z64 diff --git a/sm64.sh.sha1 b/sm64.sh.sha1 deleted file mode 100644 index fd0d7f91c..000000000 --- a/sm64.sh.sha1 +++ /dev/null @@ -1 +0,0 @@ -3f319ae697533a255a1003d09202379d78d5a2e0 build/sh/sm64.sh.z64 diff --git a/sm64.us.sha1 b/sm64.us.sha1 deleted file mode 100644 index 444a7d110..000000000 --- a/sm64.us.sha1 +++ /dev/null @@ -1 +0,0 @@ -9bef1128717f958171a4afac3ed78ee2bb4e86ce build/us/sm64.us.z64 diff --git a/tools/Makefile b/tools/Makefile index affc5de80..03c5df769 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -7,7 +7,7 @@ CC := gcc CXX := g++ CFLAGS := -I. -O2 -s LDFLAGS := -lm -ALL_PROGRAMS := armips filesizer rncpack n64graphics n64graphics_ci mio0 slienc n64cksum textconv patch_elf_32bit aifc_decode aiff_extract_codebook vadpcm_enc tabledesign extract_data_for_mio skyconv flips +ALL_PROGRAMS := armips filesizer rncpack n64graphics n64graphics_ci mio0 slienc n64cksum textconv aifc_decode aiff_extract_codebook vadpcm_enc tabledesign extract_data_for_mio skyconv flips LIBAUDIOFILE := audiofile/libaudiofile.a # Only build armips from tools if it is not found on the system @@ -39,8 +39,6 @@ n64cksum_CFLAGS := -DN64CKSUM_STANDALONE textconv_SOURCES := textconv.c utf8.c hashtable.c -patch_elf_32bit_SOURCES := patch_elf_32bit.c - aifc_decode_SOURCES := aifc_decode.c aiff_extract_codebook_SOURCES := aiff_extract_codebook.c diff --git a/tools/determine-endian-bitwidth.c b/tools/determine-endian-bitwidth.c deleted file mode 100644 index 15efe99a8..000000000 --- a/tools/determine-endian-bitwidth.c +++ /dev/null @@ -1,11 +0,0 @@ -#include "platform_info.h" - -#if IS_BIG_ENDIAN && IS_64_BIT -#error msgbegin --endian big --bitwidth 64 msgend -#elif IS_BIG_ENDIAN && !IS_64_BIT -#error msgbegin --endian big --bitwidth 32 msgend -#elif !IS_BIG_ENDIAN && IS_64_BIT -#error msgbegin --endian little --bitwidth 64 msgend -#else -#error msgbegin --endian little --bitwidth 32 msgend -#endif diff --git a/format.sh b/tools/format.sh similarity index 100% rename from format.sh rename to tools/format.sh diff --git a/tools/gen_asset_list.cpp b/tools/gen_asset_list.cpp deleted file mode 100644 index 050bd1187..000000000 --- a/tools/gen_asset_list.cpp +++ /dev/null @@ -1,382 +0,0 @@ -// WARNING: THIS SCRIPT IS CURRENTLY BROKEN. -// It doesn't handle skyboxes/cake images correctly. - -// Usage: -// g++-8 -std=c++17 ./tools/gen_asset_list.cpp -lstdc++fs -O1 -Wall -o tools/gen_asset_list -// ./tools/gen_asset_list - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -using namespace std; - -#define BSWAP32(x) ((((x) >> 24) & 0xff) | (((x) >> 8) & 0xff00) | (((x) << 8) & 0xff0000) | (((x) << 24) & 0xff000000U)) -#define BSWAP16(x) ((((x) >> 8) & 0xff) | (((x) << 8) & 0xff00)) - -const char* OUTPUT_FILE = "assets.json"; -const size_t CHUNK_SIZE = 16; -const vector LANGS = {"jp", "us", "eu", "sh"}; - -typedef uint8_t u8; -typedef uint64_t u64; - -struct Pos { - size_t pos; - size_t mio0; -}; - -const u64 C = 12318461241ULL; - -size_t findCutPos(const string& s) { - size_t ind = s.find_first_not_of(s[0], 1); - if (ind == string::npos) ind = 0; - else ind--; - if (ind + CHUNK_SIZE <= s.size()) - return ind; - return s.size() - CHUNK_SIZE; -} - -pair hashString(const string& inp) { - size_t cutPos = findCutPos(inp); - string s = inp.substr(cutPos, CHUNK_SIZE); - u64 ret = 0; - for (u8 c : s) { - ret *= C; - ret += c; - } - return {cutPos, ret}; -} - -template -void rollingHashes(const string& str, size_t chunkSize, F&& f) { - if (str.size() < chunkSize) return; - u64 h = 0, pw = 1; - for (size_t i = 0; i < chunkSize; i++) - h = h * C + (u8)str[i], pw = pw * C; - f(0, h); - for (size_t i = chunkSize; i < str.size(); i++) { - h = h * C + (u8)str[i] - pw * (u8)str[i-chunkSize]; - f(i - chunkSize + 1, h); - } -} - -bool stringMatches(const string& base, size_t pos, const string& target) { - if (pos + target.size() > base.size()) return false; - for (int it = 0; it < 10; it++) { - size_t i = rand() % target.size(); - if (base[pos + i] != target[i]) return false; - } - for (size_t i = 0; i < target.size(); i++) { - if (base[pos + i] != target[i]) return false; - } - return true; -} - -string mio0_decompress(uint32_t *src) { - uint32_t size = BSWAP32(src[1]); - string output(size, '\0'); - char *dest = output.data(); - char *destEnd = (size + dest); - uint16_t *cmpOffset = (uint16_t *)((char *)src + BSWAP32(src[2])); - char *rawOffset = ((char *)src + BSWAP32(src[3])); - int counter = 0; - uint32_t controlBits; - - src += 4; - - while (dest != destEnd) { - if (counter == 0) { - controlBits = *src++; - controlBits = BSWAP32(controlBits); - counter = 32; - } - - if (controlBits & 0x80000000) { - *dest++ = *rawOffset++; - } - else { - uint16_t dcmpParam = *cmpOffset++; - dcmpParam = BSWAP16(dcmpParam); - int dcmpCount = (dcmpParam >> 12) + 3; - char* dcmpPtr = dest - (dcmpParam & 0x0FFF); - - while (dcmpCount) { - *dest++ = dcmpPtr[-1]; - dcmpCount--; - dcmpPtr++; - } - } - - counter--; - controlBits <<= 1; - } - return output; -} - -string readFile(const string& p, bool allowMissing = false) { - ifstream fin(p, ios::binary); - if (!fin) { - if (allowMissing) return ""; - cerr << "missing file " << p << endl; - exit(1); - } - fin.seekg(0, fin.end); - auto length = fin.tellg(); - fin.seekg(0, fin.beg); - string data(length, '\0'); - fin.read(data.data(), length); - assert(fin); - return data; -} - -pair getPngSize(const string& fname) { - string buffer(16, '\0'); - uint32_t w, h; - ifstream fin(fname, ios::binary); - fin.read(buffer.data(), 16); - fin.read((char*)&w, 4); - fin.read((char*)&h, 4); - assert(fin); - assert(buffer.substr(0, 4) == "\x89PNG"); - assert(buffer.substr(12, 4) == "IHDR"); - w = BSWAP32(w); - h = BSWAP32(h); - return {w, h}; -} - -string exec(const string& cmd) { - char buffer[128]; - string result; - FILE* pipe = popen(cmd.c_str(), "r"); - assert(pipe); - size_t s; - while ((s = fread(buffer, 1, sizeof(buffer), pipe))) { - result += string(buffer, buffer + s); - } - assert(!ferror(pipe)); - pclose(pipe); - return result; -} - -string compileAsset(const string& fname) { - auto ind = fname.rfind('.'); - if (ind == string::npos) return ""; - string q = fname.substr(ind + 1); - if (q == "png") { - string prev = fname.substr(0, ind); - - for (const string& lang : LANGS) { - string ret = readFile("build/" + lang + "/" + prev, true); - if (!ret.empty()) return ret; - } - - ind = prev.rfind('.'); - if (ind == string::npos) return ""; - q = prev.substr(ind + 1); - if (q == "rgba16" || q == "ia16" || q == "ia8" || q == "ia4" || q == "ia1") { - return exec("./tools/n64graphics -i /dev/stdout -g " + fname + " -f " + q); - } - } - if (q == "m64") - return readFile(fname); - if (q == "bin" && fname.find("assets") != string::npos) - return readFile(fname); - return ""; -} - -tuple> compileSoundData(const string& lang) { - string upper_lang = lang; - for (char& ch : upper_lang) ch = (char)(ch + 'A' - 'a'); - string build_dir = "build/" + lang; - string dir = build_dir + "/sound"; - string ctl = dir + "/sound_data.ctl"; - string tbl = dir + "/sound_data.tbl"; - exec("make " + tbl + " VERSION=" + lang + " NOEXTRACT=1"); - string sampleFilesStr = - exec("python3 tools/assemble_sound.py " + - dir + "/samples/ " - "sound/sound_banks/ " + - dir + "/sound_data.ctl " + - dir + "/sound_data.tbl " + - "-DVERSION_" + upper_lang + - " --print-samples"); - vector sampleFiles; - istringstream iss(sampleFilesStr); - string line; - while (getline(iss, line)) { - line = line.substr(build_dir.size() + 1); - line[line.size() - 1] = 'f'; - sampleFiles.push_back(line); - } - string ctlData = readFile(ctl); - string tblData = readFile(tbl); - return {ctlData, tblData, sampleFiles}; -} - -int main() { - intentional syntax error; // (see comment at top of file) - map assets; - map>> soundAssets; - - cout << "compiling assets..." << endl; - int totalAssets = 0; - for (string base_dir : {"assets", "sound/sequences", "textures", "levels", "actors"}) { - for (auto& ent: filesystem::recursive_directory_iterator(base_dir)) { - string p = ent.path().string(); - string bin = compileAsset(p); - if (bin.empty()) continue; - if (bin.size() < CHUNK_SIZE) { - cerr << "asset " << p << " is too small (" << bin.size() << " bytes), expected at least " << CHUNK_SIZE << " bytes" << endl; - continue; - } - assets[p] = bin; - totalAssets++; - } - } - for (const string& lang : LANGS) { - string ctl, tbl; - vector sampleFiles; - tie(ctl, tbl, sampleFiles) = compileSoundData(lang); - assets["@sound ctl " + lang] = ctl; - assets["@sound tbl " + lang] = tbl; - totalAssets += 2; - for (size_t i = 0; i < sampleFiles.size(); i++) { - soundAssets[sampleFiles[i]].emplace_back(lang, i); - } - } - cout << "compiled " << totalAssets << " assets" << endl; - - unordered_map>>> hashes; - for (const auto& asset : assets) { - size_t cutPos; - u64 hash; - tie(cutPos, hash) = hashString(asset.second); - hashes[hash].push_back(make_pair(cutPos, asset)); - } - - map, Pos> assetPositions; - for (const string& lang : LANGS) { - cout << "searching " << lang << "..." << endl; - auto remHashes = hashes; - auto search = [&](string& str, string lang, size_t mio0) { - rollingHashes(str, CHUNK_SIZE, [&](size_t hashPos, u64 hash) { - if (!remHashes.count(hash)) return; - vector>>& conts = remHashes.at(hash); - auto it = remove_if(conts.begin(), conts.end(), - [&](const pair>& pa) { - size_t cutPos = pa.first; - const string& name = pa.second.first; - const string& data = pa.second.second; - size_t assetPos = hashPos - cutPos; - if (stringMatches(str, assetPos, data)) { - assetPositions[make_pair(lang, name)] = {assetPos, mio0}; - return true; - } - return false; - }); - conts.erase(it, conts.end()); - if (conts.empty()) remHashes.erase(hash); - }); - }; - - string rom = readFile("baserom." + lang + ".z64"); - - for (size_t i = 0; i < rom.size(); i += 4) { - if (rom[i] == 'M' && rom[i+1] == 'I' && rom[i+2] == 'O' && rom[i+3] == '0') { - string data = mio0_decompress((uint32_t*)&rom[i]); - search(data, lang, i); - } - } - - search(rom, lang, 0); - } - - cout << "generating " << OUTPUT_FILE << "..." << endl; - ofstream fout(OUTPUT_FILE); - assert(fout); - fout << - "{\n" - "\"@comment\": \"This file was generated by tools/gen_asset_list.cpp. " - "When renaming a file, either change its name in this file directly, " - "or regenerate this file using that script.\""; - - bool first1 = true; - vector notFound; - for (const auto& asset : assets) { - const string& name = asset.first; - const string& data = asset.second; - vector> positions; - for (const string& lang : LANGS) { - auto it = assetPositions.find(make_pair(lang, name)); - if (it != assetPositions.end()) { - positions.push_back(make_pair(lang, it->second)); - } - } - - if (positions.empty()) { - notFound.push_back(name); - } - else { - fout << ",\n"; - if (first1) fout << "\n"; - first1 = false; - fout << "\"" << name << "\": ["; - if (name.substr(name.size() - 4) == ".png") { - int w, h; - tie(w, h) = getPngSize(name); - fout << w << "," << h << ","; - } - fout << data.size() << ",{"; - bool first2 = true; - for (auto& pa : positions) { - auto p = pa.second; - if (!first2) fout << ","; - first2 = false; - fout << "\"" << pa.first << "\":["; - if (p.mio0) - fout << p.mio0 << ","; - fout << p.pos << ']'; - } - fout << "}]"; - } - } - for (const auto& asset : soundAssets) { - const string& name = asset.first; - const vector>& locs = asset.second; - fout << ",\n"; - fout << "\"" << name << "\": [0,{"; - bool first2 = true; - for (auto& pa : locs) { - if (!first2) fout << ","; - first2 = false; - fout << "\"" << pa.first << "\":[\"@sound\"," << pa.second << ']'; - } - fout << "}]"; - } - fout << "\n}" << endl; - assert(fout); - fout.close(); - - if (!notFound.empty()) { - cout << endl; - cout << "Missing " << notFound.size() << " assets." << endl; - if (notFound.size() <= 10) { - for (auto& s : notFound) { - cout << s << endl; - } - } - return 1; - } - - cout << "done!" << endl; - - return 0; -} diff --git a/tools/patch_elf_32bit.c b/tools/patch_elf_32bit.c deleted file mode 100644 index 8e5295207..000000000 --- a/tools/patch_elf_32bit.c +++ /dev/null @@ -1,161 +0,0 @@ -#include -#include -#include -#include - -#define ARMAG "!\n" -#define SARMAG 8 - -/* from elf.h */ - -/* Type for a 16-bit quantity. */ -typedef uint16_t Elf32_Half; - -/* Types for signed and unsigned 32-bit quantities. */ -typedef uint32_t Elf32_Word; - -/* Type of addresses. */ -typedef uint32_t Elf32_Addr; - -/* Type of file offsets. */ -typedef uint32_t Elf32_Off; - -/* The ELF file header. This appears at the start of every ELF file. */ - -#define EI_NIDENT (16) - -typedef struct -{ - unsigned char e_ident[EI_NIDENT]; /* Magic number and other info */ - Elf32_Half e_type; /* Object file type */ - Elf32_Half e_machine; /* Architecture */ - Elf32_Word e_version; /* Object file version */ - Elf32_Addr e_entry; /* Entry point virtual address */ - Elf32_Off e_phoff; /* Program header table file offset */ - Elf32_Off e_shoff; /* Section header table file offset */ - Elf32_Word e_flags; /* Processor-specific flags */ - Elf32_Half e_ehsize; /* ELF header size in bytes */ - Elf32_Half e_phentsize; /* Program header table entry size */ - Elf32_Half e_phnum; /* Program header table entry count */ - Elf32_Half e_shentsize; /* Section header table entry size */ - Elf32_Half e_shnum; /* Section header table entry count */ - Elf32_Half e_shstrndx; /* Section header string table index */ -} Elf32_Ehdr; - -/* Conglomeration of the identification bytes, for easy testing as a word. */ -#define ELFMAG "\177ELF" -#define SELFMAG 4 - -#define EI_CLASS 4 /* File class byte index */ -#define ELFCLASS32 1 /* 32-bit objects */ - -#define EI_DATA 5 /* Data encoding byte index */ -#define ELFDATA2MSB 2 /* 2's complement, big endian */ - -/* end from elf.h */ - -// This file will find all mips3 object files in an ar archive and set the ABI flags to O32 -// this allows gcc to link them with the mips2 object files. -// Irix CC doesn't set the elf e_flags properly. - -// the AR file is structured as followed -//"!" followed by 0x0A (linefeed) 8 characters -// then a file header that follows the following structure -// everything is represented using space padded characters -// the last two characters are alwos 0x60 0x0A -// then come the file contents -// you can find the location of the next header by adding file_size_in_bytes (after parsing) -// all file headers start at an even offset so if the file size in bytes is odd you have to add 1 -// the first two "files" are special. One is a symbol table with a pointer to the header of the file -// contaning the symbol the other is an extended list of filenames -struct ar_header { - char identifier[16]; - char file_modification_timestamp[12]; - char owner_id[6]; - char group_id[6]; - char file_mode[8]; - char file_size_in_bytes[10]; - char ending[2]; -}; - -//These constants found by inspecting output of objdump -#define FLAGS_MIPS3 0x20 -#define FLAGS_O32ABI 0x100000 - -int fix_mips_elf(FILE *f, size_t filesize) -{ - Elf32_Ehdr hdr; - if (filesize < sizeof(hdr) || (1 != fread(&hdr, sizeof(hdr), 1, f))) { - printf("Failed to read ELF header\n"); - return -1; - } - - if (strncmp((const char *) hdr.e_ident, ELFMAG, SELFMAG) == 0) { - // found an ELF file. - if (hdr.e_ident[EI_CLASS] != ELFCLASS32 || hdr.e_ident[EI_DATA] != ELFDATA2MSB) { - printf("Expected 32bit big endian object files\n"); - return -1; - } - - if ((hdr.e_flags & 0xFF) == FLAGS_MIPS3 && (hdr.e_flags & FLAGS_O32ABI) == 0) { - hdr.e_flags |= FLAGS_O32ABI; - fseek(f, -(long)sizeof(hdr), SEEK_CUR); - if (1 != fwrite(&hdr, sizeof(hdr), 1, f)) { - printf("Failed to write back ELF header after patching.\n"); - return -1; - } - } - } - return 0; -} - -int fix_mips_ar(FILE *f) -{ - struct ar_header current_header; - fseek(f, 0x8, SEEK_SET); // skip header, this is safe enough given that we check to make sure the - // file header is valid - - while (1 == fread(¤t_header, sizeof(current_header), 1, f)) { - if (current_header.ending[0] != 0x60 && current_header.ending[1] != 0x0A) { - printf("Expected file header\n"); - return -1; - } - size_t filesize = atoi(current_header.file_size_in_bytes); - if (fix_mips_elf(f, filesize)) { - return -1; - } - if (filesize % 2 == 1) - filesize++; - fseek(f, filesize - sizeof(Elf32_Ehdr), SEEK_CUR); - } - return 0; -} - -int main(int argc, char **argv) { - FILE *f = fopen(argv[1], "r+b"); - uint8_t magic[8]; - int status = 0; - - if (f == NULL) { - printf("Failed to open file! %s\n", argv[1]); - return -1; - } - if (1 != fread(&magic, sizeof(magic), 1, f)) { - printf("Failed to read file magic\n"); - return -1; - } - rewind(f); - if (!memcmp(ARMAG, magic, SARMAG)) { - status = fix_mips_ar(f); - } else if (!memcmp(ELFMAG, magic, SELFMAG)) { - fseek(f, 0, SEEK_END); - size_t filesize = ftell(f); - rewind(f); - status = fix_mips_elf(f, filesize); - } else { - printf("Unknown file magic: %02x%02x%02X%02X\n", magic[0], magic[1], magic[2], magic[3]); - status = -1; - } - fclose(f); - return status; -} diff --git a/tools/patch_libmalloc.py b/tools/patch_libmalloc.py deleted file mode 100755 index 582504bf0..000000000 --- a/tools/patch_libmalloc.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python -# -# Patches the malloc() function in libmalloc.so to allocate more than the -# specified number of bytes. This is needed to work around issues with the -# compiler occasionally crashing. -# -# This script replaces the "move a1, a0" (00 80 28 25) instruction with -# "addiu a1, a0, n" (24 85 nn nn), which causes the malloc function to add n to -# the size parameter that was passed in. - -import hashlib -import os.path -import sys - -# file to patch -filename = 'tools/ido5.3_compiler/lib/libmalloc.so' -# Expected (unpatched) hash of file -filehash = 'adde672b5d79b52ca3cce9a47c7cb648' -# location in file to patch -address = 0xAB4 - -# Get parameter -if len(sys.argv) != 2: - print('Usage: ' + sys.argv[0] + ' n\n where n is the number of extra bytes to allocate in malloc()') - exit(1) -n = int(sys.argv[1]) - -# Original instruction "move a1, a0" -oldinsn = bytearray([0x00, 0x80, 0x28, 0x25]) - -# New instruction "addiu a1, a0, n" -newinsn = bytearray([0x24, 0x85, (n >> 8) & 0xFF, (n & 0xFF)]) - -# Patch the file -try: - with open(filename, 'rb+') as f: - # Read file contents - contents = bytearray(f.read()) - - # Unpatch the file by restoring original instruction - contents[address:address+4] = oldinsn - - # Verify the (unpatched) hash of the file - md5 = hashlib.md5() - md5.update(contents) - if md5.hexdigest() != filehash: - print('Error: ' + filename + ' does not appear to be the correct version.') - exit(1) - - # Patch the file - if n != 0: - contents[address:address+4] = newinsn - - # Write file - f.seek(0, os.SEEK_SET) - f.write(contents) -except IOError as e: - print('Error: Could not open library file for writing: ' + str(e)) diff --git a/tools/patch_libultra_math b/tools/patch_libultra_math deleted file mode 100755 index 5adb761d513fb3e6e6ca285b91ab5f701ec98d77..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 14472 zcmb<-^>JfjWMqH=W(GS35HCO&BH{p{7(5K23!)Oi&mw_3oPXfeXU|>L}WuWR{G{{XLArK9+55&fXIiME6Xa)g@Dv&-_ z5EIIWiNollP(xufOdO;SETsU_%)r2aMjJr=2cwbofx@Oj7oxAi7L}d>)wcjj!}Ni; zAblNBeH~DJF!}??K@1EGFdFJza2$Z#X#ovSbQ;as4CwkIp!(2h7l^YNU^K`MkWlc` zk`xdd-5wYpmwf?HeH)+(&}o=GFdAeB$iBd*B`KhA0kMg}uxJi~+J`G1IzR!=z`y{b zp~1_bpOcwnW}=^yqMMVMS6ZQ4VPU3gW};V|uV)0d9^@{N9iTMj?ib3y#K3R>BnAye zaD0d`Fo4rJNPb%1o2|jzmD`@L-J5W6;g`QQGvEIKsRx+>vJ<2RWH3lS)JY&N$Xo_K z1_p4Rg!sP^EDjO_F)pT_%LUQ6@C>9Ri>yYDfq_8?SpdrR$06>BLtGh$cnS{jEjZj` zf&tXU_PEE~bNK4MiFHU7hD=*3{No7bY zN=-~*h>uUMNQ_U*%uCG4tV(5wj|W)-GNq^_J~uHlkHOu?)5$sBNY7Z$1kOgVjPy(y z;ypur<5N`^8L5M&0YJP?Zo z%x3`eLFtegoEAap7NopHGLsWj1C%i^fQvAZ%uHTTs&9hwOQk^aObk6B#S9D#uyPPq z{uqEV2?GN|1GJojiGP6BKOE4C1SSsS->`=m2+9+n`~%@JFg!pK2g$+2Um%II!vsO) zD3UlYlK2lKaadjhNl$=^gUmqAr!$bmL3tRY7K9fdiG%VxNDPEmAc=#@5Re!MZ$J`< z`57d>14*0{Bml(+ki@y5Vj$`Sk~k=@fW$!f0+KinNMIC?hQMeDjE2By2#kinXb6nB z5cte5_sgUC4TncJ>rH(I29MSQB~1S>cr+j30JV%682+0^=`%2V`mY+T&%nSh&j3=N z0Scv0FCYH@|9=9Q4{CsYdU+Af2Q`d7y*vo#gBnDiUT%c*K@Fi#FBii3pa#&VmlNT9 zP{Zxh%SJdK)Y$s;vJlP(HH<#JOoZ`2{a4imyUv9H?7mO`Rk!MaT#MkZMdL3;d|# z1A|9ri%JIr1B2m#7fb*D|L@UyfWM`Mk%7VR(ms%nJvxtjbRKa? z|MzJA!N}kG57hFmQK?EO;clo=spjHu{lUP%unW|XH2mh%`ToV*fB*mcB>(W}JnY&0 z-k*oR9-ZLOoUqF)A@ZL$fg(b zK!PBbxA1_J_S){{Wng&0@$dit*E}A*wkaUqufPBQ@Awar?(P8xXNre04>){F!#o;~ zfC4`{)-lF0)-ld8{;)^q`xh_%{{L@y;PrWrURz#}2|ND&|NnZAN3X3ph_?#FTj$Xo z42rKSJPZu`LE&t8;Kj$k|Nl3i`2TwPi|v2^|L2!)0A&kMelUFN*?BR}qw|qR^OFw& zAwIpVZj1~J9=*KhL6O{R3$pLOXp#;C!gvX=X_M0{X!;6)F{{MfG^8f$;W2|qr85kI2 z5A(}2AnSiw{vXA^j^RGNw%VEu431%*o!1;gJv+ZThB$Wq3H9jJmCwZ3o2t? z^#A_<-{at4<`Q|s+fe`bbbj*beCN{n-0{E2eUIkDj2@PEOJ9M?AEZc1OVi_*Zvh1_ zIDdmmCXhcsW`b0L_+ZIVVl)IsLtr!nMnhmU1V%$(Gz3ONU^E0qLjZ3GFfuSP6lpWK zC1&QNrYMx;D}d&)6w)$tQWX_ci@6wFD+*GROHxz7LJAqFi7BZ?NUA~8R|>8^ZZH{8 z*AHwTT%WN~Qf7%lQf9hBYFoG7&Gh2h^hCxFD1@Hg=uK>+6efa->0calY!~g#~7#J8feE9$W0s{lX zf=~bdConKDO!)l&zXGV!_T~To3=qx0zyO_gVqmNaVqmNgV3g)z=a|3<5eKa~u=w=< zzdpz(F6`i`E)@m_22dCD!#jvw415A^d=g&#+~ph%4E9o1TE;5i=|YgXF$@e08Snr9 z2WteG0kyA$fq^06{r~?Hpkg4(hk*e+H+JLw|Nk#RZgb-k=x1``ljvi1;#25lapKeH zVRhs)Xk&BavuI}b;=91eT+7JCXW__a;K--p#HZlIC*i~=;Kaw_2%6^sh0Pt1{h$8- zPX$>3HqR4eo-3b1FN-V4JQt989w77F`5f}FnFXH9Q(DDzW}qoykn{}@0qqZgn4o?fh!zJC z3=9kgPyr~#PylrZ%-si|>R@Z&VD^L7vx1~x>VE%+_>YkRvefw>ly3kv??03e3-1q5 zJ|9&6CzKEK2gqz{V%U0abTbX0@$CSm1E6#QlrDhM4N!UllwJU(H$Z80a|!Wb>nh!y zovjo!LX%4KN=g+h^^EllbPdf*!Cb>SBLh7H6HSOP1A|_1254n#ViJR1W?o5ZQ2~Qq zUVcfcUV2`sUO`cQL26M+C6Z85X=YA}Ze|Kp$kEADwDP0a)K>IsD|VR=w~0;NS*KEbsg0%j6OEsPDKLCa-9c?PB*whkXgLzmlw zl!7rVpMW_G3=C=v3=GKn(f0v>#y?U0kW3A;oPmKM5Lza|^ux+67!4cu0r?RWb};|L%8y9sT78&)SUCuz zVdWmoN{|^aHi%ASU|;~1DKP!8as)=h$|sonVd`LX9s>gdsEmT~VdWHzhAx8!#}UYG znEOF?gK#Z$KMhPjtlWgruyPY-KTJQ&|LsuyP9VdO%3)Y}3@f)m`ayco^-o0EqW~I| zgBk=ak6`r^R1~~F0>%c>p!MM(e}T*g@fD!@U^Hla4%Rb%fYHcfb|3?w(s27%qWM2T5TuHM0Y-;H^8-Yb0j>|m2Q836 zwjWl{!)RFj4^xkxf3`yHhs8hoKBWyXjZhjEe^4&NUNrr<_T|9TquYNPP5%#QhKJD> zAkV=xGe9e4B>q(f2Jk)?STMo%3BdO4pt~2whtUri7#R4Wc^G6YY(GH*bRQK=9Y_v@ zVSE_V*3q620Rgohb^f1qIsvtIzJUjV8fW)Dn1T68dgwg7^}pvqy&7-0If;35zP aNG*tkOvCI3iNmlJ)IctnLZ}29mjM9Y)Aq6e diff --git a/tools/rasm2armips.py b/tools/rasm2armips.py deleted file mode 100755 index ae92d5862..000000000 --- a/tools/rasm2armips.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env python - -import argparse -import re -import sys - -def read_file(filepath): - with open(filepath) as f: - lines = f.readlines() - split_lines = [re.split(r'[ ,]+', l.strip().replace('$', '')) for l in lines] - return split_lines - -# jumps and branches with named targets -jumps = ['jal', 'j'] -branches = ['beq', 'bgez', 'bgtz', 'blez', 'bltz', 'bne'] -jump_branches = jumps + branches -# jumps and branches with delay slots -has_delay_slot = jump_branches + ['jr'] - -def decode_references(instructions): - refs = [] - for ins in instructions: - if ins[3] in jump_branches: - target = int(ins[-1], 0) - if target not in refs: - refs.append(target) - return refs - -def reassemble(args, instructions, refs): - print('.rsp') - print('\n.create DATA_FILE, 0x%04X' % 0x0000) - print('\n.close // DATA_FILE\n') - print('.create CODE_FILE, 0x%08X\n' % args.base) - delay_slot = False - for ins in instructions: - addr = int(ins[0], 0) - if (addr & 0xFFFF) in refs: - print('%s_%08x:' % (args.name, addr)) - sys.stdout.write(' ' * args.indent) - if delay_slot: - sys.stdout.write(' ') - delay_slot = False - if ins[3] in jumps: - target = int(ins[-1], 0) | (args.base & 0xFFFF0000) - ins[-1] = '%s_%08x' % (args.name, target) - elif ins[3] in branches: - if ins[3][-1] =='z' and ins[5] == 'zero': - del ins[5] # remove 'zero' operand from branch - target = (int(ins[-1], 0) & 0x1FFF) + (args.base & 0xFFFF0000) - ins[-1] = '%s_%08x' % (args.name, target) - elif ins[3] == 'vsar': # fixup last operand of vsar - reg_map = {'ACC_H': 0, 'ACC_M': 1, 'ACC_L': 2} - reg = ins[4].split(r'[')[0] - num = reg_map[ins[-1]] - ins[-1] = '%s[%d]' % (reg, num) - if ins[3] in has_delay_slot: - delay_slot = True - if len(ins) > 4: # with args - print('%-5s %s' % (ins[3], ', '.join(ins[4:]))) - else: - print('%s' % ins[3]) - print('\n.close // CODE_FILE') - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('input_file', help="input assembly file generated from `rasm2 -D -e -a rsp -B -o 0x04001000 -f`") - parser.add_argument('-b', type=int, help="base address of file", dest='base', default=0x04001000) - parser.add_argument('-i', type=int, help="amount of indentation", dest='indent', default=4) - parser.add_argument('-n', help="name to prefex labels with", dest='name', default='f3d') - args = parser.parse_args() - - lines = read_file(args.input_file) - refs = decode_references(lines) - reassemble(args, lines, refs) - -main()