From caf53d3d53d3d07f2f3b291f128551f773d774b1 Mon Sep 17 00:00:00 2001 From: Ryan Myers Date: Sun, 19 Sep 2021 12:33:24 -0400 Subject: [PATCH] Add some tools I've found helpful --- first_diff.sh | 1 + tools/permuter_settings.toml | 7 + tools/python/first_diff.py | 255 +++++++++++++++++++++++++++++++++++ tools/python/m2ctx.py | 72 ++++++++++ 4 files changed, 335 insertions(+) create mode 100755 first_diff.sh create mode 100644 tools/permuter_settings.toml create mode 100644 tools/python/first_diff.py create mode 100644 tools/python/m2ctx.py diff --git a/first_diff.sh b/first_diff.sh new file mode 100755 index 00000000..dd51b5ae --- /dev/null +++ b/first_diff.sh @@ -0,0 +1 @@ +python3 tools/python/first_diff.py \ No newline at end of file diff --git a/tools/permuter_settings.toml b/tools/permuter_settings.toml new file mode 100644 index 00000000..860e1502 --- /dev/null +++ b/tools/permuter_settings.toml @@ -0,0 +1,7 @@ +[preserve_macros] +"g[DS]P.*" = "void" +"gDma.*" = "void" +"_SHIFTL" = "unsigned int" + +[decompme.compilers] +"tools/ido5.3_recomp/cc" = "ido5.3" diff --git a/tools/python/first_diff.py b/tools/python/first_diff.py new file mode 100644 index 00000000..cbe59cf4 --- /dev/null +++ b/tools/python/first_diff.py @@ -0,0 +1,255 @@ +#!/usr/bin/env python3 + +import os.path +import argparse +from subprocess import check_call + +parser = argparse.ArgumentParser( + description="Find the first difference(s) between the built ROM and the base ROM." +) +parser.add_argument( + "-c", + "--count", + type=int, + default=5, + help="find up to this many instruction difference(s)", +) +parser.add_argument( + "-d", + "--diff", + dest="diff_args", + nargs="?", + action="store", + default=False, + const="prompt", + help="run diff.py on the result with the provided arguments" +) +parser.add_argument( + "-m", "--make", help="run ninja before finding difference(s)", action="store_true" +) +args = parser.parse_args() + +diff_count = args.count + +if args.make: + check_call(["ninja", "baseroms/dkr.z64"]) + +baseimg = f"baseroms/dkr.z64" +basemap = f"build/dkr.map" + +myimg = f"build/us_1.0/dkr.z64" +mymap = f"build/us_1.0/dkr.map" + +if not os.path.isfile(baseimg): + print(f"{baseimg} must exist.") + exit(1) +if not os.path.isfile(myimg) or not os.path.isfile(mymap): + print(f"{myimg} and {mymap} must exist.") + exit(1) + +mybin = open(myimg, "rb").read() +basebin = open(baseimg, "rb").read() + +if len(mybin) != len(basebin): + print("Modified ROM has different size...") + exit(1) + +if mybin == basebin: + print("No differences!") + exit(0) + + +def search_rom_address(target_addr): + ram_offset = None + prev_ram = 0 + prev_rom = 0 + prev_sym = "" + cur_file = "" + prev_file = cur_file + prev_line = "" + with open(mymap) as f: + for line in f: + if "load address" in line: + # Ignore .bss sections since we're looking for a ROM address + if ".bss" in line or ".bss" in prev_line: + ram_offset = None + continue + ram = int(line[16 : 16 + 18], 0) + rom = int(line[59 : 59 + 18], 0) + ram_offset = ram - rom + continue + + prev_line = line + + if ( + ram_offset is None + or "=" in line + or "*fill*" in line + or " 0x" not in line + ): + continue + + ram = int(line[16 : 16 + 18], 0) + rom = ram - ram_offset + sym = line.split()[-1] + + if "0x" in sym: + ram_offset = None + continue + if "/" in sym: + cur_file = sym + continue + + if rom > target_addr: + return f"{prev_sym} (RAM 0x{prev_ram:X}, ROM 0x{prev_rom:X}, {prev_file})" + + prev_ram = ram + prev_rom = rom + prev_sym = sym + prev_file = cur_file + + return "at end of rom?" + + +def parse_map(map_fname): + ram_offset = None + cur_file = "" + syms = {} + prev_sym = None + prev_line = "" + with open(map_fname) as f: + for line in f: + if "load address" in line: + ram = int(line[16 : 16 + 18], 0) + rom = int(line[59 : 59 + 18], 0) + ram_offset = ram - rom + continue + + prev_line = line + + if ( + ram_offset is None + or "=" in line + or "*fill*" in line + or " 0x" not in line + ): + continue + + ram = int(line[16 : 16 + 18], 0) + rom = ram - ram_offset + sym = line.split()[-1] + + if "0x" in sym: + ram_offset = None + continue + elif "/" in sym: + cur_file = sym + continue + + syms[sym] = (rom, cur_file, prev_sym, ram) + prev_sym = sym + + return syms + + +def map_diff(): + map1 = parse_map(mymap) + map2 = parse_map(basemap) + min_ram = None + found = None + for sym, addr in map1.items(): + if sym not in map2: + continue + if addr[0] != map2[sym][0]: + if min_ram is None or addr[0] < min_ram: + min_ram = addr[0] + found = (sym, addr[1], addr[2]) + if min_ram is None: + return False + else: + print( + f"Map appears to have shifted just before {found[0]} ({found[1]}) -- in {found[2]}?" + ) + if found[2] is not None and found[2] not in map2: + print( + f"(Base map file {basemap} out of date due to new or renamed symbols, so result may be imprecise.)" + ) + return True + + +def hexbytes(bs): + return ":".join("{:02X}".format(c) for c in bs) + + +found_instr_diff = [] +map_search_diff = [] +diffs = 0 +shift_cap = 1000 +for i in range(24, len(mybin), 4): + # (mybin[i:i+4] != basebin[i:i+4], but that's slightly slower in CPython...) + if diffs <= shift_cap and ( + mybin[i] != basebin[i] + or mybin[i + 1] != basebin[i + 1] + or mybin[i + 2] != basebin[i + 2] + or mybin[i + 3] != basebin[i + 3] + ): + if diffs == 0: + print(f"First difference at ROM addr 0x{i:X}, {search_rom_address(i)}") + print( + f"Bytes: {hexbytes(mybin[i : i + 4])} vs {hexbytes(basebin[i : i + 4])}" + ) + diffs += 1 + if ( + len(found_instr_diff) < diff_count + and mybin[i] >> 2 != basebin[i] >> 2 + and not search_rom_address(i) in map_search_diff + ): + found_instr_diff.append(i) + map_search_diff.append(search_rom_address(i)) + +if diffs == 0: + print("No differences but ROMs differ?") + exit() + +if len(found_instr_diff) > 0: + for i in found_instr_diff: + print(f"Instruction difference at ROM addr 0x{i:X}, {search_rom_address(i)}") + print( + f"Bytes: {hexbytes(mybin[i : i + 4])} vs {hexbytes(basebin[i : i + 4])}" + ) +print() + +definite_shift = diffs > shift_cap +if definite_shift: + print(f"Over {shift_cap} differing words, must be a shifted ROM.") +else: + print(f"{diffs} differing word(s).") + +if diffs > 100: + if not os.path.isfile(basemap): + print( + f"To find ROM shifts, copy a clean .map file to {basemap} and rerun this script." + ) + elif not map_diff(): + print(f"No ROM shift{' (!?)' if definite_shift else ''}") + +if args.diff_args: + if len(found_instr_diff) < 1: + print(f"No instruction difference to run diff.py on") + exit() + + diff_sym = search_rom_address(found_instr_diff[0]).split()[0] + if args.diff_args == "prompt": + diff_args = input("Call diff.py with which arguments? ") or "--" + else: + diff_args = args.diff_args + if diff_args[0] != "-": + diff_args = "-" + diff_args + check_call( + [ + "python3", + "diff.py", + diff_args, + diff_sym, + ] + ) diff --git a/tools/python/m2ctx.py b/tools/python/m2ctx.py new file mode 100644 index 00000000..025bd073 --- /dev/null +++ b/tools/python/m2ctx.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 + +import argparse, os, subprocess, sys +from pathlib import Path + +script_dir = os.path.dirname(os.path.realpath(__file__)) +root_dir = script_dir + "/../../" +src_dir = root_dir + "src/" +lib_dir = root_dir + "lib/" + + +def get_c_dir(dirname): + for root, dirs, files in os.walk(src_dir): + for directory in dirs: + if directory == dirname: + return os.path.join(root, directory) + for root, dirs, files in os.walk(lib_dir): + for directory in dirs: + if directory == dirname: + return os.path.join(root, directory) + + +def get_c_file(directory): + for root, dirs, files in os.walk(directory): + for file in files: + if file.endswith(".c") and "data" not in file: + return file + + +def import_c_file(in_file): + in_file = os.path.relpath(in_file, root_dir) + cpp_command = ["gcc", "-E", "-P", "-Iinclude", "-Iassets", "-Isrc", "-undef", "-D__sgi", "-D_LANGUAGE_C", + "-DNON_MATCHING", "-D_Static_assert(x, y)=", "-D__attribute__(x)=", in_file] + try: + return subprocess.check_output(cpp_command, cwd=root_dir, encoding="utf-8") + except subprocess.CalledProcessError: + print( + "Failed to preprocess input file, when running command:\n" + + cpp_command, + file=sys.stderr, + ) + sys.exit(1) + + +def main(): + parser = argparse.ArgumentParser(usage="./m2ctx.py path/to/file.c", + description="Creates a ctx.c file for mips2c. " + "Output will be saved as oot/ctx.c") + parser.add_argument('filepath', help="path of c file to be processed") + args = parser.parse_args() + + if args.filepath: + c_file_path = args.filepath + print("Using file: {}".format(c_file_path)) + else: + this_dir = Path.cwd() + c_dir_path = get_c_dir(this_dir.name) + if c_dir_path is None: + sys.exit( + "Cannot find appropriate c file dir. In argumentless mode, run this script from the c file's corresponding asm dir.") + c_file = get_c_file(c_dir_path) + c_file_path = os.path.join(c_dir_path, c_file) + print("Using file: {}".format(c_file_path)) + + output = import_c_file(c_file_path) + + with open(os.path.join(root_dir, "ctx.c"), "w", encoding="UTF-8") as f: + f.write(output) + + +if __name__ == "__main__": + main()