From 679f0086d9b4b41563027d09ea55056fc8a93780 Mon Sep 17 00:00:00 2001 From: Yanis42 <35189056+Yanis42@users.noreply.github.com> Date: Sun, 4 Sep 2022 23:17:10 +0200 Subject: [PATCH 1/4] merged daf --- Makefile | 10 ++- tools/daf/README.md | 14 +++ tools/daf/daf.py | 61 +++++++++++++ tools/daf/data.py | 61 +++++++++++++ tools/daf/functions.py | 189 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 331 insertions(+), 4 deletions(-) create mode 100644 tools/daf/README.md create mode 100644 tools/daf/daf.py create mode 100644 tools/daf/data.py create mode 100644 tools/daf/functions.py diff --git a/Makefile b/Makefile index d5202f6e6..616c3ea3a 100644 --- a/Makefile +++ b/Makefile @@ -84,7 +84,7 @@ endif # Detect compiler and set variables appropriately. ifeq ($(COMPILER),gcc) CC := $(MIPS_BINUTILS_PREFIX)gcc -else +else $(error Unsupported compiler. Please use either gcc as the COMPILER variable.) endif @@ -118,7 +118,7 @@ ASFLAGS := -march=vr4300 -32 -no-pad-sections -Iinclude ifeq ($(COMPILER),gcc) CFLAGS += -G 0 -nostdinc $(INC) -march=vr4300 -mfix4300 -mabi=32 -mno-abicalls -mdivide-breaks -fno-zero-initialized-in-bss -fno-toplevel-reorder -ffreestanding -fno-common -fno-merge-constants -mno-explicit-relocs -mno-split-addresses $(CHECK_WARNINGS) -funsigned-char MIPS_VERSION := -mips3 -else +else # we support Microsoft extensions such as anonymous structs, which the compiler does support but warns for their usage. Surpress the warnings with -woff. CFLAGS += -G 0 -non_shared -fullwarn -verbose -Xcpluscomm $(INC) -Wab,-r4300_mul -woff 516,649,838,712 MIPS_VERSION := -mips2 @@ -194,12 +194,14 @@ setup: python3 fixbaserom.py python3 extract_baserom.py python3 extract_assets.py -j$(N_THREADS) + python3 tools/daf/daf.py -a -p ./ +# ``./`` ok for relative path? test: $(ROM) $(EMULATOR) $(EMU_FLAGS) $< -.PHONY: all clean setup test distclean assetclean +.PHONY: all clean setup test distclean assetclean daf #### Various Recipes #### @@ -209,7 +211,7 @@ $(ROM): $(ELF) $(ELF): $(TEXTURE_FILES_OUT) $(ASSET_FILES_OUT) $(O_FILES) $(OVL_RELOC_FILES) build/ldscript.txt build/undefined_syms.txt $(LD) -T build/undefined_syms.txt -T build/ldscript.txt --no-check-sections --accept-unknown-input-arch --emit-relocs -Map build/z64.map -o $@ -## Order-only prerequisites +## Order-only prerequisites # These ensure e.g. the O_FILES are built before the OVL_RELOC_FILES. # The intermediate phony targets avoid quadratically-many dependencies between the targets and prerequisites. diff --git a/tools/daf/README.md b/tools/daf/README.md new file mode 100644 index 000000000..57eb41b42 --- /dev/null +++ b/tools/daf/README.md @@ -0,0 +1,14 @@ +# Decompilation Assets Fixer (DAF) +The tool used for extract the assets ([ZAPD](https://github.com/zeldaret/ZAPD)) needs to be updated because it's using old names for structs and macros, this project is a workaround attempt until the main tool is updated. + +## Usage +- Change the decomp path in ``data.py`` +- Run ``daf.py`` (tested under Python 3.10, should work with 3.9+) +- If you have any issues when compiling the decomp, try ``make clean && make -j``, if it's not working feel free to open an issue on this repo + +### Operating modes: +- ``daf.py -m (--mode) fix_types``, this will update types and macros, intended to be used after using ZAPD +- ``daf.py -m (--mode) name_entrances``, this will remove hex numbers from exit lists +- ``daf.py -m (--mode) fix_segments``, this will add casts to segment symbols inside room lists +- ``daf.py -a (--all)``, this will run all modes +- ``daf.py -v (--verbose)``, this will display extra informations diff --git a/tools/daf/daf.py b/tools/daf/daf.py new file mode 100644 index 000000000..cc7907dfe --- /dev/null +++ b/tools/daf/daf.py @@ -0,0 +1,61 @@ +from time import time + +try: + from data import fileTypes + from functions import replaceOldData, replaceEntranceHex, fixSegments, getArguments +except: + print("[DAF:Error]: Files are missing. Make sure everything is in the same folder.") + quit() + +# verbose settings +fixTypeTime = entranceTime = segmentsTime = None +dashAmount = 60 +infoPrefix = "[DAF:Info]" + +# general things +args, parser = getArguments() + +hasArgs = False +runTime = time() + +if args.mode == "fix_types" or args.run_all: + startingTime = time() + + if args.verbose: + print(f"{infoPrefix}: Fixing types and macros...") + + hasArgs = True + for type in fileTypes: + replaceOldData(f"{args.decompPath}/assets/", type) + + if args.verbose: + print(f"{infoPrefix}: Done in {(time() - startingTime):.2f}s!\n{'-' * dashAmount}") + +if args.mode == "name_entrances" or args.run_all: + startingTime = time() + + if args.verbose: + print(f"{infoPrefix}: Removing hexadecimal from exit lists...") + hasArgs = True + replaceEntranceHex(args.decompPath) + + if args.verbose: + print(f"{infoPrefix}: Done in {(time() - startingTime):.2f}s!\n{'-' * dashAmount}") + +if args.mode == "fix_segments" or args.run_all: + startingTime = time() + + if args.verbose: + print(f"{infoPrefix}: Adding missing casts to rooms symbols...") + hasArgs = True + fixSegments(args.decompPath) + + if args.verbose: + print(f"{infoPrefix}: Done in {(time() - startingTime):.2f}s!\n{'-' * dashAmount}") + +if hasArgs and args.verbose: + print(f"{infoPrefix}: All Done in {(time() - runTime):.2f}s!") + +if not hasArgs: + parser.print_help() + quit() diff --git a/tools/daf/data.py b/tools/daf/data.py new file mode 100644 index 000000000..d8f5fed90 --- /dev/null +++ b/tools/daf/data.py @@ -0,0 +1,61 @@ +### [GENERAL DATA] ### + +# Add types here +fileTypes = [".h", ".c"] + +# ------------------------------------------------------- + +### [FIX TYPES MODE] ### + +# Format: ``"OLD": "NEW"`` + +camData = { + "CamData": "BgCamInfo", +} + +curveData = { + "TransformData": "CurveInterpKnot", + "TransformUpdateIndex": "CurveAnimationHeader", + "SkelCurveLimbList": "CurveSkeletonHeader", +} + +roomData = { + "PolygonDlist": "RoomShapeDListsEntry", + "PolygonType0": "RoomShapeNormal", + "MeshHeader1Single": "RoomShapeImageSingle", + "BgImage": "RoomShapeImageMultiBgEntry", + "MeshHeader1Multi": "RoomShapeImageMulti", + "PolygonDlist2": "RoomShapeCullableEntry", + "PolygonType2": "RoomShapeCullable", + "SCENE_CMD_MESH": "SCENE_CMD_ROOM_SHAPE", +} + +skinData = { + "Struct_800A57C0": "SkinVertex", + "Struct_800A598C_2": "SkinTransformation", + "Struct_800A5E28": "SkinAnimatedLimbData", + "Struct_800A598C": "SkinLimbModif", +} + +# Add or remove dictionnaries to this list to replace types +dataToFix = [ + camData, + curveData, + roomData, + skinData, +] + +# ------------------------------------------------------- + +### [NAME ENTRANCES] ### + +# Dictionnary containing special entrance values +entrDictSpecial = { + "0x7FF9": "ENTR_RETURN_YOUSEI_IZUMI_YOKO", # Great Fairy Fountain (spells) + "0x7FFA": "ENTR_RETURN_SYATEKIJYOU", # Shooting gallery + "0x7FFB": "ENTR_RETURN_2", # Unused + "0x7FFC": "ENTR_RETURN_SHOP1", # Bazaar + "0x7FFD": "ENTR_RETURN_4", # Unused + "0x7FFE": "ENTR_RETURN_DAIYOUSEI_IZUMI", # Great Fairy Fountain (magic, double magic, double defense) + "0x7FFF": "ENTR_RETURN_GROTTO", # Grottos and normal Fairy Fountain +} diff --git a/tools/daf/functions.py b/tools/daf/functions.py new file mode 100644 index 000000000..570f2f2c9 --- /dev/null +++ b/tools/daf/functions.py @@ -0,0 +1,189 @@ +from os import walk +from re import sub +from argparse import ArgumentParser as Parser + +try: + from data import dataToFix, entrDictSpecial +except: + print("[DAF:Error]: ``data.py`` not found! Make sure everything is in the same folder.") + quit() + +# ------------------------------------------------------- + +### [GENERAL FUNCTIONS] ### + + +def getArguments(): + """Initialisation of the argument parser""" + + parser = Parser(description="Fix various things related to assets for the OoT Decomp") + + parser.add_argument( + "-m", + "--mode", + dest="mode", + type=str, + default="", + help="available modes: `fix_types`, `name_entrances`, `fix_segments`", + ) + + parser.add_argument( + "-a", + "--all", + dest="run_all", + default=False, + action="store_true", + help="run every mode", + ) + + parser.add_argument( + "-v", + "--verbose", + dest="verbose", + default=False, + action="store_true", + help="show extra informations", + ) + + parser.add_argument( + "-p", + "--path", + dest="decompPath", + default="", + required=True, + help="set decomp root", + ) + + return parser.parse_args(), parser + + +def getPaths(path: str, fileType: str): + """Returns a list of data paths with the specified extension""" + paths = [] + for path, dirs, files in walk(path): + for file in files: + if file.endswith(fileType): + paths.append(f"{path}/{file}") + paths.sort() + return paths + + +def getArrayInfos(data: list, arrayName: str): + """Returns arrays containing line numbers for the start and the end of relevant C data""" + arrayStartIndices = [] + arrayEndIndices = [] + hasListStarted = False + + for lineNb, line in enumerate(data): + if arrayName in line and line.endswith("] = {\n"): + arrayStartIndices.append(lineNb + 1) + hasListStarted = True + + if hasListStarted and line.startswith("};\n"): + arrayEndIndices.append(lineNb) + hasListStarted = False + try: + if len(arrayStartIndices) != len(arrayEndIndices): + raise IndexError + except IndexError: + print("[DAF:Error]: Start Length != End Length") + + return arrayStartIndices, arrayEndIndices + + +# ------------------------------------------------------- + +### [FIX TYPES MODE] ### + + +def replaceOldData(path: str, extension: str): + """Replaces older names by newer ones""" + paths = getPaths(path, extension) + for path in paths: + with open(path, "r") as curFile: + fileData = curFile.read() + for data in dataToFix: + for key in data.keys(): + fileData = sub(rf"{key}\b", data[key], fileData) + fileData = sub(rf"{key}_\B", f"{data[key]}_", fileData) + with open(path, "w") as curFile: + curFile.write(fileData) + + +# ------------------------------------------------------- + +### [NAME ENTRANCES] ### + + +def getEntranceDict(path: str): + """Returns a list containing every entrances""" + entranceList = [] + + # read the entrance table + try: + with open(f"{path}/include/tables/entrance_table.h", "r") as fileData: + # keep the relevant data + for line in fileData.readlines(): + if line.startswith("/* 0x"): + startIndex = line.find("ENTR_") + entranceList.append(line[startIndex : line.find(",", startIndex)]) + except FileNotFoundError: + raise print("[DAF:Error]: Can't find entrance_table.h!") + + # return a dictionnary from the entrance list + return {f"0x{i:04X}": entrance for i, entrance in enumerate(entranceList)} | entrDictSpecial + + +def getNewFileData(data: list, dataDict: dict, arrayName: str): + """Returns the current data with the updated values""" + startList, endList = getArrayInfos(data, arrayName) + if len(startList) == len(endList): + for curStart, curEnd in zip(startList, endList): + for i in range(curEnd - curStart): + curLine = curStart + i + try: + data[curLine] = (" " * 4) + f"{dataDict[data[curLine][:-2].lstrip()]},\n" + except KeyError: + # why??? + pass + return data + + +def replaceEntranceHex(decompRoot: str): + """Updates the entrances from OoT scenes""" + entrDict = getEntranceDict(decompRoot) + scenePaths = getPaths(f"{decompRoot}/assets/scenes/", ".c") + + for path in scenePaths: + data = [] + sceneName = None + with open(path, "r") as file: + if file.name.find("room") == -1: + data = file.readlines() + sceneName = file.name.split("/")[6][:-2] + if sceneName is not None: + newData = getNewFileData(data, entrDict, f"{sceneName}ExitList") + with open(path, "w") as file: + for line in newData: + file.write(line) + + +# ------------------------------------------------------- + +### [FIX SEGMENTS] ### + + +def fixSegments(decompRoot): + """Adds u32 casts to room's segment symbols""" + paths = getPaths(f"{decompRoot}/assets/scenes", ".c") + for path in paths: + with open(path, "r") as curFile: + fileData = curFile.read() + + fileData = sub( + r"\w*SegmentRom\w*|\(u32\)\w*SegmentRom\w*", + lambda match: f"(u32){match.group(0)}" if not match.group(0).startswith("(u32)") else match.group(0), + fileData, + ) + with open(path, "w") as curFile: + curFile.write(fileData) From 7ccc189f18bfaee86121422fc9c532b2d2c91397 Mon Sep 17 00:00:00 2001 From: Yanis42 <35189056+Yanis42@users.noreply.github.com> Date: Sun, 4 Sep 2022 23:37:04 +0200 Subject: [PATCH 2/4] removed zapd compatibility typedefs/defines --- include/z64bgcheck.h | 2 -- include/z64curve.h | 6 ------ include/z64scene.h | 10 ---------- include/z64skin.h | 7 ------- 4 files changed, 25 deletions(-) diff --git a/include/z64bgcheck.h b/include/z64bgcheck.h index 8b5236406..78bf74e03 100644 --- a/include/z64bgcheck.h +++ b/include/z64bgcheck.h @@ -55,8 +55,6 @@ typedef struct { /* 0x4 */ Vec3s* bgCamFuncData; // s16 data grouped in threes (ex. Vec3s), is usually of type `BgCamFuncData`, but can be a list of points of type `Vec3s` for crawlspaces } BgCamInfo; // size = 0x8 -typedef BgCamInfo CamData; // Todo: Zapd compatibility - // The structure used for all instances of s16 data from `BgCamInfo` with the exception of crawlspaces. // See `Camera_Subj4` for Vec3s data usage in crawlspaces typedef struct { diff --git a/include/z64curve.h b/include/z64curve.h index a526effcb..9ff0a4d9a 100644 --- a/include/z64curve.h +++ b/include/z64curve.h @@ -59,10 +59,4 @@ s32 SkelCurve_Update(struct PlayState* play, SkelCurve* skelCurve); void SkelCurve_Draw(Actor* actor, struct PlayState* play, SkelCurve* skelCurve, OverrideCurveLimbDraw overrideLimbDraw, PostCurveLimbDraw postLimbDraw, s32 lod, void* data); -// ZAPD compatibility typedefs -// TODO: Remove when ZAPD adds support for them -typedef CurveInterpKnot TransformData; -typedef CurveAnimationHeader TransformUpdateIndex; -typedef CurveSkeletonHeader SkelCurveLimbList; - #endif diff --git a/include/z64scene.h b/include/z64scene.h index ede0d5249..57e5c7773 100644 --- a/include/z64scene.h +++ b/include/z64scene.h @@ -154,16 +154,6 @@ typedef union { RoomShapeCullable cullable; } RoomShape; // "Ground Shape" -// TODO update ZAPD -typedef RoomShapeDListsEntry PolygonDlist; -typedef RoomShapeNormal PolygonType0; -typedef RoomShapeImageSingle MeshHeader1Single; -typedef RoomShapeImageMultiBgEntry BgImage; -typedef RoomShapeImageMulti MeshHeader1Multi; -typedef RoomShapeCullableEntry PolygonDlist2; -typedef RoomShapeCullable PolygonType2; -#define SCENE_CMD_MESH SCENE_CMD_ROOM_SHAPE - #define ROOM_DRAW_OPA (1 << 0) #define ROOM_DRAW_XLU (1 << 1) diff --git a/include/z64skin.h b/include/z64skin.h index e9d8689d8..a199ed41b 100644 --- a/include/z64skin.h +++ b/include/z64skin.h @@ -43,13 +43,6 @@ typedef struct { /* 0x08 */ Gfx* dlist; } SkinAnimatedLimbData; // size = 0xC -// ZAPD compatibility typedefs -// TODO: Remove when ZAPD adds support for them -typedef SkinVertex Struct_800A57C0; -typedef SkinTransformation Struct_800A598C_2; -typedef SkinAnimatedLimbData Struct_800A5E28; -typedef SkinLimbModif Struct_800A598C; - #define SKIN_LIMB_TYPE_ANIMATED 4 #define SKIN_LIMB_TYPE_NORMAL 11 From a607424e5fe277338a87853ada055c3d76db44d1 Mon Sep 17 00:00:00 2001 From: Yanis42 <35189056+Yanis42@users.noreply.github.com> Date: Sun, 18 Sep 2022 21:20:47 +0200 Subject: [PATCH 3/4] python 3.7 compatibility --- tools/daf/README.md | 18 ++++++++++++++++-- tools/daf/functions.py | 3 ++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/tools/daf/README.md b/tools/daf/README.md index 57eb41b42..69d20e7da 100644 --- a/tools/daf/README.md +++ b/tools/daf/README.md @@ -2,8 +2,8 @@ The tool used for extract the assets ([ZAPD](https://github.com/zeldaret/ZAPD)) needs to be updated because it's using old names for structs and macros, this project is a workaround attempt until the main tool is updated. ## Usage -- Change the decomp path in ``data.py`` -- Run ``daf.py`` (tested under Python 3.10, should work with 3.9+) +- Set the decomp path with the ``-p (--path)`` argument (example: ``daf.py -p ./ (-a || -m MODE)``) +- Run ``daf.py`` (tested under Python 3.10, should work with 3.7+) - If you have any issues when compiling the decomp, try ``make clean && make -j``, if it's not working feel free to open an issue on this repo ### Operating modes: @@ -12,3 +12,17 @@ The tool used for extract the assets ([ZAPD](https://github.com/zeldaret/ZAPD)) - ``daf.py -m (--mode) fix_segments``, this will add casts to segment symbols inside room lists - ``daf.py -a (--all)``, this will run all modes - ``daf.py -v (--verbose)``, this will display extra informations + +### ROM should build OK on an unmodifed codebase + +``` +f0b7f35375f9cc8ca1b2d59d78e35405 zelda_ocarina_mq_dbg.z64 +zelda_ocarina_mq_dbg.z64: OK +``` + +## Add More Data +If you need to add more data to change, add a dictionnary with the following format: ``"OLD": "NEW"``, then add your dictionnary to ``dataToFix``. + +Next, run ``daf.py`` and it should make the changes. + +## Contributions are welcome! diff --git a/tools/daf/functions.py b/tools/daf/functions.py index 570f2f2c9..a4babfbfe 100644 --- a/tools/daf/functions.py +++ b/tools/daf/functions.py @@ -131,7 +131,8 @@ def getEntranceDict(path: str): raise print("[DAF:Error]: Can't find entrance_table.h!") # return a dictionnary from the entrance list - return {f"0x{i:04X}": entrance for i, entrance in enumerate(entranceList)} | entrDictSpecial + entranceDict = {f"0x{i:04X}": entrance for i, entrance in enumerate(entranceList)} + return dict(entranceDict, **entrDictSpecial) def getNewFileData(data: list, dataDict: dict, arrayName: str): From 58880bb5377d76669219ef846f919927bf83667e Mon Sep 17 00:00:00 2001 From: Yanis42 <35189056+Yanis42@users.noreply.github.com> Date: Sat, 3 Dec 2022 20:43:56 +0100 Subject: [PATCH 4/4] remove useless comment --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index 121daa652..eda6d1e1d 100644 --- a/Makefile +++ b/Makefile @@ -195,7 +195,6 @@ setup: python3 extract_baserom.py python3 extract_assets.py -j$(N_THREADS) python3 tools/daf/daf.py -a -p ./ -# ``./`` ok for relative path? test: $(ROM) $(EMULATOR) $(EMU_FLAGS) $<