hackern64/HackerOoT
0
0
Fork 0
mirror of https://github.com/HackerN64/HackerOoT.git synced 2026-01-21 10:37:37 -08:00
Code Issues Packages Projects Releases Wiki Activity

Setup rom decompression and compression (#1614)

Browse Source 
* decompress baserom

* cleanup

* specific hash check

* rename baserom

* git subrepo clone (merge) --branch=5da3132606e4fd427a8d72b8428e4f921cd6e56f git@github.com:z64tools/z64compress.git tools/z64compress

subrepo:
  subdir:   "tools/z64compress"
  merged:   "5da313260"
upstream:
  origin:   "git@github.com:z64tools/z64compress.git"
  branch:   "5da3132606e4fd427a8d72b8428e4f921cd6e56f"
  commit:   "5da313260"
git-subrepo:
  version:  "0.4.3"
  origin:   "https://github.com/ingydotnet/git-subrepo.git"
  commit:   "2f68596"

* setup compression

* Add all compressed segments to the spec

* Update md5 files

* readme instructions

* cleanup

* Setup python dependencies on Jenkinsfile

* Update Makefile

Co-authored-by: cadmic <cadmic24@gmail.com>

* review

* . .venv/bin/activate

* update readme

* whoops

* Yeet other versions from decompress_baserom.py

* my bad

* Move everything to baseroms/VERSION/

* Active venv on each command

* jenkinsfile: use multiline syntax

* Put the correct path on the jenkinsfile

* Forgot to call per_version_fixes

* CC0

* Update readme

* Change where baserom segments are put into

* Update Makefile

Co-authored-by: Dragorn421 <Dragorn421@users.noreply.github.com>

* Update crunch64

* Label compressed instead of uncompressed

* Update README.md

Co-authored-by: fig02 <fig02srl@gmail.com>

* Fix

* `make rom`

* baserom-uncompressed

* Update README.md

Co-authored-by: Dragorn421 <Dragorn421@users.noreply.github.com>

* Update README.md

Co-authored-by: Dragorn421 <Dragorn421@users.noreply.github.com>

* Update README.md

Co-authored-by: Dragorn421 <Dragorn421@users.noreply.github.com>

* Update README.md

Co-authored-by: Dragorn421 <Dragorn421@users.noreply.github.com>

* Update README.md

Co-authored-by: Dragorn421 <Dragorn421@users.noreply.github.com>

* review

* baserom-decompressed.z64

* ignore baseroms

* rm -rf tools/z64compress

* wip crunch64-based compress.py

* OK compress

* use ipl3checksum sum directly for cic update on compressed rom

* multithreading

* "docs"

* fix MT: move set_sigint_ignored to global level for pickling

* license compress.py

* rm junk

* Fix (or at least sort out) compress_ranges.txt dependencies

* Update tools/overlayhelpers/damage_table.py

Co-authored-by: Dragorn421 <Dragorn421@users.noreply.github.com>

---------

Co-authored-by: cadmic <cadmic24@gmail.com>
Co-authored-by: Dragorn421 <Dragorn421@users.noreply.github.com>
Co-authored-by: fig02 <fig02srl@gmail.com>
This commit is contained in:
Anghelo Carvajal
2024-01-24 15:00:10 -03:00
committed by GitHub
parent 5e406f754e
commit 3d9db8d34d
36 changed files with 2282 additions and 187 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@@ -10,12 +10,14 @@ __pycache__/
CMakeLists.txt
cmake-build-debug
venv/
.venv/
# Project-specific ignores
build/
expected/
notes/
baserom/
baseroms/*/segments/
docs/doxygen/
*.elf
*.sra
@@ -25,6 +27,7 @@ docs/doxygen/
*.map
*.dump
out.txt
*.ram
# Tool artifacts
tools/mipspro7.2_compiler/

23
Jenkinsfile vendored
View File

@@ -4,10 +4,21 @@ pipeline {
}
stages {
stage('Install Python dependencies') {
steps {
echo 'Installing Python dependencies'
sh 'python3 -m venv .venv'
sh '''. .venv/bin/activate
python3 -m pip install -U -r requirements.txt
'''
}
}
stage('Setup') {
steps {
sh 'cp /usr/local/etc/roms/baserom_oot.z64 baserom_original.z64'
sh 'make -j setup'
sh 'cp /usr/local/etc/roms/baserom_oot.z64 baseroms/gc-eu-mq-dbg/baserom.z64'
sh '''. .venv/bin/activate
make -j setup
'''
}
}
stage('Build (qemu-irix)') {
@@ -15,7 +26,9 @@ pipeline {
branch 'main'
}
steps {
sh 'ORIG_COMPILER=1 make -j'
sh '''. .venv/bin/activate
make -j ORIG_COMPILER=1
'''
}
}
stage('Build') {
@@ -25,7 +38,9 @@ pipeline {
}
}
steps {
sh 'make -j'
sh '''. .venv/bin/activate
make -j
'''
}
}
stage('Report Progress') {

64
Makefile
View File

@@ -17,6 +17,8 @@ COMPILER := ido
# Target game version. Currently only the following version is supported:
# gc-eu-mq-dbg GameCube Europe/PAL Master Quest Debug (default)
VERSION := gc-eu-mq-dbg
# Number of threads to extract and compress with
N_THREADS := $(shell nproc)
CFLAGS ?=
CPPFLAGS ?=
@@ -72,8 +74,6 @@ else
endif
endif
N_THREADS := $(shell nproc)
#### Tools ####
ifneq ($(shell type $(MIPS_BINUTILS_PREFIX)ld >/dev/null 2>/dev/null; echo $$?), 0)
$(error Unable to find $(MIPS_BINUTILS_PREFIX)ld. Please install or build MIPS binutils, commonly mips-linux-gnu. (or set MIPS_BINUTILS_PREFIX if your MIPS binutils install uses another prefix))
@@ -105,6 +105,7 @@ AS := $(MIPS_BINUTILS_PREFIX)as
LD := $(MIPS_BINUTILS_PREFIX)ld
OBJCOPY := $(MIPS_BINUTILS_PREFIX)objcopy
OBJDUMP := $(MIPS_BINUTILS_PREFIX)objdump
NM := $(MIPS_BINUTILS_PREFIX)nm
N64_EMULATOR ?=
@@ -119,6 +120,7 @@ MKDMADATA := tools/mkdmadata
ELF2ROM := tools/elf2rom
ZAPD := tools/ZAPD/ZAPD.out
FADO := tools/fado/fado.elf
PYTHON ?= python3
# Command to replace path variables in the spec file. We can't use the C
# preprocessor for this because it won't substitute inside string literals.
@@ -161,6 +163,7 @@ OBJDUMP_FLAGS := -d -r -z -Mreg-names=32
#### Files ####
# ROM image
ROMC := oot-$(VERSION)-compressed.z64
ROM := oot-$(VERSION).z64
ELF := $(ROM:.z64=.elf)
# description of ROM segments
@@ -184,9 +187,10 @@ UNDECOMPILED_DATA_DIRS := $(shell find data -type d)
# source files
C_FILES := $(filter-out %.inc.c,$(foreach dir,$(SRC_DIRS) $(ASSET_BIN_DIRS),$(wildcard $(dir)/*.c)))
S_FILES := $(foreach dir,$(SRC_DIRS) $(UNDECOMPILED_DATA_DIRS),$(wildcard $(dir)/*.s))
BASEROM_BIN_FILES := $(wildcard baseroms/$(VERSION)/segments/*)
O_FILES := $(foreach f,$(S_FILES:.s=.o),$(BUILD_DIR)/$f) \
$(foreach f,$(C_FILES:.c=.o),$(BUILD_DIR)/$f) \
$(foreach f,$(wildcard baserom/*),$(BUILD_DIR)/$f.o)
$(foreach f,$(BASEROM_BIN_FILES),$(BUILD_DIR)/baserom/$(notdir $f).o)
OVL_RELOC_FILES := $(shell $(CPP) $(CPPFLAGS) $(SPEC) | $(SPEC_REPLACE_VARS) | grep -o '[^"]*_reloc.o' )
@@ -246,11 +250,11 @@ $(BUILD_DIR)/src/libultra/rmon/%.o: CC := $(CC_OLD)
$(BUILD_DIR)/src/code/jpegutils.o: CC := $(CC_OLD)
$(BUILD_DIR)/src/code/jpegdecoder.o: CC := $(CC_OLD)
$(BUILD_DIR)/src/boot/%.o: CC := python3 tools/asm_processor/build.py $(CC) -- $(AS) $(ASFLAGS) --
$(BUILD_DIR)/src/code/%.o: CC := python3 tools/asm_processor/build.py $(CC) -- $(AS) $(ASFLAGS) --
$(BUILD_DIR)/src/overlays/%.o: CC := python3 tools/asm_processor/build.py $(CC) -- $(AS) $(ASFLAGS) --
$(BUILD_DIR)/src/boot/%.o: CC := $(PYTHON) tools/asm_processor/build.py $(CC) -- $(AS) $(ASFLAGS) --
$(BUILD_DIR)/src/code/%.o: CC := $(PYTHON) tools/asm_processor/build.py $(CC) -- $(AS) $(ASFLAGS) --
$(BUILD_DIR)/src/overlays/%.o: CC := $(PYTHON) tools/asm_processor/build.py $(CC) -- $(AS) $(ASFLAGS) --
$(BUILD_DIR)/assets/%.o: CC := python3 tools/asm_processor/build.py $(CC) -- $(AS) $(ASFLAGS) --
$(BUILD_DIR)/assets/%.o: CC := $(PYTHON) tools/asm_processor/build.py $(CC) -- $(AS) $(ASFLAGS) --
else
$(BUILD_DIR)/src/libultra/libc/ll.o: OPTFLAGS := -Ofast
$(BUILD_DIR)/src/%.o: CC := $(CC) -fexec-charset=euc-jp
@@ -258,14 +262,20 @@ endif
#### Main Targets ###
all: $(ROM)
ifeq ($(COMPARE),1)
rom: $(ROM)
ifneq ($(COMPARE),0)
@md5sum $(ROM)
@md5sum -c checksum.md5
@md5sum -c baseroms/$(VERSION)/checksum.md5
endif
compressed: $(ROMC)
ifneq ($(COMPARE),0)
@md5sum $(ROMC)
@md5sum -c baseroms/$(VERSION)/checksum-compressed.md5
endif
clean:
$(RM) -r $(ROM) $(ELF) $(BUILD_DIR)
$(RM) -r $(ROMC) $(ROM) $(ELF) $(BUILD_DIR)
assetclean:
$(RM) -r $(ASSET_BIN_DIRS)
@@ -274,14 +284,14 @@ assetclean:
$(RM) -r .extracted-assets.json
distclean: clean assetclean
$(RM) -r baserom/
$(RM) -r baseroms/$(VERSION)/segments
$(MAKE) -C tools distclean
setup:
$(MAKE) -C tools
python3 fixbaserom.py
python3 extract_baserom.py
python3 extract_assets.py -j$(N_THREADS)
$(PYTHON) tools/decompress_baserom.py $(VERSION)
$(PYTHON) extract_baserom.py
$(PYTHON) extract_assets.py -j$(N_THREADS)
run: $(ROM)
ifeq ($(N64_EMULATOR),)
@@ -290,13 +300,20 @@ endif
$(N64_EMULATOR) $<
.PHONY: all clean setup run distclean assetclean
.PHONY: all rom compressed clean setup run distclean assetclean
.DEFAULT_GOAL := rom
all: rom compressed
#### Various Recipes ####
$(ROM): $(ELF)
$(ELF2ROM) -cic 6105 $< $@
$(ROMC): $(ROM) $(ELF) $(BUILD_DIR)/compress_ranges.txt
# note: $(BUILD_DIR)/compress_ranges.txt should only be used for nonmatching builds. it works by chance for matching builds too though
$(PYTHON) tools/compress.py --in $(ROM) --out $@ --dma-range `./tools/dmadata_range.sh $(NM) $(ELF)` --compress `cat $(BUILD_DIR)/compress_ranges.txt` --threads $(N_THREADS)
$(PYTHON) -m ipl3checksum sum --cic 6105 --update $@
$(ELF): $(TEXTURE_FILES_OUT) $(ASSET_FILES_OUT) $(O_FILES) $(OVL_RELOC_FILES) $(BUILD_DIR)/ldscript.txt $(BUILD_DIR)/undefined_syms.txt
$(LD) -T $(BUILD_DIR)/undefined_syms.txt -T $(BUILD_DIR)/ldscript.txt --no-check-sections --accept-unknown-input-arch --emit-relocs -Map $(BUILD_DIR)/z64.map -o $@
@@ -321,14 +338,14 @@ $(BUILD_DIR)/ldscript.txt: $(BUILD_DIR)/$(SPEC)
$(BUILD_DIR)/undefined_syms.txt: undefined_syms.txt
$(CPP) $(CPPFLAGS) $< > $@
$(BUILD_DIR)/baserom/%.o: baserom/%
$(BUILD_DIR)/baserom/%.o: baseroms/$(VERSION)/segments/%
$(OBJCOPY) -I binary -O elf32-big $< $@
$(BUILD_DIR)/data/%.o: data/%.s
$(AS) $(ASFLAGS) $< -o $@
$(BUILD_DIR)/assets/text/%.enc.h: assets/text/%.h assets/text/charmap.txt
python3 tools/msgenc.py assets/text/charmap.txt $< $@
$(PYTHON) tools/msgenc.py assets/text/charmap.txt $< $@
# Dependencies for files including message data headers
# TODO remove when full header dependencies are used.
@@ -345,8 +362,8 @@ $(BUILD_DIR)/assets/%.o: assets/%.c
$(BUILD_DIR)/src/%.o: src/%.s
$(CPP) $(CPPFLAGS) -Iinclude $< | $(AS) $(ASFLAGS) -o $@
$(BUILD_DIR)/dmadata_table_spec.h: $(BUILD_DIR)/$(SPEC)
$(MKDMADATA) $< $@
$(BUILD_DIR)/dmadata_table_spec.h $(BUILD_DIR)/compress_ranges.txt: $(BUILD_DIR)/$(SPEC)
$(MKDMADATA) $< $(BUILD_DIR)/dmadata_table_spec.h $(BUILD_DIR)/compress_ranges.txt
# Dependencies for files that may include the dmadata header automatically generated from the spec file
$(BUILD_DIR)/src/boot/z_std_dma.o: $(BUILD_DIR)/dmadata_table_spec.h
@@ -370,13 +387,13 @@ $(BUILD_DIR)/src/%.o: src/%.c
$(BUILD_DIR)/src/libultra/libc/ll.o: src/libultra/libc/ll.c
$(CC_CHECK) $<
$(CC) -c $(CFLAGS) $(MIPS_VERSION) $(OPTFLAGS) -o $@ $<
python3 tools/set_o32abi_bit.py $@
$(PYTHON) tools/set_o32abi_bit.py $@
@$(OBJDUMP) $(OBJDUMP_FLAGS) $@ > $(@:.o=.s)
$(BUILD_DIR)/src/libultra/libc/llcvt.o: src/libultra/libc/llcvt.c
$(CC_CHECK) $<
$(CC) -c $(CFLAGS) $(MIPS_VERSION) $(OPTFLAGS) -o $@ $<
python3 tools/set_o32abi_bit.py $@
$(PYTHON) tools/set_o32abi_bit.py $@
@$(OBJDUMP) $(OBJDUMP_FLAGS) $@ > $(@:.o=.s)
$(BUILD_DIR)/src/overlays/%_reloc.o: $(BUILD_DIR)/$(SPEC)
@@ -393,3 +410,6 @@ $(BUILD_DIR)/assets/%.jpg.inc.c: assets/%.jpg
$(ZAPD) bren -eh -i $< -o $@
-include $(DEP_FILES)
# Print target for debugging
print-% : ; $(info $* is a $(flavor $*) variable set to [$($*)]) @true

42
README.md
View File

@@ -68,13 +68,15 @@ The build process has the following package requirements:
* build-essential
* binutils-mips-linux-gnu
* python3
* python3-pip
* python3-venv
* libpng-dev
Under Debian / Ubuntu (which we recommend using), you can install them with the following commands:
```bash
sudo apt-get update
sudo apt-get install git build-essential binutils-mips-linux-gnu python3 libpng-dev
sudo apt-get install git build-essential binutils-mips-linux-gnu python3 python3-pip python3-venv libpng-dev
```
If you are using GCC as the compiler for Ocarina of Time, you will also need:
@@ -98,12 +100,37 @@ This will copy the GitHub repository contents into a new folder in the current d
cd oot
```
#### 3. Prepare a base ROM
#### 3. Install python dependencies
Copy over your copy of the Master Quest (Debug) ROM inside the root of this new project directory.
Rename the file to "baserom_original.z64", "baserom_original.n64" or "baserom_original.v64", depending on the original extension.
The build process has a few python packages required that are located in `requirements.txt`.
#### 4. Setup the ROM and build process
It is recommended to set up a virtual environment for python to contain all dependencies. To create a virtual environment:
```bash
python3 -m venv .venv
```
To start using the virtual environment in your current terminal run:
```bash
. .venv/bin/activate
```
Keep in mind that for each new terminal session, you will need to **activate** the Python virtual environment again. That is, run the above `. .venv/bin/activate` command.
Now you can install the Python dependencies, to do so run:
```bash
python3 -m pip install -U -r requirements.txt
```
#### 4. Prepare a base ROM
Place a copy of the Master Quest (Debug) ROM inside the `baseroms/gc-eu-mq-dbg/` folder.
Rename the file to `baserom.z64`, `baserom.n64` or `baserom.v64`, depending on the original extension.
#### 5. Setup the ROM and build process
Setup and extract everything from your ROM with the following command:
@@ -111,10 +138,10 @@ Setup and extract everything from your ROM with the following command:
make setup
```
This will generate a new ROM called "baserom.z64" that will have the overdump removed and the header patched.
This will generate a new ROM "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64" that will have the overdump removed and the header patched.
It will also extract the individual assets from the ROM.
#### 5. Build the ROM
#### 6. Build the ROM
Run make to build the ROM.
Make sure your path to the project is not too long, otherwise this process may error.
@@ -145,7 +172,6 @@ This means that the built ROM isn't the same as the base one, so something went
Both of these have the disadvantage that the ordering of the terminal output is scrambled, so for debugging it is best to stick to one thread (i.e. not pass `-j` or `-jN`).
## Contributing
All contributions are welcome. This is a group effort, and even small contributions can make a difference.

1
baseroms/gc-eu-mq-dbg/checksum-compressed.md5 Normal file
View File

@@ -0,0 +1 @@
5831385a7f216370cdbea55616b12fed oot-gc-eu-mq-dbg-compressed.z64

0
checksum.md5 → baseroms/gc-eu-mq-dbg/checksum.md5
View File

8
data/audio_tables.rodata.s
View File

@@ -10,13 +10,13 @@
.balign 16
glabel gSoundFontTable
.incbin "baserom.z64", 0xBCC270, 0x270
.incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0xBCC270, 0x270
glabel gSequenceFontTable
.incbin "baserom.z64", 0xBCC4E0, 0x1C0
.incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0xBCC4E0, 0x1C0
glabel gSequenceTable
.incbin "baserom.z64", 0xBCC6A0, 0x6F0
.incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0xBCC6A0, 0x6F0
glabel gSampleBankTable
.incbin "baserom.z64", 0xBCCD90, 0x80
.incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0xBCCD90, 0x80

10
data/rsp.rodata.s
View File

@@ -10,21 +10,21 @@
.balign 16
glabel aspMainDataStart
.incbin "baserom.z64", 0xBCCE10, 0x2E0
.incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0xBCCE10, 0x2E0
glabel aspMainDataEnd
glabel gspF3DZEX2_NoN_PosLight_fifoTextStart
.incbin "baserom.z64", 0xBCD0F0, 0x1630
.incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0xBCD0F0, 0x1630
glabel gspF3DZEX2_NoN_PosLight_fifoTextEnd
glabel gspF3DZEX2_NoN_PosLight_fifoDataStart
.incbin "baserom.z64", 0xBCE720, 0x420
.incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0xBCE720, 0x420
glabel gspF3DZEX2_NoN_PosLight_fifoDataEnd
glabel gspS2DEX2d_fifoDataStart
.incbin "baserom.z64", 0xBCEB40, 0x390
.incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0xBCEB40, 0x390
glabel gspS2DEX2d_fifoDataEnd
glabel njpgdspMainDataStart
.incbin "baserom.z64", 0xBCEED0, 0x60
.incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0xBCEED0, 0x60
glabel njpgdspMainDataEnd

6
data/rsp.text.s
View File

@@ -10,13 +10,13 @@
.balign 16
glabel aspMainTextStart
.incbin "baserom.z64", 0xB89260, 0xFB0
.incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0xB89260, 0xFB0
glabel aspMainTextEnd
glabel gspS2DEX2d_fifoTextStart
.incbin "baserom.z64", 0xB8A210, 0x18C0
.incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0xB8A210, 0x18C0
glabel gspS2DEX2d_fifoTextEnd
glabel njpgdspMainTextStart
.incbin "baserom.z64", 0xB8BAD0, 0xAF0
.incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0xB8BAD0, 0xAF0
glabel njpgdspMainTextEnd

2
data/rsp_boot.text.s
View File

@@ -10,5 +10,5 @@
.balign 16
glabel rspbootTextStart
.incbin "baserom.z64", 0x9F20, 0xD0
.incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0x9F20, 0xD0
glabel rspbootTextEnd

4
data/z_text.data.s
View File

@@ -12,7 +12,7 @@
# temporary file name, rename to something more appropriate when decompiled
glabel gMojiFontTLUTs
.incbin "baserom.z64", 0xBA18E0, 0x80
.incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0xBA18E0, 0x80
glabel gMojiFontTex
.incbin "baserom.z64", 0xBA1960, 0x400
.incbin "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64", 0xBA1960, 0x400

2
diff_settings.py
View File

@@ -1,6 +1,6 @@
def apply(config, args):
config['mapfile'] = 'build/gc-eu-mq-dbg/z64.map'
config['myimg'] = 'oot-gc-eu-mq-dbg.z64'
config['baseimg'] = 'baserom.z64'
config['baseimg'] = 'baseroms/gc-eu-mq-dbg/baserom-decompressed.z64'
config['makeflags'] = []
config['source_directories'] = ['src', 'include', 'spec']

4
docs/tutorial/beginning_decomp.md
View File

@@ -229,7 +229,7 @@ which initialises common properties of actor using an InitChain, which is usuall
The InitChain script is also in the tools directory, and is called `ichaindis.py`. Simply passing it the ROM address will spit out the entire contents of the InitChain, in this case:
```
$ ./tools/ichaindis.py baserom.z64 80A88CE0
$ ./tools/ichaindis.py baseroms/gc-eu-mq-dbg/baserom-decompressed.z64 80A88CE0
static InitChainEntry sInitChain[] = {
ICHAIN_VEC3F_DIV1000(unk_50, 87, ICHAIN_CONTINUE),
ICHAIN_F32(unk_F4, 4000, ICHAIN_CONTINUE),
@@ -240,7 +240,7 @@ static InitChainEntry sInitChain[] = {
However, some of these variables have now been given names in the Actor struct. Pass it `--names` to fill these in automatically:
```
$ ./tools/ichaindis.py --names baserom.z64 80A88CE0
$ ./tools/ichaindis.py --names baseroms/gc-eu-mq-dbg/baserom-decompressed.z64 80A88CE0
static InitChainEntry sInitChain[] = {
ICHAIN_VEC3F_DIV1000(scale, 87, ICHAIN_CONTINUE),
ICHAIN_F32(uncullZoneForward, 4000, ICHAIN_CONTINUE),

2
docs/tutorial/data.md
View File

@@ -557,7 +557,7 @@ Ignore the first line: `gDmaDataTable` is always different if the ROM is shifted
To fix this, we use a binary diff program. A suitable one is `vbindiff`: run it on the baserom and the zelda_whatever one the compiler generates:
```
vbindiff baserom.z64 oot-gc-eu-mq-dbg.z64
vbindiff baseroms/gc-eu-mq-dbg/baserom-decompressed.z64 oot-gc-eu-mq-dbg.z64
```
In this, press `g` to open up goto position, and paste in the address `0xE3ED10` from the first important line of the `first_diff` output. This gives us the following:

2
docs/vscode.md
View File

@@ -74,7 +74,7 @@ Add the following to (or create) the `.vscode/settings.json` file for VSCode to
"search.useIgnoreFiles": false,
"search.exclude": {
"**/.git": true,
"baserom/**": true,
"baseroms/**": true,
"build/**": true,
"expected/**": true,
},

2
extract_assets.py
View File

@@ -28,7 +28,7 @@ def ExtractFile(xmlPath, outputPath, outputSourcePath):
Path(outputPath).mkdir(parents=True, exist_ok=True)
Path(outputSourcePath).mkdir(parents=True, exist_ok=True)
execStr = f"{zapdPath} e -eh -i {xmlPath} -b baserom -o {outputPath} -osf {outputSourcePath} -gsf 1 -rconf {configPath} {ZAPDArgs}"
execStr = f"{zapdPath} e -eh -i {xmlPath} -b baseroms/gc-eu-mq-dbg/segments -o {outputPath} -osf {outputSourcePath} -gsf 1 -rconf {configPath} {ZAPDArgs}"
if "overlays" in xmlPath:
execStr += " --static"

26
extract_baserom.py
View File

@@ -1,12 +1,12 @@
#!/usr/bin/python3
import os
import sys
import struct
from multiprocessing import Pool, cpu_count
from pathlib import Path
ROM_FILE_NAME = 'baserom.z64'
ROM_FILE_PATH = Path('baseroms/gc-eu-mq-dbg/baserom-decompressed.z64')
SEGMENTS_PATH = Path('baseroms/gc-eu-mq-dbg/segments/')
FILE_TABLE_OFFSET = 0x12F70
FILE_NAMES = [
@@ -1562,7 +1562,7 @@ def write_output_file(name, offset, size):
print('failed to write file ' + name)
def ExtractFunc(i):
filename = 'baserom/' + FILE_NAMES[i]
filename = SEGMENTS_PATH / FILE_NAMES[i]
entryOffset = FILE_TABLE_OFFSET + 16 * i
virtStart = read_uint32_be(entryOffset + 0)
@@ -1571,32 +1571,24 @@ def ExtractFunc(i):
physEnd = read_uint32_be(entryOffset + 12)
if physEnd == 0: # uncompressed
compressed = False
size = virtEnd - virtStart
else: # compressed
compressed = True
size = physEnd - physStart
print('extracting ' + filename + " (0x%08X, 0x%08X)" % (virtStart, virtEnd))
print(f'extracting {filename} (0x{virtStart:08X}, 0x{virtEnd:08X})')
write_output_file(filename, physStart, size)
if compressed:
os.system('tools/yaz0 -d ' + filename + ' ' + filename)
#####################################################################
def main():
try:
os.mkdir('baserom')
except:
pass
SEGMENTS_PATH.mkdir(parents=True, exist_ok=True)
# read baserom data
try:
with open(ROM_FILE_NAME, 'rb') as f:
rom_data = f.read()
rom_data = ROM_FILE_PATH.read_bytes()
except IOError:
print('failed to read file' + ROM_FILE_NAME)
sys.exit(1)
print(f'failed to read file {ROM_FILE_PATH}')
exit(1)
# extract files
num_cores = cpu_count()

2
first_diff.py
View File

@@ -49,7 +49,7 @@ def firstDiffMain():
BUILTROM = Path(f"oot-{args.version}.z64")
BUILTMAP = buildFolder / "z64.map"
EXPECTEDROM = Path("baserom.z64")
EXPECTEDROM = Path(f"baseroms/{args.version}/baserom-decompressed.z64")
EXPECTEDMAP = "expected" / BUILTMAP
mapfile_parser.frontends.first_diff.doFirstDiff(BUILTMAP, EXPECTEDMAP, BUILTROM, EXPECTEDROM, args.count, mismatchSize=True, addColons=args.add_colons, bytesConverterCallback=decodeInstruction)

96
fixbaserom.py
View File

@@ -1,96 +0,0 @@
from os import path
import sys
import struct
import hashlib
def get_str_hash(byte_array):
return str(hashlib.md5(byte_array).hexdigest())
# If the baserom exists and is correct, we don't need to change anything
if path.exists("baserom.z64"):
with open("baserom.z64", mode="rb") as file:
fileContent = bytearray(file.read())
if get_str_hash(fileContent) == "f0b7f35375f9cc8ca1b2d59d78e35405":
print("Found valid baserom - exiting early")
sys.exit(0)
# Determine if we have a ROM file
romFileExtensions = ["z64", "n64", "v64"]
def find_baserom_original():
for romFileExtLower in romFileExtensions:
for romFileExt in (romFileExtLower, romFileExtLower.upper()):
romFileNameCandidate = "baserom_original." + romFileExt
if path.exists(romFileNameCandidate):
return romFileNameCandidate
return None
romFileName = find_baserom_original()
if romFileName is None:
print("Error: Could not find baserom_original.z64/baserom_original.n64/baserom_original.v64.")
sys.exit(1)
# Read in the original ROM
print("File '" + romFileName + "' found.")
with open(romFileName, mode="rb") as file:
fileContent = bytearray(file.read())
# Strip the overdump
print("Stripping overdump...")
fileContent = fileContent[0:0x3600000]
fileContentLen = len(fileContent)
# Check if ROM needs to be byte/word swapped
# Little-endian
if fileContent[0] == 0x40:
# Word Swap ROM
print("ROM needs to be word swapped...")
words = str(int(fileContentLen/4))
little_byte_format = "<" + words + "I"
big_byte_format = ">" + words + "I"
tmp = struct.unpack_from(little_byte_format, fileContent, 0)
struct.pack_into(big_byte_format, fileContent, 0, *tmp)
print("Word swapping done.")
# Byte-swapped
elif fileContent[0] == 0x37:
# Byte Swap ROM
print("ROM needs to be byte swapped...")
halfwords = str(int(fileContentLen/2))
little_byte_format = "<" + halfwords + "H"
big_byte_format = ">" + halfwords + "H"
tmp = struct.unpack_from(little_byte_format, fileContent, 0)
struct.pack_into(big_byte_format, fileContent, 0, *tmp)
print("Byte swapping done.")
# Patch the header
print("Patching header...")
fileContent[0x3E] = 0x50
for i in range(0x35CF000, len(fileContent)):
fileContent[i] = 0xFF
# Check to see if the ROM is a "vanilla" Debug ROM
str_hash = get_str_hash(bytearray(fileContent))
if str_hash != "f0b7f35375f9cc8ca1b2d59d78e35405":
print("Error: Expected a hash of f0b7f35375f9cc8ca1b2d59d78e35405 but got " + str_hash + ". " +
"The baserom has probably been tampered, find a new one")
if str_hash == "32fe2770c0f9b1a9cd2a4d449348c1cb":
print("The provided baserom is a rom which has been edited with ZeldaEdit and is not suitable for use with decomp. " +
"Find a new one.")
sys.exit(1)
# Write out our new ROM
print("Writing new ROM 'baserom.z64'.")
with open("baserom.z64", mode="wb") as file:
file.write(bytes(fileContent))
print("Done!")

3
requirements.txt Normal file
View File

@@ -0,0 +1,3 @@
# Setup and compression
crunch64>=0.3.1,<1.0.0
ipl3checksum>=1.2.0,<2.0.0

Some files were not shown because too many files have changed in this diff Show More