From 8b0d8d457b2907bff2e0aafe917600dc82b4455c Mon Sep 17 00:00:00 2001 From: David Benepe Date: Sat, 27 Jun 2020 20:31:57 -0500 Subject: [PATCH] Started decompilation! --- .gitignore | 3 + Makefile | 34 +- asm/non_matchings/func_80065C38.s | 69 ++ asm/unknown_064800.s | 1 - asm/unknown_0667D0.s | 206 +++--- asm/unknown_080500.s | 84 +-- diff.py | 1012 ++++++++++++++++++++++++++ diff_settings.py | 20 + src/macros.h | 65 ++ src/types.h | 11 + src/unknown_0667D0.c | 32 + src/unknown_09F860.c | 27 + tools/asm_processor/asm-processor.py | 868 ++++++++++++++++++++++ tools/asm_processor/build.py | 36 + tools/asm_processor/prelude.inc | 7 + tools/python/generate_ld.py | 27 +- 16 files changed, 2335 insertions(+), 167 deletions(-) create mode 100644 asm/non_matchings/func_80065C38.s create mode 100644 diff.py create mode 100644 diff_settings.py create mode 100644 src/macros.h create mode 100644 src/types.h create mode 100644 src/unknown_0667D0.c create mode 100644 src/unknown_09F860.c create mode 100755 tools/asm_processor/asm-processor.py create mode 100755 tools/asm_processor/build.py create mode 100755 tools/asm_processor/prelude.inc diff --git a/.gitignore b/.gitignore index fbb79003..c6f6a240 100755 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,9 @@ # Baseroms is where the user will place their ROM files. baseroms/* +# Ignore pycache +__pycache__ + # Automatically generated folders should be ignored build/ assets/ diff --git a/Makefile b/Makefile index 8d18579c..a70453a7 100755 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ SHELL := /bin/bash BUILD_DIR = build ##################### Compiler Options ####################### -#IRIX_ROOT := tools/ido5.3_compiler +IRIX_ROOT := tools/ido5.3_compiler ifeq ($(shell type mips-linux-gnu-ld >/dev/null 2>/dev/null; echo $$?), 0) CROSS := mips-linux-gnu- @@ -21,24 +21,22 @@ else endif # check that either QEMU_IRIX is set or qemu-irix package installed -#ifndef QEMU_IRIX -# QEMU_IRIX := $(shell which qemu-irix 2>/dev/null) -# ifeq (, $(QEMU_IRIX)) -# $(error Please install qemu-irix package or set QEMU_IRIX env var to the full qemu-irix binary path) -# endif -#endif +ifndef QEMU_IRIX + QEMU_IRIX := $(shell which qemu-irix 2>/dev/null) + ifeq (, $(QEMU_IRIX)) + $(error Please install qemu-irix package or set QEMU_IRIX env var to the full qemu-irix binary path) + endif +endif AS = $(CROSS)as -#CC = $(CROSS)gcc CC := $(QEMU_IRIX) -silent -L $(IRIX_ROOT) $(IRIX_ROOT)/usr/bin/cc CPP := cpp -P -Wno-trigraphs LD = $(CROSS)ld OBJDUMP = $(CROSS)objdump OBJCOPY = $(CROSS)objcopy --pad-to=0xC00000 --gap-fill=0xFF -INCLUDE_FLAGS := -I$(BUILD_DIR) ASFLAGS = -mtune=vr4300 -march=vr4300 $(INCLUDE_FLAGS) -CFLAGS = -Wall -O2 -mtune=vr4300 -march=vr4300 -G 0 -c +CFLAGS = -c -Wab,-r4300_mul -non_shared -G 0 -Xcpluscomm -Xfullwarn -signed -O2 -mips1 LDFLAGS = undefined_syms.txt -T $(LD_SCRIPT) -Map $(BUILD_DIR)/dkr.map ####################### Other Tools ######################### @@ -61,12 +59,15 @@ ASM_DIRS := asm asm/boot asm/assets SRC_DIRS := src ASSETS_DIRS := animations audio billboards bin cheats fonts levels objects particles text textures textures/2d textures/3d tt_ghosts ucode -S_FILES := $(foreach dir,$(ASM_DIRS),$(wildcard $(dir)/*.s)) +GLOBAL_ASM_C_FILES != grep -rl 'GLOBAL_ASM(' $(wildcard src/*/*.c) +GLOBAL_ASM_O_FILES = $(foreach file,$(GLOBAL_ASM_C_FILES),$(BUILD_DIR)/$(file:.c=.o)) -BUILD_ASM_DIRS := $(foreach dir,$(ASM_DIRS),$(wildcard $(dir)/**/)) +S_FILES := $(foreach dir,$(ASM_DIRS),$(wildcard $(dir)/*.s)) +C_FILES := $(foreach dir,$(SRC_DIRS),$(wildcard $(dir)/*.c)) # Object files -O_FILES := $(foreach file,$(S_FILES),$(BUILD_DIR)/$(file:.s=.o)) +O_FILES := $(foreach file,$(S_FILES),$(BUILD_DIR)/$(file:.s=.o)) \ + $(foreach file,$(C_FILES),$(BUILD_DIR)/$(file:.c=.o)) ####################### ASSETS ######################### @@ -167,7 +168,7 @@ clean: rm -r $(BUILD_DIR) $(BUILD_DIR): - mkdir $(BUILD_DIR) $(addprefix $(BUILD_DIR)/,$(ASM_DIRS) $(ASSETS_DIRS)) + mkdir $(BUILD_DIR) $(addprefix $(BUILD_DIR)/,$(ASM_DIRS) $(SRC_DIRS) $(ASSETS_DIRS)) # This is here to prevent make from deleting all the asset files after the build completes/fails. dont_remove_asset_files: $(ALL_ASSETS_BUILT) @@ -233,6 +234,9 @@ $(UCODE_OUT_DIR)/%.bin: $(UCODE_IN_DIR)/%.bin $(BUILD_DIR)/%.o: %.s Makefile $(MAKEFILE_SPLIT) | $(BUILD_DIR) $(ALL_ASSETS_BUILT) $(AS) $(ASFLAGS) -o $@ $< +$(BUILD_DIR)/%.o: %.c + $(CC) $(CFLAGS) -o $@ $< + $(BUILD_DIR)/$(LD_SCRIPT): $(LD_SCRIPT) $(CPP) $(VERSION_CFLAGS) -DBUILD_DIR=$(BUILD_DIR) -MMD -MP -MT $@ -MF $@.d -o $@ $< @@ -253,6 +257,8 @@ $(BUILD_DIR)/$(TARGET).hex: $(BUILD_DIR)/$(TARGET).z64 $(BUILD_DIR)/$(TARGET).objdump: $(BUILD_DIR)/$(TARGET).elf $(OBJDUMP) -D $< > $@ + +$(GLOBAL_ASM_O_FILES): CC := $(PYTHON) tools/asm_processor/build.py $(CC) -- $(AS) $(ASFLAGS) -- test: $(BUILD_DIR)/$(TARGET).z64 $(EMULATOR) $(EMU_FLAGS) $< diff --git a/asm/non_matchings/func_80065C38.s b/asm/non_matchings/func_80065C38.s new file mode 100644 index 00000000..ae821636 --- /dev/null +++ b/asm/non_matchings/func_80065C38.s @@ -0,0 +1,69 @@ +glabel func_80065C38 +/* 066838 80065C38 27BDFFE0 */ addiu $sp, $sp, -0x20 +/* 06683C 80065C3C AFBF0014 */ sw $ra, 0x14($sp) +/* 066840 80065C40 AFA40020 */ sw $a0, 0x20($sp) +/* 066844 80065C44 AFA50024 */ sw $a1, 0x24($sp) +/* 066848 80065C48 AFA60028 */ sw $a2, 0x28($sp) +/* 06684C 80065C4C AFA7002C */ sw $a3, 0x2c($sp) +/* 066850 80065C50 8CAF0008 */ lw $t7, 8($a1) +/* 066854 80065C54 00000000 */ nop +/* 066858 80065C58 11E00036 */ beqz $t7, .L80065D34 +/* 06685C 80065C5C 8FBF0014 */ lw $ra, 0x14($sp) +/* 066860 80065C60 0C01959A */ jal func_80065668 +/* 066864 80065C64 00000000 */ nop +/* 066868 80065C68 10400031 */ beqz $v0, .L80065D30 +/* 06686C 80065C6C 00403025 */ move $a2, $v0 +/* 066870 80065C70 8FA20024 */ lw $v0, 0x24($sp) +/* 066874 80065C74 8FB80020 */ lw $t8, 0x20($sp) +/* 066878 80065C78 8C480008 */ lw $t0, 8($v0) +/* 06687C 80065C7C 8F19001C */ lw $t9, 0x1c($t8) +/* 066880 80065C80 8D0900D8 */ lw $t1, 0xd8($t0) +/* 066884 80065C84 240B000D */ li $t3, 13 +/* 066888 80065C88 03295021 */ addu $t2, $t9, $t1 +/* 06688C 80065C8C ACCA0004 */ sw $t2, 4($a2) +/* 066890 80065C90 ACC00000 */ sw $zero, ($a2) +/* 066894 80065C94 A4CB0008 */ sh $t3, 8($a2) +/* 066898 80065C98 844C001A */ lh $t4, 0x1a($v0) +/* 06689C 80065C9C 00000000 */ nop +/* 0668A0 80065CA0 A4CC000A */ sh $t4, 0xa($a2) +/* 0668A4 80065CA4 93A40037 */ lbu $a0, 0x37($sp) +/* 0668A8 80065CA8 0C0196FB */ jal func_80065BEC +/* 0668AC 80065CAC AFA6001C */ sw $a2, 0x1c($sp) +/* 0668B0 80065CB0 8FA6001C */ lw $a2, 0x1c($sp) +/* 0668B4 80065CB4 00000000 */ nop +/* 0668B8 80065CB8 A0C20012 */ sb $v0, 0x12($a2) +/* 0668BC 80065CBC 87AD0032 */ lh $t5, 0x32($sp) +/* 0668C0 80065CC0 00000000 */ nop +/* 0668C4 80065CC4 A4CD0010 */ sh $t5, 0x10($a2) +/* 0668C8 80065CC8 93AE003B */ lbu $t6, 0x3b($sp) +/* 0668CC 80065CCC 00000000 */ nop +/* 0668D0 80065CD0 A0CE0013 */ sb $t6, 0x13($a2) +/* 0668D4 80065CD4 C7A4002C */ lwc1 $f4, 0x2c($sp) +/* 0668D8 80065CD8 00000000 */ nop +/* 0668DC 80065CDC E4C4000C */ swc1 $f4, 0xc($a2) +/* 0668E0 80065CE0 8FA5003C */ lw $a1, 0x3c($sp) +/* 0668E4 80065CE4 8FA40020 */ lw $a0, 0x20($sp) +/* 0668E8 80065CE8 0C0195F1 */ jal func_800657C4 +/* 0668EC 80065CEC 00000000 */ nop +/* 0668F0 80065CF0 8FA6001C */ lw $a2, 0x1c($sp) +/* 0668F4 80065CF4 24050003 */ li $a1, 3 +/* 0668F8 80065CF8 ACC20014 */ sw $v0, 0x14($a2) +/* 0668FC 80065CFC 8FAF0028 */ lw $t7, 0x28($sp) +/* 066900 80065D00 00000000 */ nop +/* 066904 80065D04 ACCF0018 */ sw $t7, 0x18($a2) +/* 066908 80065D08 8FB80024 */ lw $t8, 0x24($sp) +/* 06690C 80065D0C 00000000 */ nop +/* 066910 80065D10 8F080008 */ lw $t0, 8($t8) +/* 066914 80065D14 00000000 */ nop +/* 066918 80065D18 8D04000C */ lw $a0, 0xc($t0) +/* 06691C 80065D1C 00000000 */ nop +/* 066920 80065D20 8C990008 */ lw $t9, 8($a0) +/* 066924 80065D24 00000000 */ nop +/* 066928 80065D28 0320F809 */ jalr $t9 +/* 06692C 80065D2C 00000000 */ nop +.L80065D30: +/* 066930 80065D30 8FBF0014 */ lw $ra, 0x14($sp) +.L80065D34: +/* 066934 80065D34 27BD0020 */ addiu $sp, $sp, 0x20 +/* 066938 80065D38 03E00008 */ jr $ra +/* 06693C 80065D3C 00000000 */ nop diff --git a/asm/unknown_064800.s b/asm/unknown_064800.s index 9812cbf6..87642770 100755 --- a/asm/unknown_064800.s +++ b/asm/unknown_064800.s @@ -17,6 +17,5 @@ glabel func_80063C00 /* 06481C 80063C1C 93220010 */ lbu $v0, 0x10($t9) /* 064820 80063C20 03E00008 */ jr $ra /* 064824 80063C24 00000000 */ nop - /* 064828 80063C28 00000000 */ nop /* 06482C 80063C2C 00000000 */ nop diff --git a/asm/unknown_0667D0.s b/asm/unknown_0667D0.s index a7b369bd..972a86f6 100755 --- a/asm/unknown_0667D0.s +++ b/asm/unknown_0667D0.s @@ -6,112 +6,110 @@ .set noreorder # dont insert nops after branches .set gp=64 # 64-bit instructions are used -glabel func_80065BD0 -/* 0667D0 80065BD0 3C01800E */ lui $at, 0x800e -/* 0667D4 80065BD4 03E00008 */ jr $ra -/* 0667D8 80065BD8 A424D050 */ sh $a0, -0x2fb0($at) +#glabel func_80065BD0 +#/* 0667D0 80065BD0 3C01800E */ lui $at, 0x800e +#/* 0667D4 80065BD4 03E00008 */ jr $ra +#/* 0667D8 80065BD8 A424D050 */ sh $a0, -0x2fb0($at) +# +#/* 0667DC 80065BDC 3C02800E */ lui $v0, %hi(D_800DD050) # $v0, 0x800e +#/* 0667E0 80065BE0 8442D050 */ lh $v0, %lo(D_800DD050)($v0) +#/* 0667E4 80065BE4 03E00008 */ jr $ra +#/* 0667E8 80065BE8 00000000 */ nop -/* 0667DC 80065BDC 3C02800E */ lui $v0, %hi(D_800DD050) # $v0, 0x800e -/* 0667E0 80065BE0 8442D050 */ lh $v0, %lo(D_800DD050)($v0) -/* 0667E4 80065BE4 03E00008 */ jr $ra -/* 0667E8 80065BE8 00000000 */ nop +#glabel func_80065BEC +#/* 0667EC 80065BEC 3C02800E */ lui $v0, %hi(D_800DD050) # $v0, 0x800e +#/* 0667F0 80065BF0 8442D050 */ lh $v0, %lo(D_800DD050)($v0) +#/* 0667F4 80065BF4 24010001 */ li $at, 1 +#/* 0667F8 80065BF8 10400007 */ beqz $v0, .L80065C18 +#/* 0667FC 80065BFC 00000000 */ nop +#/* 066800 80065C00 1041000A */ beq $v0, $at, .L80065C2C +#/* 066804 80065C04 24010002 */ li $at, 2 +#/* 066808 80065C08 10410005 */ beq $v0, $at, .L80065C20 +#/* 06680C 80065C0C 2484FFC0 */ addiu $a0, $a0, -0x40 +#/* 066810 80065C10 10000007 */ b .L80065C30 +#/* 066814 80065C14 24020040 */ li $v0, 64 +#.L80065C18: +#/* 066818 80065C18 03E00008 */ jr $ra +#/* 06681C 80065C1C 00801025 */ move $v0, $a0 +#.L80065C20: +#/* 066820 80065C20 00047043 */ sra $t6, $a0, 1 +#/* 066824 80065C24 03E00008 */ jr $ra +#/* 066828 80065C28 25C20040 */ addiu $v0, $t6, 0x40 +#.L80065C2C: +#/* 06682C 80065C2C 24020040 */ li $v0, 64 +#.L80065C30: +#/* 066830 80065C30 03E00008 */ jr $ra +#/* 066834 80065C34 00000000 */ nop -glabel func_80065BEC -/* 0667EC 80065BEC 3C02800E */ lui $v0, %hi(D_800DD050) # $v0, 0x800e -/* 0667F0 80065BF0 8442D050 */ lh $v0, %lo(D_800DD050)($v0) -/* 0667F4 80065BF4 24010001 */ li $at, 1 -/* 0667F8 80065BF8 10400007 */ beqz $v0, .L80065C18 -/* 0667FC 80065BFC 00000000 */ nop -/* 066800 80065C00 1041000A */ beq $v0, $at, .L80065C2C -/* 066804 80065C04 24010002 */ li $at, 2 -/* 066808 80065C08 10410005 */ beq $v0, $at, .L80065C20 -/* 06680C 80065C0C 2484FFC0 */ addiu $a0, $a0, -0x40 -/* 066810 80065C10 10000007 */ b .L80065C30 -/* 066814 80065C14 24020040 */ li $v0, 64 -.L80065C18: -/* 066818 80065C18 03E00008 */ jr $ra -/* 06681C 80065C1C 00801025 */ move $v0, $a0 - -.L80065C20: -/* 066820 80065C20 00047043 */ sra $t6, $a0, 1 -/* 066824 80065C24 03E00008 */ jr $ra -/* 066828 80065C28 25C20040 */ addiu $v0, $t6, 0x40 - -.L80065C2C: -/* 06682C 80065C2C 24020040 */ li $v0, 64 -.L80065C30: -/* 066830 80065C30 03E00008 */ jr $ra -/* 066834 80065C34 00000000 */ nop - -glabel func_80065C38 -/* 066838 80065C38 27BDFFE0 */ addiu $sp, $sp, -0x20 -/* 06683C 80065C3C AFBF0014 */ sw $ra, 0x14($sp) -/* 066840 80065C40 AFA40020 */ sw $a0, 0x20($sp) -/* 066844 80065C44 AFA50024 */ sw $a1, 0x24($sp) -/* 066848 80065C48 AFA60028 */ sw $a2, 0x28($sp) -/* 06684C 80065C4C AFA7002C */ sw $a3, 0x2c($sp) -/* 066850 80065C50 8CAF0008 */ lw $t7, 8($a1) -/* 066854 80065C54 00000000 */ nop -/* 066858 80065C58 11E00036 */ beqz $t7, .L80065D34 -/* 06685C 80065C5C 8FBF0014 */ lw $ra, 0x14($sp) -/* 066860 80065C60 0C01959A */ jal func_80065668 -/* 066864 80065C64 00000000 */ nop -/* 066868 80065C68 10400031 */ beqz $v0, .L80065D30 -/* 06686C 80065C6C 00403025 */ move $a2, $v0 -/* 066870 80065C70 8FA20024 */ lw $v0, 0x24($sp) -/* 066874 80065C74 8FB80020 */ lw $t8, 0x20($sp) -/* 066878 80065C78 8C480008 */ lw $t0, 8($v0) -/* 06687C 80065C7C 8F19001C */ lw $t9, 0x1c($t8) -/* 066880 80065C80 8D0900D8 */ lw $t1, 0xd8($t0) -/* 066884 80065C84 240B000D */ li $t3, 13 -/* 066888 80065C88 03295021 */ addu $t2, $t9, $t1 -/* 06688C 80065C8C ACCA0004 */ sw $t2, 4($a2) -/* 066890 80065C90 ACC00000 */ sw $zero, ($a2) -/* 066894 80065C94 A4CB0008 */ sh $t3, 8($a2) -/* 066898 80065C98 844C001A */ lh $t4, 0x1a($v0) -/* 06689C 80065C9C 00000000 */ nop -/* 0668A0 80065CA0 A4CC000A */ sh $t4, 0xa($a2) -/* 0668A4 80065CA4 93A40037 */ lbu $a0, 0x37($sp) -/* 0668A8 80065CA8 0C0196FB */ jal func_80065BEC -/* 0668AC 80065CAC AFA6001C */ sw $a2, 0x1c($sp) -/* 0668B0 80065CB0 8FA6001C */ lw $a2, 0x1c($sp) -/* 0668B4 80065CB4 00000000 */ nop -/* 0668B8 80065CB8 A0C20012 */ sb $v0, 0x12($a2) -/* 0668BC 80065CBC 87AD0032 */ lh $t5, 0x32($sp) -/* 0668C0 80065CC0 00000000 */ nop -/* 0668C4 80065CC4 A4CD0010 */ sh $t5, 0x10($a2) -/* 0668C8 80065CC8 93AE003B */ lbu $t6, 0x3b($sp) -/* 0668CC 80065CCC 00000000 */ nop -/* 0668D0 80065CD0 A0CE0013 */ sb $t6, 0x13($a2) -/* 0668D4 80065CD4 C7A4002C */ lwc1 $f4, 0x2c($sp) -/* 0668D8 80065CD8 00000000 */ nop -/* 0668DC 80065CDC E4C4000C */ swc1 $f4, 0xc($a2) -/* 0668E0 80065CE0 8FA5003C */ lw $a1, 0x3c($sp) -/* 0668E4 80065CE4 8FA40020 */ lw $a0, 0x20($sp) -/* 0668E8 80065CE8 0C0195F1 */ jal func_800657C4 -/* 0668EC 80065CEC 00000000 */ nop -/* 0668F0 80065CF0 8FA6001C */ lw $a2, 0x1c($sp) -/* 0668F4 80065CF4 24050003 */ li $a1, 3 -/* 0668F8 80065CF8 ACC20014 */ sw $v0, 0x14($a2) -/* 0668FC 80065CFC 8FAF0028 */ lw $t7, 0x28($sp) -/* 066900 80065D00 00000000 */ nop -/* 066904 80065D04 ACCF0018 */ sw $t7, 0x18($a2) -/* 066908 80065D08 8FB80024 */ lw $t8, 0x24($sp) -/* 06690C 80065D0C 00000000 */ nop -/* 066910 80065D10 8F080008 */ lw $t0, 8($t8) -/* 066914 80065D14 00000000 */ nop -/* 066918 80065D18 8D04000C */ lw $a0, 0xc($t0) -/* 06691C 80065D1C 00000000 */ nop -/* 066920 80065D20 8C990008 */ lw $t9, 8($a0) -/* 066924 80065D24 00000000 */ nop -/* 066928 80065D28 0320F809 */ jalr $t9 -/* 06692C 80065D2C 00000000 */ nop -.L80065D30: -/* 066930 80065D30 8FBF0014 */ lw $ra, 0x14($sp) -.L80065D34: -/* 066934 80065D34 27BD0020 */ addiu $sp, $sp, 0x20 -/* 066938 80065D38 03E00008 */ jr $ra -/* 06693C 80065D3C 00000000 */ nop +#glabel func_80065C38 +#/* 066838 80065C38 27BDFFE0 */ addiu $sp, $sp, -0x20 +#/* 06683C 80065C3C AFBF0014 */ sw $ra, 0x14($sp) +#/* 066840 80065C40 AFA40020 */ sw $a0, 0x20($sp) +#/* 066844 80065C44 AFA50024 */ sw $a1, 0x24($sp) +#/* 066848 80065C48 AFA60028 */ sw $a2, 0x28($sp) +#/* 06684C 80065C4C AFA7002C */ sw $a3, 0x2c($sp) +#/* 066850 80065C50 8CAF0008 */ lw $t7, 8($a1) +#/* 066854 80065C54 00000000 */ nop +#/* 066858 80065C58 11E00036 */ beqz $t7, .L80065D34 +#/* 06685C 80065C5C 8FBF0014 */ lw $ra, 0x14($sp) +#/* 066860 80065C60 0C01959A */ jal func_80065668 +#/* 066864 80065C64 00000000 */ nop +#/* 066868 80065C68 10400031 */ beqz $v0, .L80065D30 +#/* 06686C 80065C6C 00403025 */ move $a2, $v0 +#/* 066870 80065C70 8FA20024 */ lw $v0, 0x24($sp) +#/* 066874 80065C74 8FB80020 */ lw $t8, 0x20($sp) +#/* 066878 80065C78 8C480008 */ lw $t0, 8($v0) +#/* 06687C 80065C7C 8F19001C */ lw $t9, 0x1c($t8) +#/* 066880 80065C80 8D0900D8 */ lw $t1, 0xd8($t0) +#/* 066884 80065C84 240B000D */ li $t3, 13 +#/* 066888 80065C88 03295021 */ addu $t2, $t9, $t1 +#/* 06688C 80065C8C ACCA0004 */ sw $t2, 4($a2) +#/* 066890 80065C90 ACC00000 */ sw $zero, ($a2) +#/* 066894 80065C94 A4CB0008 */ sh $t3, 8($a2) +#/* 066898 80065C98 844C001A */ lh $t4, 0x1a($v0) +#/* 06689C 80065C9C 00000000 */ nop +#/* 0668A0 80065CA0 A4CC000A */ sh $t4, 0xa($a2) +#/* 0668A4 80065CA4 93A40037 */ lbu $a0, 0x37($sp) +#/* 0668A8 80065CA8 0C0196FB */ jal func_80065BEC +#/* 0668AC 80065CAC AFA6001C */ sw $a2, 0x1c($sp) +#/* 0668B0 80065CB0 8FA6001C */ lw $a2, 0x1c($sp) +#/* 0668B4 80065CB4 00000000 */ nop +#/* 0668B8 80065CB8 A0C20012 */ sb $v0, 0x12($a2) +#/* 0668BC 80065CBC 87AD0032 */ lh $t5, 0x32($sp) +#/* 0668C0 80065CC0 00000000 */ nop +#/* 0668C4 80065CC4 A4CD0010 */ sh $t5, 0x10($a2) +#/* 0668C8 80065CC8 93AE003B */ lbu $t6, 0x3b($sp) +#/* 0668CC 80065CCC 00000000 */ nop +#/* 0668D0 80065CD0 A0CE0013 */ sb $t6, 0x13($a2) +#/* 0668D4 80065CD4 C7A4002C */ lwc1 $f4, 0x2c($sp) +#/* 0668D8 80065CD8 00000000 */ nop +#/* 0668DC 80065CDC E4C4000C */ swc1 $f4, 0xc($a2) +#/* 0668E0 80065CE0 8FA5003C */ lw $a1, 0x3c($sp) +#/* 0668E4 80065CE4 8FA40020 */ lw $a0, 0x20($sp) +#/* 0668E8 80065CE8 0C0195F1 */ jal func_800657C4 +#/* 0668EC 80065CEC 00000000 */ nop +#/* 0668F0 80065CF0 8FA6001C */ lw $a2, 0x1c($sp) +#/* 0668F4 80065CF4 24050003 */ li $a1, 3 +#/* 0668F8 80065CF8 ACC20014 */ sw $v0, 0x14($a2) +#/* 0668FC 80065CFC 8FAF0028 */ lw $t7, 0x28($sp) +#/* 066900 80065D00 00000000 */ nop +#/* 066904 80065D04 ACCF0018 */ sw $t7, 0x18($a2) +#/* 066908 80065D08 8FB80024 */ lw $t8, 0x24($sp) +#/* 06690C 80065D0C 00000000 */ nop +#/* 066910 80065D10 8F080008 */ lw $t0, 8($t8) +#/* 066914 80065D14 00000000 */ nop +#/* 066918 80065D18 8D04000C */ lw $a0, 0xc($t0) +#/* 06691C 80065D1C 00000000 */ nop +#/* 066920 80065D20 8C990008 */ lw $t9, 8($a0) +#/* 066924 80065D24 00000000 */ nop +#/* 066928 80065D28 0320F809 */ jalr $t9 +#/* 06692C 80065D2C 00000000 */ nop +#.L80065D30: +#/* 066930 80065D30 8FBF0014 */ lw $ra, 0x14($sp) +#.L80065D34: +#/* 066934 80065D34 27BD0020 */ addiu $sp, $sp, 0x20 +#/* 066938 80065D38 03E00008 */ jr $ra +#/* 06693C 80065D3C 00000000 */ nop glabel D_80065D40 /* 066940 80065D40 27BDFFE0 */ addiu $sp, $sp, -0x20 diff --git a/asm/unknown_080500.s b/asm/unknown_080500.s index 5503ebd8..3c495dfd 100755 --- a/asm/unknown_080500.s +++ b/asm/unknown_080500.s @@ -34470,45 +34470,45 @@ glabel func_8009EC60 /* 09F868 8009EC68 03E00008 */ jr $ra /* 09F86C 8009EC6C 31E20001 */ andi $v0, $t7, 1 -glabel func_8009EC70 -/* 09F870 8009EC70 3C02800E */ lui $v0, %hi(D_800DF494) # $v0, 0x800e -/* 09F874 8009EC74 8C42F494 */ lw $v0, %lo(D_800DF494)($v0) -/* 09F878 8009EC78 03E00008 */ jr $ra -/* 09F87C 8009EC7C 00000000 */ nop - -glabel func_8009EC80 -/* 09F880 8009EC80 27BDFFE8 */ addiu $sp, $sp, -0x18 -/* 09F884 8009EC84 AFBF0014 */ sw $ra, 0x14($sp) -/* 09F888 8009EC88 0C0270B4 */ jal func_8009C2D0 -/* 09F88C 8009EC8C 00000000 */ nop -/* 09F890 8009EC90 8FBF0014 */ lw $ra, 0x14($sp) -/* 09F894 8009EC94 10400003 */ beqz $v0, .L8009ECA4 -/* 09F898 8009EC98 00000000 */ nop -/* 09F89C 8009EC9C 10000004 */ b .L8009ECB0 -/* 09F8A0 8009ECA0 00001025 */ move $v0, $zero -.L8009ECA4: -/* 09F8A4 8009ECA4 3C02800E */ lui $v0, %hi(D_800DF4C0) # $v0, 0x800e -/* 09F8A8 8009ECA8 8C42F4C0 */ lw $v0, %lo(D_800DF4C0)($v0) -/* 09F8AC 8009ECAC 00000000 */ nop -.L8009ECB0: -/* 09F8B0 8009ECB0 03E00008 */ jr $ra -/* 09F8B4 8009ECB4 27BD0018 */ addiu $sp, $sp, 0x18 - -glabel func_8009ECB8 -/* 09F8B8 8009ECB8 3C02800E */ lui $v0, %hi(D_800DFD98) # $v0, 0x800e -/* 09F8BC 8009ECBC 8C42FD98 */ lw $v0, %lo(D_800DFD98)($v0) -/* 09F8C0 8009ECC0 00000000 */ nop -/* 09F8C4 8009ECC4 304E0001 */ andi $t6, $v0, 1 -/* 09F8C8 8009ECC8 03E00008 */ jr $ra -/* 09F8CC 8009ECCC 01C01025 */ move $v0, $t6 - -glabel func_8009ECD0 -/* 09F8D0 8009ECD0 3C02800E */ lui $v0, %hi(D_800DFD98) # $v0, 0x800e -/* 09F8D4 8009ECD4 8C42FD98 */ lw $v0, %lo(D_800DFD98)($v0) -/* 09F8D8 8009ECD8 00000000 */ nop -/* 09F8DC 8009ECDC 304E0002 */ andi $t6, $v0, 2 -/* 09F8E0 8009ECE0 03E00008 */ jr $ra -/* 09F8E4 8009ECE4 01C01025 */ move $v0, $t6 - -/* 09F8E8 8009ECE8 00000000 */ nop -/* 09F8EC 8009ECEC 00000000 */ nop +#glabel func_8009EC70 +#/* 09F870 8009EC70 3C02800E */ lui $v0, %hi(D_800DF494) # $v0, 0x800e +#/* 09F874 8009EC74 8C42F494 */ lw $v0, %lo(D_800DF494)($v0) +#/* 09F878 8009EC78 03E00008 */ jr $ra +#/* 09F87C 8009EC7C 00000000 */ nop +# +#glabel func_8009EC80 +#/* 09F880 8009EC80 27BDFFE8 */ addiu $sp, $sp, -0x18 +#/* 09F884 8009EC84 AFBF0014 */ sw $ra, 0x14($sp) +#/* 09F888 8009EC88 0C0270B4 */ jal func_8009C2D0 +#/* 09F88C 8009EC8C 00000000 */ nop +#/* 09F890 8009EC90 8FBF0014 */ lw $ra, 0x14($sp) +#/* 09F894 8009EC94 10400003 */ beqz $v0, .L8009ECA4 +#/* 09F898 8009EC98 00000000 */ nop +#/* 09F89C 8009EC9C 10000004 */ b .L8009ECB0 +#/* 09F8A0 8009ECA0 00001025 */ move $v0, $zero +#.L8009ECA4: +#/* 09F8A4 8009ECA4 3C02800E */ lui $v0, %hi(D_800DF4C0) # $v0, 0x800e +#/* 09F8A8 8009ECA8 8C42F4C0 */ lw $v0, %lo(D_800DF4C0)($v0) +#/* 09F8AC 8009ECAC 00000000 */ nop +#.L8009ECB0: +#/* 09F8B0 8009ECB0 03E00008 */ jr $ra +#/* 09F8B4 8009ECB4 27BD0018 */ addiu $sp, $sp, 0x18 +# +#glabel func_8009ECB8 +#/* 09F8B8 8009ECB8 3C02800E */ lui $v0, %hi(D_800DFD98) # $v0, 0x800e +#/* 09F8BC 8009ECBC 8C42FD98 */ lw $v0, %lo(D_800DFD98)($v0) +#/* 09F8C0 8009ECC0 00000000 */ nop +#/* 09F8C4 8009ECC4 304E0001 */ andi $t6, $v0, 1 +#/* 09F8C8 8009ECC8 03E00008 */ jr $ra +#/* 09F8CC 8009ECCC 01C01025 */ move $v0, $t6 +# +#glabel func_8009ECD0 +#/* 09F8D0 8009ECD0 3C02800E */ lui $v0, %hi(D_800DFD98) # $v0, 0x800e +#/* 09F8D4 8009ECD4 8C42FD98 */ lw $v0, %lo(D_800DFD98)($v0) +#/* 09F8D8 8009ECD8 00000000 */ nop +#/* 09F8DC 8009ECDC 304E0002 */ andi $t6, $v0, 2 +#/* 09F8E0 8009ECE0 03E00008 */ jr $ra +#/* 09F8E4 8009ECE4 01C01025 */ move $v0, $t6 +# +#/* 09F8E8 8009ECE8 00000000 */ nop +#/* 09F8EC 8009ECEC 00000000 */ nop diff --git a/diff.py b/diff.py new file mode 100644 index 00000000..d9d39a1c --- /dev/null +++ b/diff.py @@ -0,0 +1,1012 @@ +#!/usr/bin/env python3 +import sys +import re +import os +import ast +import argparse +import subprocess +import difflib +import string +import itertools +import threading +import queue +import time + + +def fail(msg): + print(msg, file=sys.stderr) + sys.exit(1) + + +MISSING_PREREQUISITES = ( + "Missing prerequisite python module {}. " + "Run `python3 -m pip install --user colorama ansiwrap attrs watchdog python-Levenshtein` to install prerequisites (python-Levenshtein only needed for --algorithm=levenshtein)." +) + +try: + import attr + from colorama import Fore, Style, Back + import ansiwrap + import watchdog +except ModuleNotFoundError as e: + fail(MISSING_PREREQUISITES.format(e.name)) + +# Prefer to use diff_settings.py from the current working directory +sys.path.insert(0, ".") +try: + import diff_settings +except ModuleNotFoundError: + fail("Unable to find diff_settings.py in the same directory.") + +# ==== CONFIG ==== + +parser = argparse.ArgumentParser(description="Diff MIPS assembly.") +parser.add_argument("start", help="Function name or address to start diffing from.") +parser.add_argument("end", nargs="?", help="Address to end diff at.") +parser.add_argument( + "-o", + dest="diff_obj", + action="store_true", + help="Diff .o files rather than a whole binary. This makes it possible to see symbol names. (Recommended)", +) +parser.add_argument( + "--base-asm", + dest="base_asm", + metavar="FILE", + help="Read assembly from given file instead of configured base img.", +) +parser.add_argument( + "--write-asm", + dest="write_asm", + metavar="FILE", + help="Write the current assembly output to file, e.g. for use with --base-asm.", +) +parser.add_argument( + "-m", + "--make", + dest="make", + action="store_true", + help="Automatically run 'make' on the .o file or binary before diffing.", +) +parser.add_argument( + "-l", + "--skip-lines", + dest="skip_lines", + type=int, + default=0, + help="Skip the first N lines of output.", +) +parser.add_argument( + "-f", + "--stop-jr-ra", + dest="stop_jrra", + action="store_true", + help="Stop disassembling at the first 'jr ra'. Some functions have multiple return points, so use with care!", +) +parser.add_argument( + "-i", + "--ignore-large-imms", + dest="ignore_large_imms", + action="store_true", + help="Pretend all large enough immediates are the same.", +) +parser.add_argument( + "-B", + "--no-show-branches", + dest="show_branches", + action="store_false", + help="Don't visualize branches/branch targets.", +) +parser.add_argument( + "-S", + "--base-shift", + dest="base_shift", + type=str, + default="0", + help="Diff position X in our img against position X + shift in the base img. " + 'Arithmetic is allowed, so e.g. |-S "0x1234 - 0x4321"| is a reasonable ' + "flag to pass if it is known that position 0x1234 in the base img syncs " + "up with position 0x4321 in our img. Not supported together with -o.", +) +parser.add_argument( + "-w", + "--watch", + dest="watch", + action="store_true", + help="Automatically update when source/object files change. " + "Recommended in combination with -m.", +) +parser.add_argument( + "--width", + dest="column_width", + type=int, + default=50, + help="Sets the width of the left and right view column.", +) +parser.add_argument( + "--algorithm", + dest="algorithm", + default="difflib", + choices=["levenshtein", "difflib"], + help="Diff algorithm to use.", +) + +parser.add_argument( + "--max-size", + "--max-lines", + dest="max_lines", + type=int, + default=1024, + help="The maximum length of the diff, in lines. Not recommended when -f is used.", +) + +# Project-specific flags, e.g. different versions/make arguments. +if hasattr(diff_settings, "add_custom_arguments"): + diff_settings.add_custom_arguments(parser) + +args = parser.parse_args() + +# Set imgs, map file and make flags in a project-specific manner. +config = {} +diff_settings.apply(config, args) + +baseimg = config.get("baseimg", None) +myimg = config.get("myimg", None) +mapfile = config.get("mapfile", None) +makeflags = config.get("makeflags", []) +source_directories = config.get("source_directories", None) + +MAX_FUNCTION_SIZE_LINES = args.max_lines +MAX_FUNCTION_SIZE_BYTES = MAX_FUNCTION_SIZE_LINES * 4 + +COLOR_ROTATION = [ + Fore.MAGENTA, + Fore.CYAN, + Fore.GREEN, + Fore.RED, + Fore.LIGHTYELLOW_EX, + Fore.LIGHTMAGENTA_EX, + Fore.LIGHTCYAN_EX, + Fore.LIGHTGREEN_EX, + Fore.LIGHTBLACK_EX, +] + +BUFFER_CMD = ["tail", "-c", str(10 ** 9)] +LESS_CMD = ["less", "-Ric"] + +DEBOUNCE_DELAY = 0.1 +FS_WATCH_EXTENSIONS = [".c", ".h"] + +# ==== LOGIC ==== + +if args.algorithm == "levenshtein": + try: + import Levenshtein + except ModuleNotFoundError as e: + fail(MISSING_PREREQUISITES.format(e.name)) + +binutils_prefix = None + +for binutils_cand in ["mips-linux-gnu-", "mips64-elf-"]: + try: + subprocess.check_call( + [binutils_cand + "objdump", "--version"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + binutils_prefix = binutils_cand + break + except subprocess.CalledProcessError: + pass + except FileNotFoundError: + pass + +if not binutils_prefix: + fail( + "Missing binutils; please ensure mips-linux-gnu-objdump or mips64-elf-objdump exist." + ) + + +def eval_int(expr, emsg=None): + try: + ret = ast.literal_eval(expr) + if not isinstance(ret, int): + raise Exception("not an integer") + return ret + except Exception: + if emsg is not None: + fail(emsg) + return None + + +def run_make(target, capture_output=False): + if capture_output: + return subprocess.run( + ["make"] + makeflags + [target], + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + else: + subprocess.check_call(["make"] + makeflags + [target]) + + +def restrict_to_function(dump, fn_name): + out = [] + search = f"<{fn_name}>:" + found = False + for line in dump.split("\n"): + if found: + if len(out) >= MAX_FUNCTION_SIZE_LINES: + break + out.append(line) + elif search in line: + found = True + return "\n".join(out) + + +def run_objdump(cmd): + flags, target, restrict = cmd + out = subprocess.check_output( + [binutils_prefix + "objdump"] + flags + [target], universal_newlines=True + ) + if restrict is not None: + return restrict_to_function(out, restrict) + return out + + +base_shift = eval_int( + args.base_shift, "Failed to parse --base-shift (-S) argument as an integer." +) + + +def search_map_file(fn_name): + if not mapfile: + fail(f"No map file configured; cannot find function {fn_name}.") + + try: + with open(mapfile) as f: + lines = f.read().split("\n") + except Exception: + fail(f"Failed to open map file {mapfile} for reading.") + + try: + cur_objfile = None + ram_to_rom = None + cands = [] + last_line = "" + for line in lines: + if line.startswith(" .text"): + cur_objfile = line.split()[3] + if "load address" in line: + tokens = last_line.split() + line.split() + ram = int(tokens[1], 0) + rom = int(tokens[5], 0) + ram_to_rom = rom - ram + if line.endswith(" " + fn_name): + ram = int(line.split()[0], 0) + if cur_objfile is not None and ram_to_rom is not None: + cands.append((cur_objfile, ram + ram_to_rom)) + last_line = line + except Exception as e: + import traceback + + traceback.print_exc() + fail(f"Internal error while parsing map file") + + if len(cands) > 1: + fail(f"Found multiple occurrences of function {fn_name} in map file.") + if len(cands) == 1: + return cands[0] + return None, None + + +def dump_objfile(): + if base_shift: + fail("--base-shift not compatible with -o") + if args.end is not None: + fail("end address not supported together with -o") + if args.start.startswith("0"): + fail("numerical start address not supported with -o; pass a function name") + + objfile, _ = search_map_file(args.start) + if not objfile: + fail("Not able to find .o file for function.") + + if args.make: + run_make(objfile) + + if not os.path.isfile(objfile): + fail(f"Not able to find .o file for function: {objfile} is not a file.") + + refobjfile = "expected/" + objfile + if not os.path.isfile(refobjfile): + fail(f'Please ensure an OK .o file exists at "{refobjfile}".') + + objdump_flags = ["-drz"] + return ( + objfile, + (objdump_flags, refobjfile, args.start), + (objdump_flags, objfile, args.start), + ) + + +def dump_binary(): + if not baseimg or not myimg: + fail("Missing myimg/baseimg in config.") + if args.make: + run_make(myimg) + start_addr = eval_int(args.start) + if start_addr is None: + _, start_addr = search_map_file(args.start) + if start_addr is None: + fail("Not able to find function in map file.") + if args.end is not None: + end_addr = eval_int(args.end, "End address must be an integer expression.") + else: + end_addr = start_addr + MAX_FUNCTION_SIZE_BYTES + objdump_flags = ["-Dz", "-bbinary", "-mmips", "-EB"] + flags1 = [ + f"--start-address={start_addr + base_shift}", + f"--stop-address={end_addr + base_shift}", + ] + flags2 = [f"--start-address={start_addr}", f"--stop-address={end_addr}"] + return ( + myimg, + (objdump_flags + flags1, baseimg, None), + (objdump_flags + flags2, myimg, None), + ) + + +# Alignment with ANSI colors is broken, let's fix it. +def ansi_ljust(s, width): + needed = width - ansiwrap.ansilen(s) + if needed > 0: + return s + " " * needed + else: + return s + + +re_int = re.compile(r"[0-9]+") +re_comments = re.compile(r"<.*?>") +re_regs = re.compile(r"\$?\b(a[0-3]|t[0-9]|s[0-8]|at|v[01]|f[12]?[0-9]|f3[01]|fp)\b") +re_sprel = re.compile(r",([0-9]+|0x[0-9a-f]+)\(sp\)") +re_large_imm = re.compile(r"-?[1-9][0-9]{2,}|-?0x[0-9a-f]{3,}") +re_imm = re.compile(r"(\b|-)([0-9]+|0x[0-9a-fA-F]+)\b(?!\(sp)|%(lo|hi)\([^)]*\)") +forbidden = set(string.ascii_letters + "_") +branch_likely_instructions = { + "beql", + "bnel", + "beqzl", + "bnezl", + "bgezl", + "bgtzl", + "blezl", + "bltzl", + "bc1tl", + "bc1fl", +} +branch_instructions = branch_likely_instructions.union( + {"b", "beq", "bne", "beqz", "bnez", "bgez", "bgtz", "blez", "bltz", "bc1t", "bc1f"} +) +jump_instructions = branch_instructions.union({"jal", "j"}) + + +def hexify_int(row, pat): + full = pat.group(0) + if len(full) <= 1: + # leave one-digit ints alone + return full + start, end = pat.span() + if start and row[start - 1] in forbidden: + return full + if end < len(row) and row[end] in forbidden: + return full + return hex(int(full)) + + +def parse_relocated_line(line): + try: + ind2 = line.rindex(",") + except ValueError: + ind2 = line.rindex("\t") + before = line[: ind2 + 1] + after = line[ind2 + 1 :] + ind2 = after.find("(") + if ind2 == -1: + imm, after = after, "" + else: + imm, after = after[:ind2], after[ind2:] + if imm == "0x0": + imm = "0" + return before, imm, after + + +def process_reloc(row, prev): + before, imm, after = parse_relocated_line(prev) + repl = row.split()[-1] + if imm != "0": + if before.strip() == "jal" and not imm.startswith("0x"): + imm = "0x" + imm + repl += "+" + imm if int(imm, 0) > 0 else imm + if "R_MIPS_LO16" in row: + repl = f"%lo({repl})" + elif "R_MIPS_HI16" in row: + # Ideally we'd pair up R_MIPS_LO16 and R_MIPS_HI16 to generate a + # correct addend for each, but objdump doesn't give us the order of + # the relocations, so we can't find the right LO16. :( + repl = f"%hi({repl})" + else: + assert "R_MIPS_26" in row, f"unknown relocation type '{row}'" + return before + repl + after + + +def process(lines): + mnemonics = [] + diff_rows = [] + rows_with_imms = [] + skip_next = False + originals = [] + line_nums = [] + branch_targets = [] + if not args.diff_obj: + lines = lines[7:] + if lines and not lines[-1]: + lines.pop() + + for row in lines: + if args.diff_obj and (">:" in row or not row): + continue + + if "R_MIPS_" in row: + # N.B. Don't transform the diff rows, they already ignore immediates + # if diff_rows[-1] != '': + # diff_rows[-1] = process_reloc(row, rows_with_imms[-1]) + originals[-1] = process_reloc(row, originals[-1]) + continue + + row = re.sub(re_comments, "", row) + row = row.rstrip() + tabs = row.split("\t") + row = "\t".join(tabs[2:]) + line_num = tabs[0].strip() + row_parts = row.split("\t", 1) + mnemonic = row_parts[0].strip() + if mnemonic not in jump_instructions: + row = re.sub(re_int, lambda s: hexify_int(row, s), row) + original = row + if skip_next: + skip_next = False + row = "" + mnemonic = "" + if mnemonic in branch_likely_instructions: + skip_next = True + row = re.sub(re_regs, "", row) + row = re.sub(re_sprel, ",addr(sp)", row) + row_with_imm = row + if mnemonic in jump_instructions: + row = row.strip() + row, _ = split_off_branch(row) + row += "" + else: + row = re.sub(re_imm, "", row) + + mnemonics.append(mnemonic) + rows_with_imms.append(row_with_imm) + diff_rows.append(row) + originals.append(original) + line_nums.append(line_num) + if mnemonic in branch_instructions: + target = row_parts[1].strip().split(",")[-1] + if mnemonic in branch_likely_instructions: + target = hex(int(target, 16) - 4)[2:] + branch_targets.append(target) + else: + branch_targets.append(None) + if args.stop_jrra and mnemonic == "jr" and row_parts[1].strip() == "ra": + break + + # Cleanup whitespace + originals = [original.strip() for original in originals] + originals = [ + "".join(f"{o:<8s}" for o in original.split("\t")) for original in originals + ] + # return diff_rows, diff_rows, line_nums + return mnemonics, diff_rows, originals, line_nums, branch_targets + + +def format_single_line_diff(line1, line2, column_width): + return f"{ansi_ljust(line1,column_width)}{ansi_ljust(line2,column_width)}" + + +class SymbolColorer: + def __init__(self, base_index): + self.color_index = base_index + self.symbol_colors = {} + + def color_symbol(self, s, t=None): + try: + color = self.symbol_colors[s] + except: + color = COLOR_ROTATION[self.color_index % len(COLOR_ROTATION)] + self.color_index += 1 + self.symbol_colors[s] = color + t = t or s + return f"{color}{t}{Fore.RESET}" + + +def maybe_normalize_large_imms(row): + if args.ignore_large_imms: + row = re.sub(re_large_imm, "", row) + return row + + +def normalize_imms(row): + return re.sub(re_imm, "", row) + + +def normalize_stack(row): + return re.sub(re_sprel, ",addr(sp)", row) + + +def split_off_branch(line): + parts = line.split(",") + if len(parts) < 2: + parts = line.split(None, 1) + off = len(line) - len(parts[-1]) + return line[:off], line[off:] + + +def color_imms(out1, out2): + g1 = [] + g2 = [] + re.sub(re_imm, lambda s: g1.append(s.group()), out1) + re.sub(re_imm, lambda s: g2.append(s.group()), out2) + if len(g1) == len(g2): + diffs = [x != y for (x, y) in zip(g1, g2)] + it = iter(diffs) + + def maybe_color(s): + return f"{Fore.LIGHTBLUE_EX}{s}{Style.RESET_ALL}" if next(it) else s + + out1 = re.sub(re_imm, lambda s: maybe_color(s.group()), out1) + it = iter(diffs) + out2 = re.sub(re_imm, lambda s: maybe_color(s.group()), out2) + return out1, out2 + + +def color_branch_imms(br1, br2): + if br1 != br2: + br1 = f"{Fore.LIGHTBLUE_EX}{br1}{Style.RESET_ALL}" + br2 = f"{Fore.LIGHTBLUE_EX}{br2}{Style.RESET_ALL}" + return br1, br2 + + +def diff_sequences_difflib(seq1, seq2): + differ = difflib.SequenceMatcher(a=seq1, b=seq2, autojunk=False) + return differ.get_opcodes() + + +def diff_sequences(seq1, seq2): + if ( + args.algorithm != "levenshtein" + or len(seq1) * len(seq2) > 4 * 10 ** 8 + or len(seq1) + len(seq2) >= 0x110000 + ): + return diff_sequences_difflib(seq1, seq2) + + # The Levenshtein library assumes that we compare strings, not lists. Convert. + # (Per the check above we know we have fewer than 0x110000 unique elements, so chr() works.) + remapping = {} + + def remap(seq): + seq = seq[:] + for i in range(len(seq)): + val = remapping.get(seq[i]) + if val is None: + val = chr(len(remapping)) + remapping[seq[i]] = val + seq[i] = val + return "".join(seq) + + seq1 = remap(seq1) + seq2 = remap(seq2) + return Levenshtein.opcodes(seq1, seq2) + + +def do_diff(basedump, mydump): + asm_lines1 = basedump.split("\n") + asm_lines2 = mydump.split("\n") + + output = [] + + # TODO: status line? + # output.append(sha1sum(mydump)) + + mnemonics1, asm_lines1, originals1, line_nums1, branch_targets1 = process( + asm_lines1 + ) + mnemonics2, asm_lines2, originals2, line_nums2, branch_targets2 = process( + asm_lines2 + ) + + sc1 = SymbolColorer(0) + sc2 = SymbolColorer(0) + sc3 = SymbolColorer(4) + sc4 = SymbolColorer(4) + sc5 = SymbolColorer(0) + sc6 = SymbolColorer(0) + bts1 = set() + bts2 = set() + + if args.show_branches: + for (bts, btset, sc) in [ + (branch_targets1, bts1, sc5), + (branch_targets2, bts2, sc6), + ]: + for bt in bts: + if bt is not None: + btset.add(bt + ":") + sc.color_symbol(bt + ":") + + for (tag, i1, i2, j1, j2) in diff_sequences(mnemonics1, mnemonics2): + lines1 = asm_lines1[i1:i2] + lines2 = asm_lines2[j1:j2] + + for k, (line1, line2) in enumerate(itertools.zip_longest(lines1, lines2)): + if tag == "replace": + if line1 is None: + tag = "insert" + elif line2 is None: + tag = "delete" + + try: + original1 = originals1[i1 + k] + line_num1 = line_nums1[i1 + k] + except: + original1 = "" + line_num1 = "" + try: + original2 = originals2[j1 + k] + line_num2 = line_nums2[j1 + k] + except: + original2 = "" + line_num2 = "" + + has1 = has2 = True + line_color1 = line_color2 = sym_color = Fore.RESET + line_prefix = " " + if line1 == line2: + if not line1: + has1 = has2 = False + if maybe_normalize_large_imms(original1) == maybe_normalize_large_imms( + original2 + ): + out1 = original1 + out2 = original2 + elif line1 == "": + out1 = f"{Style.DIM}{original1}" + out2 = f"{Style.DIM}{original2}" + else: + mnemonic = original1.split()[0] + out1, out2 = original1, original2 + branch1 = branch2 = "" + if mnemonic in jump_instructions: + out1, branch1 = split_off_branch(original1) + out2, branch2 = split_off_branch(original2) + branchless1 = out1 + branchless2 = out2 + out1, out2 = color_imms(out1, out2) + branch1, branch2 = color_branch_imms(branch1, branch2) + out1 += branch1 + out2 += branch2 + if normalize_imms(branchless1) == normalize_imms(branchless2): + # only imms differences + sym_color = Fore.LIGHTBLUE_EX + line_prefix = "i" + else: + out1 = re.sub( + re_sprel, + lambda s: "," + sc3.color_symbol(s.group()[1:]), + out1, + ) + out2 = re.sub( + re_sprel, + lambda s: "," + sc4.color_symbol(s.group()[1:]), + out2, + ) + if normalize_stack(branchless1) == normalize_stack(branchless2): + # only stack differences (luckily stack and imm + # differences can't be combined in MIPS, so we + # don't have to think about that case) + sym_color = Fore.YELLOW + line_prefix = "s" + else: + # regs differences and maybe imms as well + out1 = re.sub( + re_regs, lambda s: sc1.color_symbol(s.group()), out1 + ) + out2 = re.sub( + re_regs, lambda s: sc2.color_symbol(s.group()), out2 + ) + line_color1 = line_color2 = sym_color = Fore.YELLOW + line_prefix = "r" + elif tag in ["replace", "equal"]: + line_prefix = "|" + line_color1 = Fore.LIGHTBLUE_EX + line_color2 = Fore.LIGHTBLUE_EX + sym_color = Fore.LIGHTBLUE_EX + out1 = original1 + out2 = original2 + elif tag == "delete": + line_prefix = "<" + line_color1 = line_color2 = sym_color = Fore.RED + has2 = False + out1 = original1 + out2 = "" + elif tag == "insert": + line_prefix = ">" + line_color1 = line_color2 = sym_color = Fore.GREEN + has1 = False + out1 = "" + out2 = original2 + + in_arrow1 = " " + in_arrow2 = " " + out_arrow1 = "" + out_arrow2 = "" + line_num1 = line_num1 if has1 else "" + line_num2 = line_num2 if has2 else "" + + if sym_color == line_color2: + line_color2 = "" + + if args.show_branches and has1: + if line_num1 in bts1: + in_arrow1 = sc5.color_symbol(line_num1, "~>") + line_color1 + if branch_targets1[i1 + k] is not None: + out_arrow1 = " " + sc5.color_symbol( + branch_targets1[i1 + k] + ":", "~>" + ) + if args.show_branches and has2: + if line_num2 in bts2: + in_arrow2 = sc6.color_symbol(line_num2, "~>") + line_color2 + if branch_targets2[j1 + k] is not None: + out_arrow2 = " " + sc6.color_symbol( + branch_targets2[j1 + k] + ":", "~>" + ) + + out1 = f"{line_color1}{line_num1} {in_arrow1} {out1}{Style.RESET_ALL}{out_arrow1}" + out2 = f"{line_color2}{line_num2} {in_arrow2} {out2}{Style.RESET_ALL}{out_arrow2}" + mid = f"{sym_color}{line_prefix} " + output.append(format_single_line_diff(out1, mid + out2, args.column_width)) + + return output[args.skip_lines :] + + +def debounced_fs_watch(targets, outq, debounce_delay): + import watchdog.events + import watchdog.observers + + class WatchEventHandler(watchdog.events.FileSystemEventHandler): + def __init__(self, queue, file_targets): + self.queue = queue + self.file_targets = file_targets + + def on_modified(self, ev): + if isinstance(ev, watchdog.events.FileModifiedEvent): + self.changed(ev.src_path) + + def on_moved(self, ev): + if isinstance(ev, watchdog.events.FileMovedEvent): + self.changed(ev.dest_path) + + def should_notify(self, path): + for target in self.file_targets: + if path == target: + return True + if args.make and any( + path.endswith(suffix) for suffix in FS_WATCH_EXTENSIONS + ): + return True + return False + + def changed(self, path): + if self.should_notify(path): + self.queue.put(time.time()) + + def debounce_thread(): + listenq = queue.Queue() + file_targets = [] + event_handler = WatchEventHandler(listenq, file_targets) + observer = watchdog.observers.Observer() + observed = set() + for target in targets: + if os.path.isdir(target): + observer.schedule(event_handler, target, recursive=True) + else: + file_targets.append(target) + target = os.path.dirname(target) or "." + if target not in observed: + observed.add(target) + observer.schedule(event_handler, target) + observer.start() + while True: + t = listenq.get() + more = True + while more: + delay = t + debounce_delay - time.time() + if delay > 0: + time.sleep(delay) + # consume entire queue + more = False + try: + while True: + t = listenq.get(block=False) + more = True + except queue.Empty: + pass + outq.put(t) + + th = threading.Thread(target=debounce_thread, daemon=True) + th.start() + + +class Display: + def __init__(self, basedump, mydump): + self.basedump = basedump + self.mydump = mydump + self.emsg = None + + def run_less(self): + if self.emsg is not None: + output = self.emsg + else: + output = "\n".join(do_diff(self.basedump, self.mydump)) + + # Pipe the output through 'tail' and only then to less, to ensure the + # write call doesn't block. ('tail' has to buffer all its input before + # it starts writing.) This also means we don't have to deal with pipe + # closure errors. + buffer_proc = subprocess.Popen( + BUFFER_CMD, stdin=subprocess.PIPE, stdout=subprocess.PIPE + ) + less_proc = subprocess.Popen(LESS_CMD, stdin=buffer_proc.stdout) + buffer_proc.stdin.write(output.encode()) + buffer_proc.stdin.close() + buffer_proc.stdout.close() + return (buffer_proc, less_proc) + + def run_sync(self): + proca, procb = self.run_less() + procb.wait() + proca.wait() + + def run_async(self, watch_queue): + self.watch_queue = watch_queue + self.ready_queue = queue.Queue() + self.pending_update = None + dthread = threading.Thread(target=self.display_thread) + dthread.start() + self.ready_queue.get() + + def display_thread(self): + proca, procb = self.run_less() + self.less_proc = procb + self.ready_queue.put(0) + while True: + ret = procb.wait() + proca.wait() + self.less_proc = None + if ret != 0: + # fix the terminal + os.system("tput reset") + if ret != 0 and self.pending_update is not None: + # killed by program with the intent to refresh + msg, error = self.pending_update + self.pending_update = None + if not error: + self.mydump = msg + self.emsg = None + else: + self.emsg = msg + proca, procb = self.run_less() + self.less_proc = procb + self.ready_queue.put(0) + else: + # terminated by user, or killed + self.watch_queue.put(None) + self.ready_queue.put(0) + break + + def progress(self, msg): + # Write message to top-left corner + sys.stdout.write("\x1b7\x1b[1;1f{}\x1b8".format(msg + " ")) + sys.stdout.flush() + + def update(self, text, error): + if not error and not self.emsg and text == self.mydump: + self.progress("Unchanged. ") + return + self.pending_update = (text, error) + if not self.less_proc: + return + self.less_proc.kill() + self.ready_queue.get() + + def terminate(self): + if not self.less_proc: + return + self.less_proc.kill() + self.ready_queue.get() + + +def main(): + if args.diff_obj: + make_target, basecmd, mycmd = dump_objfile() + else: + make_target, basecmd, mycmd = dump_binary() + + if args.write_asm is not None: + mydump = run_objdump(mycmd) + with open(args.write_asm, "w") as f: + f.write(mydump) + print(f"Wrote assembly to {args.write_asm}.") + sys.exit(0) + + if args.base_asm is not None: + with open(args.base_asm) as f: + basedump = f.read() + else: + basedump = run_objdump(basecmd) + + mydump = run_objdump(mycmd) + + display = Display(basedump, mydump) + + if not args.watch: + display.run_sync() + else: + if not args.make: + yn = input( + "Warning: watch-mode (-w) enabled without auto-make (-m). You will have to run make manually. Ok? (Y/n) " + ) + if yn.lower() == "n": + return + if args.make: + watch_sources = None + if hasattr(diff_settings, "watch_sources_for_target"): + watch_sources = diff_settings.watch_sources_for_target(make_target) + watch_sources = watch_sources or source_directories + if not watch_sources: + fail("Missing source_directories config, don't know what to watch.") + else: + watch_sources = [make_target] + q = queue.Queue() + debounced_fs_watch(watch_sources, q, DEBOUNCE_DELAY) + display.run_async(q) + last_build = 0 + try: + while True: + t = q.get() + if t is None: + break + if t < last_build: + continue + last_build = time.time() + if args.make: + display.progress("Building...") + ret = run_make(make_target, capture_output=True) + if ret.returncode != 0: + display.update( + ret.stderr.decode("utf-8-sig", "replace") + or ret.stdout.decode("utf-8-sig", "replace"), + error=True, + ) + continue + mydump = run_objdump(mycmd) + display.update(mydump, error=False) + except KeyboardInterrupt: + display.terminate() + + +main() diff --git a/diff_settings.py b/diff_settings.py new file mode 100644 index 00000000..65b54d62 --- /dev/null +++ b/diff_settings.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 + +def add_custom_arguments(parser): + group = parser.add_mutually_exclusive_group(required=False) + group.add_argument('-j', dest='lang', action='store_const', const='jp', + help="Set version to JP.") + group.add_argument('-u', dest='lang', action='store_const', const='us', + help="Set version to US.") + group.add_argument('-e', dest='lang', action='store_const', const='eu', + help="Set version to EU.") + group.add_argument('-s', dest='lang', action='store_const', const='sh', + help="Set version to SH.") + +def apply(config, args): + lang = args.lang or 'us' + config['mapfile'] = f'build/dkr.map' + config['myimg'] = f'build/dkr.z64' + config['baseimg'] = f'baseroms/Diddy Kong Racing (USA 1.0).z64' + config['makeflags'] = [f''] + config['source_directories'] = ['src'] diff --git a/src/macros.h b/src/macros.h new file mode 100644 index 00000000..3311613f --- /dev/null +++ b/src/macros.h @@ -0,0 +1,65 @@ +#ifndef _MACROS_H_ +#define _MACROS_H_ + +#ifndef __sgi +#define GLOBAL_ASM(...) +#endif + +#if !defined(__sgi) && (!defined(NON_MATCHING) || !defined(AVOID_UB)) +// asm-process isn't supported outside of IDO, and undefined behavior causes +// crashes. +#error Matching build is only possible on IDO; please build with NON_MATCHING=1. +#endif + +#define ARRAY_COUNT(arr) (s32)(sizeof(arr) / sizeof(arr[0])) + +#define GLUE(a, b) a ## b +#define GLUE2(a, b) GLUE(a, b) + +// Avoid compiler warnings for unused variables +#ifdef __GNUC__ +#define UNUSED __attribute__((unused)) +#else +#define UNUSED +#endif + +// Avoid undefined behaviour for non-returning functions +#ifdef __GNUC__ +#define NORETURN __attribute__((noreturn)) +#else +#define NORETURN +#endif + +// Static assertions +#ifdef __GNUC__ +#define STATIC_ASSERT(cond, msg) _Static_assert(cond, msg) +#else +#define STATIC_ASSERT(cond, msg) typedef char GLUE2(static_assertion_failed, __LINE__)[(cond) ? 1 : -1] +#endif + +// Align to 8-byte boundary for DMA requirements +#ifdef __GNUC__ +#define ALIGNED8 __attribute__((aligned(8))) +#else +#define ALIGNED8 +#endif + +// Align to 16-byte boundary for audio lib requirements +#ifdef __GNUC__ +#define ALIGNED16 __attribute__((aligned(16))) +#else +#define ALIGNED16 +#endif + +// convert a virtual address to physical. +#define VIRTUAL_TO_PHYSICAL(addr) ((uintptr_t)(addr) & 0x1FFFFFFF) + +// convert a physical address to virtual. +#define PHYSICAL_TO_VIRTUAL(addr) ((uintptr_t)(addr) | 0x80000000) + +// another way of converting virtual to physical +#define VIRTUAL_TO_PHYSICAL2(addr) ((u8 *)(addr) - 0x80000000U) + +#define ABSF(x) (x < 0.f ? -x : x) + +#endif diff --git a/src/types.h b/src/types.h new file mode 100644 index 00000000..44137787 --- /dev/null +++ b/src/types.h @@ -0,0 +1,11 @@ +typedef unsigned char u8; +typedef unsigned short u16; +typedef unsigned int u32; +typedef unsigned long u64; +typedef signed char s8; +typedef signed short s16; +typedef signed int s32; +typedef signed long s64; +typedef float f32; +typedef double f64; + diff --git a/src/unknown_0667D0.c b/src/unknown_0667D0.c new file mode 100644 index 00000000..b548b830 --- /dev/null +++ b/src/unknown_0667D0.c @@ -0,0 +1,32 @@ +/* The comment below is needed for this file to be picked up by generate_ld */ +/* RAM_POS: 0x80065BD0 */ + +#include "types.h" +#include "macros.h" + +extern s16 D_800DD050; + +void func_80065BD0(s32 arg0) { + D_800DD050 = arg0; +} + +/* Unused? */ +s16 func_80065BDC(void) { + return D_800DD050; +} + +s32 func_80065BEC(s32 arg0) { + switch(D_800DD050) { + case 0: + return arg0; + case 2: + arg0 = ((arg0 - 0x40) >> 1) + 0x40; + return arg0; + case 1: + default: + return 0x40; + } +} + +GLOBAL_ASM("asm/non_matchings/func_80065C38.s") + diff --git a/src/unknown_09F860.c b/src/unknown_09F860.c new file mode 100644 index 00000000..9f013917 --- /dev/null +++ b/src/unknown_09F860.c @@ -0,0 +1,27 @@ +/* The comment below is needed for this file to be picked up by generate_ld */ +/* RAM_POS: 0x8009EC70 */ + +#include "types.h" + +extern s32 D_800DF494; +extern s32 D_800DFD98; +extern s32 D_800DF4C0; + +s32 func_8009EC70(void) { + return D_800DF494; +} + +s32 func_8009EC80(void) { + if (func_8009C2D0() != 0) { + return 0; + } + return D_800DF4C0; +} + +s32 func_8009ECB8(void) { + return D_800DFD98 & 1; +} + +s32 func_8009ECD0(void) { + return D_800DFD98 & 2; +} diff --git a/tools/asm_processor/asm-processor.py b/tools/asm_processor/asm-processor.py new file mode 100755 index 00000000..b7ddbdf2 --- /dev/null +++ b/tools/asm_processor/asm-processor.py @@ -0,0 +1,868 @@ +#!/usr/bin/env python3 +import argparse +import tempfile +import struct +import copy +import sys +import re +import os + +MAX_FN_SIZE = 100 + +EI_NIDENT = 16 +EI_CLASS = 4 +EI_DATA = 5 +EI_VERSION = 6 +EI_OSABI = 7 +EI_ABIVERSION = 8 +STN_UNDEF = 0 + +SHN_UNDEF = 0 +SHN_ABS = 0xfff1 +SHN_COMMON = 0xfff2 +SHN_XINDEX = 0xffff +SHN_LORESERVE = 0xff00 + +STT_NOTYPE = 0 +STT_OBJECT = 1 +STT_FUNC = 2 +STT_SECTION = 3 +STT_FILE = 4 +STT_COMMON = 5 +STT_TLS = 6 + +STB_LOCAL = 0 +STB_GLOBAL = 1 +STB_WEAK = 2 + +STV_DEFAULT = 0 +STV_INTERNAL = 1 +STV_HIDDEN = 2 +STV_PROTECTED = 3 + +SHT_NULL = 0 +SHT_PROGBITS = 1 +SHT_SYMTAB = 2 +SHT_STRTAB = 3 +SHT_RELA = 4 +SHT_HASH = 5 +SHT_DYNAMIC = 6 +SHT_NOTE = 7 +SHT_NOBITS = 8 +SHT_REL = 9 +SHT_SHLIB = 10 +SHT_DYNSYM = 11 +SHT_INIT_ARRAY = 14 +SHT_FINI_ARRAY = 15 +SHT_PREINIT_ARRAY = 16 +SHT_GROUP = 17 +SHT_SYMTAB_SHNDX = 18 +SHT_MIPS_GPTAB = 0x70000003 +SHT_MIPS_DEBUG = 0x70000005 +SHT_MIPS_REGINFO = 0x70000006 +SHT_MIPS_OPTIONS = 0x7000000d + +SHF_WRITE = 0x1 +SHF_ALLOC = 0x2 +SHF_EXECINSTR = 0x4 +SHF_MERGE = 0x10 +SHF_STRINGS = 0x20 +SHF_INFO_LINK = 0x40 +SHF_LINK_ORDER = 0x80 +SHF_OS_NONCONFORMING = 0x100 +SHF_GROUP = 0x200 +SHF_TLS = 0x400 + +R_MIPS_32 = 2 +R_MIPS_26 = 4 +R_MIPS_HI16 = 5 +R_MIPS_LO16 = 6 + + +class ElfHeader: + """ + typedef struct { + unsigned char e_ident[EI_NIDENT]; + Elf32_Half e_type; + Elf32_Half e_machine; + Elf32_Word e_version; + Elf32_Addr e_entry; + Elf32_Off e_phoff; + Elf32_Off e_shoff; + Elf32_Word e_flags; + Elf32_Half e_ehsize; + Elf32_Half e_phentsize; + Elf32_Half e_phnum; + Elf32_Half e_shentsize; + Elf32_Half e_shnum; + Elf32_Half e_shstrndx; + } Elf32_Ehdr; + """ + + def __init__(self, data): + self.e_ident = data[:EI_NIDENT] + self.e_type, self.e_machine, self.e_version, self.e_entry, self.e_phoff, self.e_shoff, self.e_flags, self.e_ehsize, self.e_phentsize, self.e_phnum, self.e_shentsize, self.e_shnum, self.e_shstrndx = struct.unpack('>HHIIIIIHHHHHH', data[EI_NIDENT:]) + assert self.e_ident[EI_CLASS] == 1 # 32-bit + assert self.e_ident[EI_DATA] == 2 # big-endian + assert self.e_type == 1 # relocatable + assert self.e_machine == 8 # MIPS I Architecture + assert self.e_phoff == 0 # no program header + assert self.e_shoff != 0 # section header + assert self.e_shstrndx != SHN_UNDEF + + def to_bin(self): + return self.e_ident + struct.pack('>HHIIIIIHHHHHH', self.e_type, + self.e_machine, self.e_version, self.e_entry, self.e_phoff, + self.e_shoff, self.e_flags, self.e_ehsize, self.e_phentsize, + self.e_phnum, self.e_shentsize, self.e_shnum, self.e_shstrndx) + + +class Symbol: + """ + typedef struct { + Elf32_Word st_name; + Elf32_Addr st_value; + Elf32_Word st_size; + unsigned char st_info; + unsigned char st_other; + Elf32_Half st_shndx; + } Elf32_Sym; + """ + + def __init__(self, data, strtab): + self.st_name, self.st_value, self.st_size, st_info, self.st_other, self.st_shndx = struct.unpack('>IIIBBH', data) + assert self.st_shndx != SHN_XINDEX, "too many sections (SHN_XINDEX not supported)" + self.bind = st_info >> 4 + self.type = st_info & 15 + self.name = strtab.lookup_str(self.st_name) + self.visibility = self.st_other & 3 + + def to_bin(self): + st_info = (self.bind << 4) | self.type + return struct.pack('>IIIBBH', self.st_name, self.st_value, self.st_size, st_info, self.st_other, self.st_shndx) + + +class Relocation: + def __init__(self, data, sh_type): + self.sh_type = sh_type + if sh_type == SHT_REL: + self.r_offset, self.r_info = struct.unpack('>II', data) + else: + self.r_offset, self.r_info, self.r_addend = struct.unpack('>III', data) + self.sym_index = self.r_info >> 8 + self.rel_type = self.r_info & 0xff + + def to_bin(self): + self.r_info = (self.sym_index << 8) | self.rel_type + if self.sh_type == SHT_REL: + return struct.pack('>II', self.r_offset, self.r_info) + else: + return struct.pack('>III', self.r_offset, self.r_info, self.r_addend) + + +class Section: + """ + typedef struct { + Elf32_Word sh_name; + Elf32_Word sh_type; + Elf32_Word sh_flags; + Elf32_Addr sh_addr; + Elf32_Off sh_offset; + Elf32_Word sh_size; + Elf32_Word sh_link; + Elf32_Word sh_info; + Elf32_Word sh_addralign; + Elf32_Word sh_entsize; + } Elf32_Shdr; + """ + + def __init__(self, header, data, index): + self.sh_name, self.sh_type, self.sh_flags, self.sh_addr, self.sh_offset, self.sh_size, self.sh_link, self.sh_info, self.sh_addralign, self.sh_entsize = struct.unpack('>IIIIIIIIII', header) + assert not self.sh_flags & SHF_LINK_ORDER + if self.sh_entsize != 0: + assert self.sh_size % self.sh_entsize == 0 + if self.sh_type == SHT_NOBITS: + self.data = '' + else: + self.data = data[self.sh_offset:self.sh_offset + self.sh_size] + self.index = index + self.relocated_by = [] + + @staticmethod + def from_parts(sh_name, sh_type, sh_flags, sh_link, sh_info, sh_addralign, sh_entsize, data, index): + header = struct.pack('>IIIIIIIIII', sh_name, sh_type, sh_flags, 0, 0, len(data), sh_link, sh_info, sh_addralign, sh_entsize) + return Section(header, data, index) + + def lookup_str(self, index): + assert self.sh_type == SHT_STRTAB + to = self.data.find(b'\0', index) + assert to != -1 + return self.data[index:to].decode('utf-8') + + def add_str(self, string): + assert self.sh_type == SHT_STRTAB + ret = len(self.data) + self.data += bytes(string, 'utf-8') + b'\0' + return ret + + def is_rel(self): + return self.sh_type == SHT_REL or self.sh_type == SHT_RELA + + def header_to_bin(self): + if self.sh_type != SHT_NOBITS: + self.sh_size = len(self.data) + return struct.pack('>IIIIIIIIII', self.sh_name, self.sh_type, self.sh_flags, self.sh_addr, self.sh_offset, self.sh_size, self.sh_link, self.sh_info, self.sh_addralign, self.sh_entsize) + + def late_init(self, sections): + if self.sh_type == SHT_SYMTAB: + self.init_symbols(sections) + elif self.is_rel(): + self.rel_target = sections[self.sh_info] + self.rel_target.relocated_by.append(self) + self.init_relocs() + + def find_symbol(self, name): + assert self.sh_type == SHT_SYMTAB + for s in self.symbol_entries: + if s.name == name: + return (s.st_shndx, s.st_value) + return None + + def init_symbols(self, sections): + assert self.sh_type == SHT_SYMTAB + assert self.sh_entsize == 16 + self.strtab = sections[self.sh_link] + entries = [] + for i in range(0, self.sh_size, self.sh_entsize): + entries.append(Symbol(self.data[i:i+self.sh_entsize], self.strtab)) + self.symbol_entries = entries + + def init_relocs(self): + assert self.is_rel() + entries = [] + for i in range(0, self.sh_size, self.sh_entsize): + entries.append(Relocation(self.data[i:i+self.sh_entsize], self.sh_type)) + self.relocations = entries + + def local_symbols(self): + assert self.sh_type == SHT_SYMTAB + return self.symbol_entries[:self.sh_info] + + def global_symbols(self): + assert self.sh_type == SHT_SYMTAB + return self.symbol_entries[self.sh_info:] + + +class ElfFile: + def __init__(self, data): + self.data = data + assert data[:4] == b'\x7fELF', "not an ELF file" + + self.elf_header = ElfHeader(data[0:52]) + + offset, size = self.elf_header.e_shoff, self.elf_header.e_shentsize + null_section = Section(data[offset:offset + size], data, 0) + num_sections = self.elf_header.e_shnum or null_section.sh_size + + self.sections = [null_section] + for i in range(1, num_sections): + ind = offset + i * size + self.sections.append(Section(data[ind:ind + size], data, i)) + + symtab = None + for s in self.sections: + if s.sh_type == SHT_SYMTAB: + assert not symtab + symtab = s + assert symtab is not None + self.symtab = symtab + + shstr = self.sections[self.elf_header.e_shstrndx] + for s in self.sections: + s.name = shstr.lookup_str(s.sh_name) + s.late_init(self.sections) + + def find_section(self, name): + for s in self.sections: + if s.name == name: + return s + return None + + def add_section(self, name, sh_type, sh_flags, sh_link, sh_info, sh_addralign, sh_entsize, data): + shstr = self.sections[self.elf_header.e_shstrndx] + sh_name = shstr.add_str(name) + s = Section.from_parts(sh_name=sh_name, sh_type=sh_type, + sh_flags=sh_flags, sh_link=sh_link, sh_info=sh_info, + sh_addralign=sh_addralign, sh_entsize=sh_entsize, data=data, + index=len(self.sections)) + self.sections.append(s) + s.name = name + s.late_init(self.sections) + return s + + def drop_irrelevant_sections(self): + # We can only drop sections at the end, since otherwise section + # references might be wrong. Luckily, these sections typically are. + while self.sections[-1].sh_type in [SHT_MIPS_DEBUG, SHT_MIPS_GPTAB]: + self.sections.pop() + + def write(self, filename): + outfile = open(filename, 'wb') + outidx = 0 + def write_out(data): + nonlocal outidx + outfile.write(data) + outidx += len(data) + def pad_out(align): + if align and outidx % align: + write_out(b'\0' * (align - outidx % align)) + + self.elf_header.e_shnum = len(self.sections) + write_out(self.elf_header.to_bin()) + + for s in self.sections: + if s.sh_type != SHT_NOBITS and s.sh_type != SHT_NULL: + pad_out(s.sh_addralign) + s.sh_offset = outidx + write_out(s.data) + + pad_out(4) + self.elf_header.e_shoff = outidx + for s in self.sections: + write_out(s.header_to_bin()) + + outfile.seek(0) + outfile.write(self.elf_header.to_bin()) + outfile.close() + + +def is_temp_name(name): + return name.startswith('_asmpp_') + +class GlobalState: + def __init__(self, min_instr_count, skip_instr_count): + # A value that hopefully never appears as a 32-bit rodata constant (or we + # miscompile late rodata). Increases by 1 in each step. + self.late_rodata_hex = 0xE0123456 + self.namectr = 0 + self.min_instr_count = min_instr_count + self.skip_instr_count = skip_instr_count + + def make_name(self, cat): + self.namectr += 1 + return '_asmpp_{}{}'.format(cat, self.namectr) + +class GlobalAsmBlock: + def __init__(self): + self.cur_section = '.text' + self.asm_conts = [] + self.late_rodata_asm_conts = [] + self.late_rodata_alignment = 0 + self.text_glabels = [] + self.fn_section_sizes = { + '.text': 0, + '.data': 0, + '.bss': 0, + '.rodata': 0, + '.late_rodata': 0, + } + self.fn_ins_inds = [] + self.num_lines = 0 + + def add_sized(self, size, line): + if self.cur_section in ['.text', '.late_rodata']: + assert size % 4 == 0, "size must be a multiple of 4 on line: " + line + assert size >= 0 + self.fn_section_sizes[self.cur_section] += size + if self.cur_section == '.text': + assert self.text_glabels, ".text block without an initial glabel" + self.fn_ins_inds.append((self.num_lines, size // 4)) + + def process_line(self, line): + line = re.sub(r'/\*.*?\*/', '', line) + line = re.sub(r'#.*', '', line) + line = line.strip() + changed_section = False + if line.startswith('glabel ') and self.cur_section == '.text': + self.text_glabels.append(line.split()[1]) + if not line: + pass # empty line + elif line.startswith('glabel ') or (' ' not in line and line.endswith(':')): + pass # label + elif line.startswith('.section') or line in ['.text', '.data', '.rdata', '.rodata', '.bss', '.late_rodata']: + # section change + self.cur_section = '.rodata' if line == '.rdata' else line.split(',')[0].split()[-1] + assert self.cur_section in ['.data', '.text', '.rodata', '.late_rodata', '.bss'], \ + "unrecognized .section directive" + changed_section = True + elif line.startswith('.late_rodata_alignment'): + assert self.cur_section == '.late_rodata' + self.late_rodata_alignment = int(line.split()[1]) + assert self.late_rodata_alignment in [4, 8] + changed_section = True + elif line.startswith('.incbin'): + self.add_sized(int(line.split(',')[-1].strip(), 0), line) + elif line.startswith('.word') or line.startswith('.float'): + self.add_sized(4 * len(line.split(',')), line) + elif line.startswith('.double'): + self.add_sized(8 * len(line.split(',')), line) + elif line.startswith('.space'): + self.add_sized(int(line.split()[1], 0), line) + elif line.startswith('.'): + # .macro, .ascii, .asciiz, .balign, .align, ... + assert False, 'not supported yet: ' + line + else: + # Unfortunately, macros are hard to support for .rodata -- + # we don't know how how space they will expand to before + # running the assembler, but we need that information to + # construct the C code. So if we need that we'll either + # need to run the assembler twice (at least in some rare + # cases), or change how this program is invoked. + # Similarly, we can't currently deal with pseudo-instructions + # that expand to several real instructions. + assert self.cur_section == '.text', "instruction or macro call in non-.text section? not supported: " + line + self.add_sized(4, line) + if self.cur_section == '.late_rodata': + if not changed_section: + self.late_rodata_asm_conts.append(line) + else: + self.asm_conts.append(line) + self.num_lines += 1 + + def finish(self, state): + src = [''] * (self.num_lines + 1) + late_rodata = [] + late_rodata_fn_output = [] + + if self.fn_section_sizes['.late_rodata'] > 0: + # Generate late rodata by emitting unique float constants. + # This requires 3 instructions for each 4 bytes of rodata. + # If we know alignment, we can use doubles, which give 3 + # instructions for 8 bytes of rodata. + size = self.fn_section_sizes['.late_rodata'] // 4 + skip_next = False + for i in range(size): + if skip_next: + skip_next = False + continue + if (state.late_rodata_hex & 0xffff) == 0: + # Avoid lui + state.late_rodata_hex += 1 + dummy_bytes = struct.pack('>I', state.late_rodata_hex) + state.late_rodata_hex += 1 + late_rodata.append(dummy_bytes) + if self.late_rodata_alignment == 4 * ((i + 1) % 2 + 1) and i + 1 < size: + late_rodata.append(dummy_bytes) + fval, = struct.unpack('>d', dummy_bytes * 2) + late_rodata_fn_output.append('*(volatile double*)0 = {};'.format(fval)) + skip_next = True + else: + fval, = struct.unpack('>f', dummy_bytes) + late_rodata_fn_output.append('*(volatile float*)0 = {}f;'.format(fval)) + late_rodata_fn_output.append('') + late_rodata_fn_output.append('') + + text_name = None + if self.fn_section_sizes['.text'] > 0 or late_rodata_fn_output: + text_name = state.make_name('func') + src[0] = 'void {}(void) {{'.format(text_name) + src[self.num_lines] = '}' + instr_count = self.fn_section_sizes['.text'] // 4 + assert instr_count >= state.min_instr_count, "too short .text block" + tot_emitted = 0 + tot_skipped = 0 + fn_emitted = 0 + fn_skipped = 0 + rodata_stack = late_rodata_fn_output[::-1] + for (line, count) in self.fn_ins_inds: + for _ in range(count): + if (fn_emitted > MAX_FN_SIZE and instr_count - tot_emitted > state.min_instr_count and + (not rodata_stack or rodata_stack[-1])): + # Don't let functions become too large. When a function reaches 284 + # instructions, and -O2 -framepointer flags are passed, the IRIX + # compiler decides it is a great idea to start optimizing more. + fn_emitted = 0 + fn_skipped = 0 + src[line] += ' }} void {}(void) {{ '.format(state.make_name('large_func')) + if fn_skipped < state.skip_instr_count: + fn_skipped += 1 + tot_skipped += 1 + elif rodata_stack: + src[line] += rodata_stack.pop() + else: + src[line] += '*(volatile int*)0 = 0;' + tot_emitted += 1 + fn_emitted += 1 + if rodata_stack: + size = len(late_rodata_fn_output) // 3 + available = instr_count - tot_skipped + print("late rodata to text ratio is too high: {} / {} must be <= 1/3" + .format(size, available), file=sys.stderr) + print("add a .late_rodata_alignment (4|8) to the .late_rodata " + "block to double the allowed ratio.", file=sys.stderr) + exit(1) + + rodata_name = None + if self.fn_section_sizes['.rodata'] > 0: + rodata_name = state.make_name('rodata') + output_line += ' const char {}[{}] = {{1}};'.format(rodata_name, self.fn_section_sizes['.rodata']) + + data_name = None + if self.fn_section_sizes['.data'] > 0: + data_name = state.make_name('data') + output_line += ' char {}[{}] = {{1}};'.format(data_name, self.fn_section_sizes['.data']) + + bss_name = None + if self.fn_section_sizes['.bss'] > 0: + bss_name = state.make_name('bss') + output_line += ' char {}[{}];'.format(bss_name, self.fn_section_sizes['.bss']) + + fn = (self.text_glabels, self.asm_conts, late_rodata, self.late_rodata_asm_conts, + { + '.text': (text_name, self.fn_section_sizes['.text']), + '.data': (data_name, self.fn_section_sizes['.data']), + '.rodata': (rodata_name, self.fn_section_sizes['.rodata']), + '.bss': (bss_name, self.fn_section_sizes['.bss']), + }) + return src, fn + +def parse_source(f, print_source, opt, framepointer): + if opt == 'O2': + if framepointer: + min_instr_count = 6 + skip_instr_count = 5 + else: + min_instr_count = 2 + skip_instr_count = 1 + elif opt == 'g': + if framepointer: + min_instr_count = 7 + skip_instr_count = 7 + else: + min_instr_count = 4 + skip_instr_count = 4 + else: + assert opt == 'g3' + if framepointer: + min_instr_count = 4 + skip_instr_count = 4 + else: + min_instr_count = 2 + skip_instr_count = 2 + + state = GlobalState(min_instr_count, skip_instr_count) + + global_asm = None + asm_functions = [] + output_lines = [] + + for raw_line in f: + raw_line = raw_line.rstrip() + line = raw_line.lstrip() + + # Print exactly one output line per source line, to make compiler + # errors have correct line numbers. These will be overridden with + # reasonable content further down. + output_lines.append('') + + if global_asm is not None: + if line.startswith(')'): + src, fn = global_asm.finish(state) + for i, line2 in enumerate(src): + output_lines[start_index + i] = line2 + asm_functions.append(fn) + global_asm = None + else: + global_asm.process_line(line) + else: + if line == 'GLOBAL_ASM(': + global_asm = GlobalAsmBlock() + start_index = len(output_lines) + elif line.startswith('GLOBAL_ASM("') and line.endswith('")'): + global_asm = GlobalAsmBlock() + fname = line[len('GLOBAL_ASM') + 2 : -2] + with open(fname) as f: + for line2 in f: + global_asm.process_line(line2) + src, fn = global_asm.finish(state) + output_lines[-1] = ''.join(src) + asm_functions.append(fn) + global_asm = None + else: + output_lines[-1] = raw_line + + if print_source: + for line in output_lines: + print(line) + + return asm_functions + +def fixup_objfile(objfile_name, functions, asm_prelude, assembler): + SECTIONS = ['.data', '.text', '.rodata', '.bss'] + + with open(objfile_name, 'rb') as f: + objfile = ElfFile(f.read()) + + prev_locs = { + '.text': 0, + '.data': 0, + '.rodata': 0, + '.bss': 0, + } + to_copy = { + '.text': [], + '.data': [], + '.rodata': [], + } + asm = [] + late_rodata = [] + late_rodata_asm = [] + late_rodata_source_name = None + + # Generate an assembly file with all the assembly we need to fill in. For + # simplicity we pad with nops/.space so that addresses match exactly, so we + # don't have to fix up relocations/symbol references. + all_text_glabels = set() + for (text_glabels, body, fn_late_rodata, fn_late_rodata_body, data) in functions: + ifdefed = False + for sectype, (temp_name, size) in data.items(): + if temp_name is None: + continue + assert size > 0 + loc = objfile.symtab.find_symbol(temp_name) + if loc is None: + ifdefed = True + break + loc = loc[1] + prev_loc = prev_locs[sectype] + assert loc >= prev_loc, sectype + if loc != prev_loc: + asm.append('.section ' + sectype) + if sectype == '.text': + for i in range((loc - prev_loc) // 4): + asm.append('nop') + else: + asm.append('.space {}'.format(loc - prev_loc)) + if sectype != '.bss': + to_copy[sectype].append((loc, size)) + prev_locs[sectype] = loc + size + if not ifdefed: + all_text_glabels.update(text_glabels) + late_rodata.extend(fn_late_rodata) + late_rodata_asm.extend(fn_late_rodata_body) + asm.append('.text') + for line in body: + asm.append(line) + if late_rodata_asm: + late_rodata_source_name = '_asmpp_late_rodata' + asm.append('.rdata') + asm.append('glabel {}'.format(late_rodata_source_name)) + asm.extend(late_rodata_asm) + + o_file = tempfile.NamedTemporaryFile(prefix='asm-processor', suffix='.o', delete=False) + o_name = o_file.name + o_file.close() + s_file = tempfile.NamedTemporaryFile(prefix='asm-processor', suffix='.s', delete=False) + s_name = s_file.name + try: + s_file.write(asm_prelude + b'\n') + for line in asm: + s_file.write(line.encode('utf-8') + b'\n') + s_file.close() + ret = os.system(assembler + " " + s_name + " -o " + o_name) + if ret != 0: + raise Exception("failed to assemble") + with open(o_name, 'rb') as f: + asm_objfile = ElfFile(f.read()) + + # Remove some clutter from objdump output + objfile.drop_irrelevant_sections() + + # Unify reginfo sections + target_reginfo = objfile.find_section('.reginfo') + source_reginfo_data = list(asm_objfile.find_section('.reginfo').data) + data = list(target_reginfo.data) + for i in range(20): + data[i] |= source_reginfo_data[i] + target_reginfo.data = bytes(data) + + # Move over section contents + modified_text_positions = set() + last_rodata_pos = 0 + for sectype in SECTIONS: + if sectype == '.bss': + continue + source = asm_objfile.find_section(sectype) + target = objfile.find_section(sectype) + if source is None or not to_copy[sectype]: + continue + assert target is not None, "must have a section to overwrite: " + sectype + data = list(target.data) + for (pos, count) in to_copy[sectype]: + data[pos:pos + count] = source.data[pos:pos + count] + if sectype == '.text': + assert count % 4 == 0 + assert pos % 4 == 0 + for i in range(count // 4): + modified_text_positions.add(pos + 4 * i) + elif sectype == '.rodata': + last_rodata_pos = pos + count + target.data = bytes(data) + + # Move over late rodata. This is heuristic, sadly, since I can't think + # of another way of doing it. + moved_late_rodata = {} + if late_rodata: + source = asm_objfile.find_section('.rodata') + target = objfile.find_section('.rodata') + source_pos = asm_objfile.symtab.find_symbol(late_rodata_source_name) + assert source_pos is not None and source_pos[0] == source.index + source_pos = source_pos[1] + new_data = list(target.data) + for dummy_bytes in late_rodata: + pos = target.data.index(dummy_bytes, last_rodata_pos) + new_data[pos:pos+4] = source.data[source_pos:source_pos+4] + moved_late_rodata[source_pos] = pos + last_rodata_pos = pos + 4 + source_pos += 4 + target.data = bytes(new_data) + + # Merge strtab data. + strtab_adj = len(objfile.symtab.strtab.data) + objfile.symtab.strtab.data += asm_objfile.symtab.strtab.data + + # Find relocated symbols + relocated_symbols = set() + for sectype in SECTIONS: + for obj in [asm_objfile, objfile]: + sec = obj.find_section(sectype) + if sec is None: + continue + for reltab in sec.relocated_by: + for rel in reltab.relocations: + relocated_symbols.add(obj.symtab.symbol_entries[rel.sym_index]) + + # Move over symbols, deleting the temporary function labels. + # Sometimes this naive procedure results in duplicate symbols, or UNDEF + # symbols that are also defined the same .o file. Hopefully that's fine. + # Skip over local symbols that aren't used relocated against, to avoid + # conflicts. + new_local_syms = [s for s in objfile.symtab.local_symbols() if not is_temp_name(s.name)] + new_global_syms = [s for s in objfile.symtab.global_symbols() if not is_temp_name(s.name)] + for i, s in enumerate(asm_objfile.symtab.symbol_entries): + is_local = (i < asm_objfile.symtab.sh_info) + if is_local and s not in relocated_symbols: + continue + if is_temp_name(s.name): + continue + if s.st_shndx not in [SHN_UNDEF, SHN_ABS]: + section_name = asm_objfile.sections[s.st_shndx].name + assert section_name in SECTIONS, "Generated assembly .o must only have symbols for .text, .data, .rodata, ABS and UNDEF, but found {}".format(section_name) + s.st_shndx = objfile.find_section(section_name).index + # glabel's aren't marked as functions, making objdump output confusing. Fix that. + if s.name in all_text_glabels: + s.type = STT_FUNC + if objfile.sections[s.st_shndx].name == '.rodata' and s.st_value in moved_late_rodata: + s.st_value = moved_late_rodata[s.st_value] + s.st_name += strtab_adj + if is_local: + new_local_syms.append(s) + else: + new_global_syms.append(s) + new_syms = new_local_syms + new_global_syms + for i, s in enumerate(new_syms): + s.new_index = i + objfile.symtab.data = b''.join(s.to_bin() for s in new_syms) + objfile.symtab.sh_info = len(new_local_syms) + + # Move over relocations + for sectype in SECTIONS: + source = asm_objfile.find_section(sectype) + target = objfile.find_section(sectype) + + if target is not None: + # fixup relocation symbol indices, since we butchered them above + for reltab in target.relocated_by: + nrels = [] + for rel in reltab.relocations: + if sectype == '.text' and rel.r_offset in modified_text_positions: + # don't include relocations for late_rodata dummy code + continue + # hopefully we don't have relocations for local or + # temporary symbols, so new_index exists + rel.sym_index = objfile.symtab.symbol_entries[rel.sym_index].new_index + nrels.append(rel) + reltab.relocations = nrels + reltab.data = b''.join(rel.to_bin() for rel in nrels) + + if not source: + continue + + target_reltab = objfile.find_section('.rel' + sectype) + target_reltaba = objfile.find_section('.rela' + sectype) + for reltab in source.relocated_by: + for rel in reltab.relocations: + rel.sym_index = asm_objfile.symtab.symbol_entries[rel.sym_index].new_index + if sectype == '.rodata' and rel.r_offset in moved_late_rodata: + rel.r_offset = moved_late_rodata[rel.r_offset] + new_data = b''.join(rel.to_bin() for rel in reltab.relocations) + if reltab.sh_type == SHT_REL: + if not target_reltab: + target_reltab = objfile.add_section('.rel' + sectype, + sh_type=SHT_REL, sh_flags=0, + sh_link=objfile.symtab.index, sh_info=target.index, + sh_addralign=4, sh_entsize=8, data=b'') + target_reltab.data += new_data + else: + if not target_reltaba: + target_reltaba = objfile.add_section('.rela' + sectype, + sh_type=SHT_RELA, sh_flags=0, + sh_link=objfile.symtab.index, sh_info=target.index, + sh_addralign=4, sh_entsize=12, data=b'') + target_reltaba.data += new_data + + objfile.write(objfile_name) + finally: + s_file.close() + os.remove(s_name) + try: + os.remove(o_name) + except: + pass + +def main(): + parser = argparse.ArgumentParser(description="Pre-process .c files and post-process .o files to enable embedding assembly into C.") + parser.add_argument('filename', help="path to .c code") + parser.add_argument('--post-process', dest='objfile', help="path to .o file to post-process") + parser.add_argument('--assembler', dest='assembler', help="assembler command (e.g. \"mips-linux-gnu-as -march=vr4300 -mabi=32\")") + parser.add_argument('--asm-prelude', dest='asm_prelude', help="path to a file containing a prelude to the assembly file (with .set and .macro directives, e.g.)") + parser.add_argument('-framepointer', dest='framepointer', action='store_true') + parser.add_argument('-g3', dest='g3', action='store_true') + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('-O2', dest='o2', action='store_true') + group.add_argument('-g', dest='o2', action='store_false') + args = parser.parse_args() + opt = 'O2' if args.o2 else 'g' + if args.g3: + if opt != 'O2': + print("-g3 is only supported together with -O2", file=sys.stderr) + exit(1) + opt = 'g3' + + if args.objfile is None: + with open(args.filename) as f: + parse_source(f, print_source=True, opt=opt, framepointer=args.framepointer) + else: + assert args.assembler is not None, "must pass assembler command" + with open(args.filename) as f: + functions = parse_source(f, print_source=False, opt=opt, framepointer=args.framepointer) + if not functions: + return + asm_prelude = b'' + if args.asm_prelude: + with open(args.asm_prelude, 'rb') as f: + asm_prelude = f.read() + fixup_objfile(args.objfile, functions, asm_prelude, args.assembler) + +if __name__ == "__main__": + main() diff --git a/tools/asm_processor/build.py b/tools/asm_processor/build.py new file mode 100755 index 00000000..04a93e4c --- /dev/null +++ b/tools/asm_processor/build.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +import sys +import os +import shlex +import subprocess +import tempfile + +dir_path = os.path.dirname(os.path.realpath(__file__)) +asm_processor = ['python3', os.path.join(dir_path, "asm-processor.py")] +prelude = os.path.join(dir_path, "prelude.inc") + +all_args = sys.argv[1:] +sep1 = all_args.index('--') +sep2 = all_args.index('--', sep1+1) + +compiler = all_args[:sep1] + +assembler = all_args[sep1+1:sep2] +assembler_sh = ' '.join(shlex.quote(x) for x in assembler) + +compile_args = all_args[sep2+1:] +in_file = compile_args[-1] +out_ind = compile_args.index('-o') +out_file = compile_args[out_ind + 1] +del compile_args[-1] +del compile_args[out_ind + 1] +del compile_args[out_ind] + +in_dir = os.path.split(os.path.realpath(in_file))[0] +opt_flags = [x for x in compile_args if x in ['-g', '-O2', '-framepointer']] + +preprocessed_file = tempfile.NamedTemporaryFile(prefix='preprocessed', suffix='.c') + +subprocess.check_call(asm_processor + opt_flags + [in_file], stdout=preprocessed_file) +subprocess.check_call(compiler + compile_args + ['-I', in_dir, '-o', out_file, preprocessed_file.name]) +subprocess.check_call(asm_processor + opt_flags + [in_file, '--post-process', out_file, '--assembler', assembler_sh, '--asm-prelude', prelude]) diff --git a/tools/asm_processor/prelude.inc b/tools/asm_processor/prelude.inc new file mode 100755 index 00000000..173701a2 --- /dev/null +++ b/tools/asm_processor/prelude.inc @@ -0,0 +1,7 @@ +.include "globals.inc" +.include "macros.inc" + +# assembler directives +.set noat # allow manual use of $at +.set noreorder # dont insert nops after branches +.set gp=64 # 64-bit instructions are used \ No newline at end of file diff --git a/tools/python/generate_ld.py b/tools/python/generate_ld.py index 16068aa4..a815d1c2 100755 --- a/tools/python/generate_ld.py +++ b/tools/python/generate_ld.py @@ -7,6 +7,7 @@ from file_util import FileUtil LD_NAME = 'dkr.ld' ASM_DIR = './asm' +SRC_DIR = './src' ASSETS_S_FILENAME = './asm/assets/assets.s' ASSETS_DIR = './assets/us_1.0' ASSETS_START = 0x0D8200 @@ -15,7 +16,7 @@ class LD: def __init__(self, file): print('Generating linker file...') self.generate_assets_file() - self.files = self.get_asm_files() + self.files = self.get_code_files() self.indentLevel = 0 self.file = file self.gen_comment('linker script generated by generate_ld.py') @@ -118,8 +119,8 @@ class LD: def gen_newline(self): self.file.write('\n') - def get_asm_files(self): - asmFiles = [] + def get_code_files(self): + files = [] asmFilenames = FileUtil.get_filenames_from_directory(ASM_DIR, ('.s',)) regex = r'[\/][*]\s*([0-9A-Fa-f]{6})\s*([0-9A-Fa-f]{8})\s*([0-9A-Fa-f]{8})\s*[*][\/]' for filename in asmFilenames: @@ -132,10 +133,24 @@ class LD: line = asmFile.readline() continue matchedGroups = matches.groups() - asmFiles.append(('build/asm/' + filename[:-2] + '.o', matchedGroups[0], matchedGroups[1])) + files.append(('build/asm/' + filename[:-2] + '.o', matchedGroups[0], matchedGroups[1], 1)) break - asmFiles.sort(key = lambda x: x[2]) # Sort tuples by RAM address - return asmFiles + srcFilenames = FileUtil.get_filenames_from_directory(SRC_DIR, ('.c',)) + regex = r'[\/][*]+\s*RAM_POS:\s*0x([0-9a-fA-F]+)\s*[*]+[\/]' + for filename in srcFilenames: + with open(SRC_DIR + '/' + filename, 'r') as srcFile: + notDone = True + line = srcFile.readline() + while line: + matches = re.match(regex, line) + if matches is None: + line = srcFile.readline() + continue + matchedGroups = matches.groups() + files.append(('build/src/' + filename[:-2] + '.o', '', matchedGroups[0], 0)) + break + files.sort(key = lambda x: (x[2], x[3])) # Sort tuples by RAM address and prioritize src files first. + return files def get_asset_files(self): assetFiles = []