mirror of
https://github.com/izzy2lost/Diddy-Kong-Racing.git
synced 2026-06-19 01:16:26 -07:00
Splat merge (#470)
* First pass * Fix n64crc and add submodules properly * Fix other versions * Match func_8005B818 for v80 code. * Formatting * Fix build for JPN in last commit. Still broken in post v77 roms though. * Fix builds for other versions. * Match load_menu_text for other versions. * Fix progress script * update m2c * Modify asset tools to remove the LD script code. * Fix asm file macro inclue * Get a working splat version for us_1.0 to build from the assets tool. * update asm differ * Update tools again * Fix the makefile to only compile assets when requested. This will build all versions successfully, and compile assets for us_1.0 when requested. * First round of suggestions * Small cleanup * Fix the gcc_generate.py path. * Make entrypointThreadStack * Small addition to the last commit * "Fix" score script. Still need to fix the score values themselves, but at least it runs and is kind of close. * Much closer matching score script * Fix the splat version due to a breaking change in 0.33.0 for this repo for now. * Fix the main function name * Add gitignore entries * Fix the padding problem to be handled by objcopy instead of a binary pad from splat. * Update the README and change dependencies to setup. * Have a hasm header that can be tweaked. * Still calculate the checksum on no_verify builds or they won't work. * Add support for boot_custom.bin * Fix custom boot ld code. * Fix score script * Fix gcc building. * Update m2c * Fix warning, stop ignoring mod assets, and add some handy make rules. * Uggh, serves me right for not testing. * First stab at modifiable entrypoint. * Fix typo, and small README change * Stop n64crd from defaulting to returning 6105, so we can properly fail if the CIC checksum fails. Also, fix the * Extract custom boot script * Update automated scripts. * Woops, fixed the MAXCONTROLLERS thing now. * Add the method for building binutils back. Sorry! * Only use -m32 when the longbit says we're on a 64 bit platform. * Woops.... * Hopefully fix arm detection for raspi ido downloads.
This commit is contained in:
Ryan Myers
Executable
+44
@@ -0,0 +1,44 @@
|
||||
#!/usr/bin/python3
|
||||
|
||||
import sys
|
||||
|
||||
NBYTES = 0x120000
|
||||
|
||||
def get_files(rom_file, mask_file):
|
||||
total = 0
|
||||
value = 0
|
||||
while True:
|
||||
# Read a block of bytes instead of a single byte
|
||||
total = rom_file.read(4096)
|
||||
if not total:
|
||||
break
|
||||
value += len(total)
|
||||
mask_file.write(total)
|
||||
return value
|
||||
|
||||
def write_dummy(out_file, n):
|
||||
|
||||
# Write in blocks of 4096 bytes instead of 1 byte
|
||||
for _ in range(n//4096):
|
||||
out_file.write(b'\xff' * 4096)
|
||||
|
||||
# write remaining bytes
|
||||
out_file.write(b'\xff' * (n % 4096))
|
||||
|
||||
def main():
|
||||
if len(sys.argv) != 3:
|
||||
print("CopyRom ROM_file MASK_file", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
try:
|
||||
with open(sys.argv[1], 'rb') as rom_file, open(sys.argv[2], 'wb') as mask_file:
|
||||
total = get_files(rom_file, mask_file)
|
||||
if total < NBYTES:
|
||||
write_dummy(mask_file, NBYTES - total)
|
||||
sys.exit(1)
|
||||
except OSError as e:
|
||||
print(e, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
@@ -4,11 +4,11 @@ import sys
|
||||
|
||||
from file_util import FileUtil
|
||||
|
||||
VERSION = sys.argv[1]
|
||||
REGION = sys.argv[1]
|
||||
VERSION = sys.argv[2]
|
||||
|
||||
BUILD_DIR = 'build/' + VERSION
|
||||
MAP_FILEPATH = BUILD_DIR + '/dkr.map'
|
||||
ROM_FILEPATH = BUILD_DIR + '/dkr.z64'
|
||||
MAP_FILEPATH = f'build/dkr.{REGION}.{VERSION}.map'
|
||||
ROM_FILEPATH = f'build/dkr.{REGION}.{VERSION}.z64'
|
||||
|
||||
FUNCTIONS_TO_CALC = [
|
||||
# function checksum variable func size variable
|
||||
|
||||
@@ -11,8 +11,7 @@ args = parser.parse_args()
|
||||
version = args.version
|
||||
|
||||
extractConfigsDirectory = './extract-ver'
|
||||
assetsDirectory = './assets'
|
||||
ucodeDirectory = './assets'
|
||||
assetsDirectory = './assets/.vanilla'
|
||||
extractConfigFilename = extractConfigsDirectory + '/' + version + '.config.json'
|
||||
configChecksumFilename = extractConfigFilename + '.md5'
|
||||
|
||||
@@ -24,9 +23,7 @@ def do_extraction(reason):
|
||||
print(reason)
|
||||
|
||||
if not FileUtil.does_file_exist(assetsDirectory):
|
||||
do_extraction("Extracting because /assets/ directory does not exist.")
|
||||
elif not FileUtil.does_file_exist(ucodeDirectory):
|
||||
do_extraction("Extracting because /ucode/ directory does not exist.")
|
||||
do_extraction("Extracting because /assets/.vanilla/ directory does not exist.")
|
||||
elif not FileUtil.does_file_exist(configChecksumFilename):
|
||||
do_extraction('Extracting because "' + configChecksumFilename + '" does not exist.')
|
||||
else:
|
||||
@@ -44,8 +41,5 @@ def run_until_done(args, hide=False):
|
||||
if needToExtract:
|
||||
md5Calculated = hashlib.md5(FileUtil.get_bytes_from_file(extractConfigFilename)).hexdigest()
|
||||
FileUtil.write_text_to_file(configChecksumFilename, md5Calculated)
|
||||
run_until_done(['make', 'clean'])
|
||||
run_until_done(['rm', '-Rf', 'assets'])
|
||||
run_until_done(['rm', '-Rf', 'ucode'])
|
||||
run_until_done(['./extract.sh', version], True)
|
||||
|
||||
@@ -1,37 +0,0 @@
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
syms = None
|
||||
|
||||
def main():
|
||||
load_syms()
|
||||
index = 0
|
||||
while index < len(syms):
|
||||
index = test_index(index)
|
||||
print('Done!')
|
||||
|
||||
def load_syms():
|
||||
global syms
|
||||
with open('undefined_syms.txt', 'r') as inFile:
|
||||
syms = inFile.read().split('\n')
|
||||
|
||||
def save_syms():
|
||||
with open('undefined_syms.txt', 'w') as outFile:
|
||||
outFile.write('\n'.join(syms))
|
||||
|
||||
def test_index(index):
|
||||
val = syms.pop(index)
|
||||
save_syms()
|
||||
command = subprocess.run(['make', 'clean'], capture_output=True)
|
||||
command = subprocess.run(['make', '-j12'], capture_output=True)
|
||||
out = str(command.stdout)[-5:]
|
||||
if out.startswith('OK'):
|
||||
print(val + ' is NOT needed! Removing!')
|
||||
return index
|
||||
else:
|
||||
print(val + ' is needed! Keeping!')
|
||||
syms.insert(index, val)
|
||||
save_syms()
|
||||
return index + 1
|
||||
|
||||
main()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,49 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
def add_custom_arguments(parser):
|
||||
group = parser.add_mutually_exclusive_group(required=False)
|
||||
group.add_argument('-jp', dest='version', action='store_const', const='jp',
|
||||
help="Set version to JP.")
|
||||
group.add_argument('-us', dest='version', action='store_const', const='us',
|
||||
help="Set version to US.")
|
||||
group.add_argument('-eu', dest='version', action='store_const', const='eu',
|
||||
help="Set version to EU.")
|
||||
|
||||
def apply(config, args):
|
||||
version = 'us_1.0' # Right now only us_1.0 is supported
|
||||
config['mapfile'] = f'build/' + version + '/dkr.map'
|
||||
config['myimg'] = f'build/' + version + '/dkr.z64'
|
||||
config['baseimg'] = find_baserom(version)
|
||||
config['source_directories'] = ['src']
|
||||
|
||||
########################################################################################
|
||||
|
||||
from os import listdir
|
||||
from os.path import isfile, join
|
||||
|
||||
CRCS = {
|
||||
'us_1.0': (0x53D440E7, 0x7519B011)
|
||||
}
|
||||
|
||||
def get_filenames_from_directory(directory, extensions=None):
|
||||
if extensions is None:
|
||||
return [f for f in listdir(directory) if isfile(join(directory, f))]
|
||||
else:
|
||||
return [f for f in listdir(directory) if isfile(join(directory, f)) and f.endswith(extensions)]
|
||||
|
||||
def get_rom_crcs(romPath):
|
||||
with open(romPath, 'rb') as inFile:
|
||||
rom = inFile.read()
|
||||
crc1 = int.from_bytes(rom[0x10:0x14], "big")
|
||||
crc2 = int.from_bytes(rom[0x14:0x18], "big")
|
||||
return (crc1, crc2)
|
||||
|
||||
def find_baserom(version):
|
||||
romFiles = get_filenames_from_directory('baseroms/', ('.z64',))
|
||||
for romFilename in romFiles:
|
||||
romFilepath = 'baseroms/' + romFilename
|
||||
if get_rom_crcs(romFilepath) == CRCS[version]:
|
||||
return romFilepath
|
||||
raise Exception('Could not find a rom file for the version: "' + version + '"')
|
||||
|
||||
########################################################################################
|
||||
@@ -1,89 +0,0 @@
|
||||
import re
|
||||
|
||||
class ConfigRange:
|
||||
def __init__(self, size, type, properties):
|
||||
self.size = size
|
||||
self.type = type.lower()
|
||||
self.properties = properties
|
||||
self.start = -1
|
||||
|
||||
def get_range_string(self):
|
||||
return "{:06x}".format(self.start) + '-' + "{:06x}".format(self.start + self.size)
|
||||
|
||||
def __repr__(self):
|
||||
return "{:06x}".format(self.size) + ', ' + self.type + ', ' + str(self.properties)
|
||||
|
||||
class Config:
|
||||
def __init__(self, directory, filename):
|
||||
with open(directory + '/' + filename, 'r') as configFile:
|
||||
self.text = configFile.read()
|
||||
self.directory = directory
|
||||
self.ranges = []
|
||||
self.name = ''
|
||||
self.md5 = ''
|
||||
self.subfolder = ''
|
||||
self.notSupported = False
|
||||
if not self._parse(self.text):
|
||||
raise Exception('Error: ' + self.parseError)
|
||||
#print(self.name)
|
||||
#print(self.md5)
|
||||
#print(str(self.ranges))
|
||||
|
||||
def _parse(self, text):
|
||||
regex_property = r'^\s*([0-9a-zA-Z\-]+)\s*:\s*["]([^"]*)["]\s*$'
|
||||
regex_range = r'^\s*\[\s*(0x[0-9a-fA-F]+)\s*\]\s*:\s*(.*)$'
|
||||
for line in text.split('\n'):
|
||||
line = line.strip() # remove leading and trailing whitespace
|
||||
line = line.partition('#')[0] # remove comments
|
||||
if len(line) > 0:
|
||||
matches = self._parse_test_re(regex_property, line)
|
||||
if matches is not None:
|
||||
self._parse_property(matches)
|
||||
continue
|
||||
matches = self._parse_test_re(regex_range, line)
|
||||
if matches is not None:
|
||||
#print(matches)
|
||||
self._parse_range(matches)
|
||||
continue
|
||||
self.parseError = 'Invalid line "' + line + '"'
|
||||
return False
|
||||
return True
|
||||
|
||||
def _parse_property(self, matches):
|
||||
propertyName = matches[0]
|
||||
propertyValue = matches[1]
|
||||
if propertyName == 'config-name':
|
||||
self.name = propertyValue
|
||||
elif propertyName == 'subfolder':
|
||||
self.subfolder = propertyValue
|
||||
elif propertyName == 'checksum-md5':
|
||||
self.md5 = propertyValue
|
||||
elif propertyName == 'not-supported':
|
||||
self.notSupported = (propertyValue.lower() == 'true')
|
||||
elif propertyName == 'include':
|
||||
with open(self.directory + '/' + propertyValue, 'r') as includeFile:
|
||||
self._parse(includeFile.read())
|
||||
else:
|
||||
print('Unknown property "' + propertyName + '" with value "' + propertyValue + '"')
|
||||
|
||||
def _parse_range(self, matches):
|
||||
rangeSize = int(matches[0], 0)
|
||||
rangeProperties = matches[1].split(',')
|
||||
|
||||
for i in range(0, len(rangeProperties)):
|
||||
rangeProperties[i] = rangeProperties[i].strip() # remove leading and trailing whitespace
|
||||
rangeProperties[i] = self._parse_range_property(rangeProperties[i])
|
||||
|
||||
self.ranges.append(ConfigRange(rangeSize, rangeProperties[0], rangeProperties[1:]))
|
||||
|
||||
def _parse_range_property(self, property):
|
||||
if property.startswith('"'): # string
|
||||
return property[1:-1]
|
||||
else:
|
||||
return int(property, 0)
|
||||
|
||||
def _parse_test_re(self, regex, line):
|
||||
matches = re.match(regex, line)
|
||||
if matches is not None:
|
||||
return matches.groups()
|
||||
return None
|
||||
@@ -0,0 +1,56 @@
|
||||
#!/usr/bin/python3
|
||||
|
||||
import sys
|
||||
import os
|
||||
import zlib
|
||||
|
||||
def crc32(data: bytes) -> int:
|
||||
return zlib.crc32(data) & 0xFFFFFFFF
|
||||
|
||||
def N64GetCIC(data: bytes) -> int:
|
||||
crc = crc32(data)
|
||||
return {
|
||||
0x6170A4A1: 6101,
|
||||
0x90BB6CB5: 6102,
|
||||
0x0B050EE0: 6103,
|
||||
0x98BC2C86: 6105,
|
||||
0xACC8580A: 6106,
|
||||
}.get(crc, 0)
|
||||
|
||||
def boot_extract(input_file, output_file="boot_custom.bin"):
|
||||
mods_missing = True
|
||||
cictype = 6103
|
||||
if not (input_file.endswith(".z64")):
|
||||
print("ROM file must be .z64. Use https://hack64.net/tools/swapper.php to convert.")
|
||||
return
|
||||
|
||||
with open(input_file, "rb") as f:
|
||||
f.seek(0x40)
|
||||
data = f.read(0xFC0)
|
||||
|
||||
if os.path.isdir('mods'):
|
||||
print("Mods folder detected, outputting there.")
|
||||
output_file = "./mods/boot_custom.bin"
|
||||
mods_missing = False
|
||||
cictype = N64GetCIC(data)
|
||||
if os.path.isfile("makefile"):
|
||||
print("Makefile detected, modifying CIC type.")
|
||||
with open("makefile", 'r') as f:
|
||||
lines = f.readlines()
|
||||
with open("makefile", 'w') as f:
|
||||
for line in lines:
|
||||
if line.startswith("BOOT_CIC ?="):
|
||||
f.write("BOOT_CIC ?= " + str(cictype) + "\n")
|
||||
else:
|
||||
f.write(line)
|
||||
|
||||
|
||||
with open(output_file, "wb") as f:
|
||||
f.write(data)
|
||||
|
||||
if (mods_missing == True):
|
||||
print("Finished! Copy boot_custom.bin to the mods folder and change BOOT_CIC in the makefile")
|
||||
else:
|
||||
print("Finished!")
|
||||
|
||||
boot_extract(sys.argv[1])
|
||||
@@ -1,54 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# SPDX-FileCopyrightText: © 2022 AngheloAlf
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import mapfile_parser
|
||||
from pathlib import Path
|
||||
import rabbitizer
|
||||
|
||||
|
||||
def decodeInstruction(bytesDiff: bytes, mapFile: mapfile_parser.MapFile) -> str:
|
||||
word = (bytesDiff[0] << 24) | (bytesDiff[1] << 16) | (bytesDiff[2] << 8) | (bytesDiff[3] << 0)
|
||||
instr = rabbitizer.Instruction(word)
|
||||
immOverride = None
|
||||
|
||||
if instr.isJumpWithAddress():
|
||||
# Instruction is a function call (jal)
|
||||
|
||||
# Get the embedded address of the function call
|
||||
symAddress = instr.getInstrIndexAsVram()
|
||||
|
||||
# Search for the address in the mapfile
|
||||
symInfo = mapFile.findSymbolByVramOrVrom(symAddress)
|
||||
if symInfo is not None:
|
||||
# Use the symbol from the mapfile instead of a raw value
|
||||
immOverride = symInfo.symbol.name
|
||||
|
||||
return instr.disassemble(immOverride=immOverride, extraLJust=-20)
|
||||
|
||||
def firstDiffMain():
|
||||
parser = argparse.ArgumentParser(description="Find the first difference(s) between the built ROM and the base ROM.")
|
||||
|
||||
parser.add_argument("-c", "--count", type=int, default=5, help="find up to this many instruction difference(s)")
|
||||
parser.add_argument("-r", "--region", help="Which region should be processed", default="us")
|
||||
parser.add_argument("-v", "--version", help="Which version should be processed", default="v77")
|
||||
parser.add_argument("-a", "--add-colons", action='store_true', help="Add colon between bytes" )
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
buildFolder = Path("build") / Path("us_1.0")
|
||||
|
||||
BUILTROM = buildFolder / f"dkr.z64"
|
||||
BUILTMAP = buildFolder / f"dkr.map"
|
||||
|
||||
EXPECTEDROM = "expected" / BUILTROM
|
||||
EXPECTEDMAP = "expected" / BUILTMAP
|
||||
|
||||
mapfile_parser.frontends.first_diff.doFirstDiff(BUILTMAP, EXPECTEDMAP, BUILTROM, EXPECTEDROM, args.count, mismatchSize=True, addColons=args.add_colons, bytesConverterCallback=decodeInstruction)
|
||||
|
||||
if __name__ == "__main__":
|
||||
firstDiffMain()
|
||||
@@ -38,7 +38,7 @@ def holecount_all(src):
|
||||
print(filename, holes, nonmatches)
|
||||
return 1
|
||||
|
||||
paths = ('src', 'lib')
|
||||
paths = ('src', 'libultra')
|
||||
|
||||
# get file encoding type
|
||||
def get_encoding_type(file):
|
||||
@@ -68,6 +68,6 @@ for directory in chain.from_iterable(os.walk(path) for path in paths):
|
||||
if (nonm_count + noneq_count + global_asm_count) == 0 :
|
||||
outfile.write(" $(BUILD_DIR)/")
|
||||
outfile.write(os.path.splitext(filename)[0])
|
||||
outfile.write(".o \\\n")
|
||||
outfile.write(".c.o \\\n")
|
||||
infile.close()
|
||||
outfile.close()
|
||||
@@ -1,69 +0,0 @@
|
||||
import sys
|
||||
import json
|
||||
import re
|
||||
import os
|
||||
import hashlib
|
||||
import time
|
||||
import subprocess
|
||||
|
||||
from file_util import FileUtil
|
||||
|
||||
INCLUDE_DIRECTORY = './include'
|
||||
ASSETS_INCLUDE = INCLUDE_DIRECTORY + '/asset_sections.h'
|
||||
|
||||
ASSETS_JSON_FILENAME = 'assets.meta.json'
|
||||
|
||||
class GenerateAssets:
|
||||
def __init__(self, rootDir, version):
|
||||
self.ASSETS_ASM_DIR = rootDir + '/asm/assets'
|
||||
self.ASSETS_FILENAME = self.ASSETS_ASM_DIR + '/assets.s'
|
||||
self.ASSETS_DIR = rootDir + '/assets'
|
||||
self.UCODE_DIR = rootDir + '/ucode/' + version
|
||||
self.UCODE_TEXT_FILENAME = self.ASSETS_ASM_DIR + '/ucode_text.s'
|
||||
self.UCODE_DATA_FILENAME = self.ASSETS_ASM_DIR + '/ucode_data.s'
|
||||
self.BUILD_DIR = rootDir + '/build'
|
||||
self.VERSION = version
|
||||
|
||||
self.generate_assets_file()
|
||||
self.generate_ucode_files()
|
||||
|
||||
def generate_assets_file(self):
|
||||
with open(self.ASSETS_DIR + "/" + self.VERSION + '/ignoreMe.txt', 'w') as a_file:
|
||||
a_file.write("This file is automatically generated by generate_assets.py, and is used to check if the assets have been updated.")
|
||||
args = ["tools/dkr_assets_tool", "-i", self.VERSION, self.ASSETS_DIR, INCLUDE_DIRECTORY, self.BUILD_DIR, self.ASSETS_ASM_DIR]
|
||||
createAssets = subprocess.Popen(args, stdout=subprocess.PIPE)
|
||||
createAssets.wait()
|
||||
streamdata = createAssets.communicate()[0]
|
||||
if createAssets.returncode != 0:
|
||||
raise SystemExit("An error occured while generating /asm/assets files. Error code " + str(createAssets.returncode) + ". Aborting!")
|
||||
with open(self.ASSETS_DIR + '/' + self.VERSION + '/' + ASSETS_JSON_FILENAME) as jsonFile:
|
||||
assetsJSON = json.load(jsonFile)
|
||||
self.numAssets = len(assetsJSON['assets']['order'])
|
||||
|
||||
# Note: I am assuming that the order of microcodes did not change between versions.
|
||||
# TODO: Will probably need to refactor this later.
|
||||
def generate_ucode_files(self):
|
||||
assetsUCodeText = '# This file was generated by generate_ld.py\n\n'
|
||||
assetsUCodeText += '.include "macros.inc"\n\n'
|
||||
assetsUCodeText += self.generate_ucode_file('aspMainTextStart', 'ucode/ucode_audio.bin')
|
||||
assetsUCodeText += self.generate_ucode_file('rspF3DDKRBootStart', 'ucode/ucode_boot.bin')
|
||||
assetsUCodeText += self.generate_ucode_file('rspF3DDKRDramStart', 'ucode/ucode_f3ddkr_dram.bin')
|
||||
assetsUCodeText += self.generate_ucode_file('rspF3DDKRFifoStart', 'ucode/ucode_f3ddkr_fifo.bin')
|
||||
assetsUCodeText += self.generate_ucode_file('rspF3DDKRXbusStart', 'ucode/ucode_f3ddkr_xbus.bin')
|
||||
assetsUCodeText += self.generate_ucode_file('rspUnknown2Start', 'ucode/ucode_unknown_2.bin')
|
||||
with open(self.UCODE_TEXT_FILENAME, "w") as assetsFile:
|
||||
assetsFile.write(assetsUCodeText)
|
||||
|
||||
assetsUCodeData = '# This file was generated by generate_ld.py\n\n'
|
||||
assetsUCodeData += '.include "macros.inc"\n\n'
|
||||
assetsUCodeData += self.generate_ucode_file('aspMainDataStart', 'ucode/data_audio.bin')
|
||||
assetsUCodeData += self.generate_ucode_file('rspF3DDKRDataDramStart', 'ucode/data_f3ddkr_dram.bin')
|
||||
assetsUCodeData += self.generate_ucode_file('rspF3DDKRDataFifoStart', 'ucode/data_f3ddkr_fifo.bin')
|
||||
assetsUCodeData += self.generate_ucode_file('rspF3DDKRDataXbusStart', 'ucode/data_f3ddkr_xbus.bin')
|
||||
assetsUCodeData += self.generate_ucode_file('rspUnknown2DataStart', 'ucode/data_unknown_2.bin')
|
||||
|
||||
with open(self.UCODE_DATA_FILENAME, "w") as assetsFile:
|
||||
assetsFile.write(assetsUCodeData)
|
||||
|
||||
def generate_ucode_file(self, label, path):
|
||||
return 'glabel ' + label + '\n.incbin "' + self.BUILD_DIR + '/' + self.VERSION + '/' + path + '"\n'
|
||||
+25
-32
@@ -1,37 +1,36 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import git
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from colour import Color
|
||||
|
||||
script_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
root_dir = os.path.join(script_dir, "..", "..")
|
||||
asm_dir = os.path.join(root_dir, "asm", "non_matchings")
|
||||
asm_lib_dir = os.path.join(root_dir, "lib", "asm", "non_matchings")
|
||||
build_dir = os.path.join(root_dir, "build", "us_1.0")
|
||||
elf_path = os.path.join(build_dir, "dkr.elf")
|
||||
asm_dir = os.path.join(root_dir, "asm")
|
||||
build_dir = os.path.join(root_dir, "build")
|
||||
elf_path = os.path.join(build_dir, "dkr.us.v77.elf")
|
||||
|
||||
def get_func_sizes():
|
||||
try:
|
||||
result = subprocess.run(['objdump', '-x', elf_path], stdout=subprocess.PIPE)
|
||||
result = subprocess.run(["objdump", "-x", elf_path], stdout=subprocess.PIPE)
|
||||
nm_lines = result.stdout.decode().split("\n")
|
||||
except:
|
||||
print(f"Error: Could not run objdump on {elf_path} - make sure that the project is built")
|
||||
print(
|
||||
f"Error: Could not run objdump on {elf_path} - make sure that the project is built"
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
sizes = {}
|
||||
total = 0
|
||||
|
||||
for line in nm_lines:
|
||||
if " F " in line:
|
||||
#This will filter out "weak" functions like fcos.
|
||||
if "g F" in line or "l F" in line:
|
||||
components = line.split()
|
||||
size = int(components[4], 16)
|
||||
name = components[5]
|
||||
#Labels are coming through here as functions, and this finds and ignores them
|
||||
if name[0:4] != "L800":
|
||||
if not name.startswith(".L") or not name.startswith("D_"):
|
||||
total += size
|
||||
sizes[name] = size
|
||||
|
||||
@@ -42,12 +41,7 @@ def get_nonmatching_funcs():
|
||||
|
||||
for root, dirs, files in os.walk(asm_dir):
|
||||
for f in files:
|
||||
if f.endswith(".s"):
|
||||
funcs.add(f[:-2])
|
||||
|
||||
for root, dirs, files in os.walk(asm_lib_dir):
|
||||
for f in files:
|
||||
if f.endswith(".s"):
|
||||
if f.endswith(".s") and not f.startswith(".L"):
|
||||
funcs.add(f[:-2])
|
||||
|
||||
return funcs
|
||||
@@ -65,7 +59,6 @@ def get_funcs_sizes(sizes, matchings, nonmatchings):
|
||||
# print(func)
|
||||
else:
|
||||
nmsize += sizes[func]
|
||||
#print("% s,%i" % (func, sizes[func]))
|
||||
|
||||
return msize, nmsize
|
||||
|
||||
@@ -79,7 +72,9 @@ def main(args):
|
||||
nonmatching_funcs = get_nonmatching_funcs()
|
||||
matching_funcs = all_funcs - nonmatching_funcs
|
||||
|
||||
matching_size, nonmatching_size = get_funcs_sizes(func_sizes, matching_funcs, nonmatching_funcs)
|
||||
matching_size, nonmatching_size = get_funcs_sizes(
|
||||
func_sizes, matching_funcs, nonmatching_funcs
|
||||
)
|
||||
|
||||
if len(all_funcs) == 0:
|
||||
funcs_matching_ratio = 0.0
|
||||
@@ -88,16 +83,9 @@ def main(args):
|
||||
funcs_matching_ratio = (len(matching_funcs) / len(all_funcs)) * 100
|
||||
matching_ratio = (matching_size / total_size) * 100
|
||||
|
||||
if args.csv:
|
||||
version = 1
|
||||
git_object = git.Repo().head.object
|
||||
timestamp = str(git_object.committed_date)
|
||||
git_hash = git_object.hexsha
|
||||
csv_list = [str(version), timestamp, git_hash, str(len(all_funcs)), str(len(nonmatching_funcs)),
|
||||
str(len(matching_funcs)), str(total_size), str(nonmatching_size), str(matching_size)]
|
||||
print(",".join(csv_list))
|
||||
elif args.shield_json:
|
||||
if args.shield_json:
|
||||
import json
|
||||
from colour import Color
|
||||
|
||||
# https://shields.io/endpoint
|
||||
color = Color("#50ca22", hue=lerp(0, 105/255, matching_ratio / 100))
|
||||
@@ -110,13 +98,18 @@ def main(args):
|
||||
else:
|
||||
if matching_size + nonmatching_size != total_size:
|
||||
print("Warning: category/total size mismatch!\n")
|
||||
print(f"{len(matching_funcs)} matched functions / {len(all_funcs)} total ({funcs_matching_ratio:.2f}%)")
|
||||
print(f"{matching_size} matching bytes / {total_size} total ({matching_ratio:.2f}%)")
|
||||
print(
|
||||
f"{len(matching_funcs)} matched functions / {len(all_funcs)} total ({funcs_matching_ratio:.2f}%)"
|
||||
)
|
||||
print(
|
||||
f"{matching_size} matching bytes / {total_size} total ({matching_ratio:.2f}%)"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="Reports progress for the project")
|
||||
parser.add_argument("--csv", action="store_true")
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Calculate the progress of the project"
|
||||
)
|
||||
parser.add_argument("--shield-json", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
@@ -1,252 +0,0 @@
|
||||
import os
|
||||
import re
|
||||
import argparse
|
||||
from file_util import FileUtil
|
||||
|
||||
C_RODATA_REGEX = r"/[*]+[ ]*[.]rodata[ ]*[*]+/(?:.*\n)+/[*]{33}/"
|
||||
C_STRING_REGEX = r"const[ ]+char[ ]+(D_[0-9A-Fa-f]{8})[[][^]]*[]][ ]*=[ ]*(\"[^\"]*\")[ ]*;"
|
||||
C_FLOAT_REGEX = r".*(D_[0-9A-Fa-f]{8})[ ]*=[ ]*{[ ]*([-]?[0-9]+.[0-9]+)[f]?[ ]*}.*"
|
||||
C_JMPTABLE_REGEX = r"(?:(?:D_)|0x)[0-9A-Fa-f]{8}"
|
||||
|
||||
# Debugging variables
|
||||
NO_OUTPUT = False # If True, then no files will be changed.
|
||||
ONLY_THE_FIRST_ASM_FILE = False # If True, then only one asm file will be changed.
|
||||
|
||||
ASM_DIRECTORY = 'asm/non_matchings/'
|
||||
#ASM_DIRECTORY = 'lib/asm/non_matchings/'
|
||||
|
||||
def getMatches(string, regex):
|
||||
out = []
|
||||
matches = re.finditer(regex, string, re.MULTILINE)
|
||||
for matchNum, match in enumerate(matches, start=1):
|
||||
start = match.start()
|
||||
end = match.end()
|
||||
out.append((string[start:end], match.groups(), (start, end)))
|
||||
return out
|
||||
|
||||
def getLineType(line):
|
||||
if line.lower().startswith('const char '):
|
||||
return 1 # Literal string
|
||||
elif line.lower().startswith('const floatliteral '):
|
||||
return 2 # Literal float
|
||||
elif line.lower().startswith('const doubleliteral '):
|
||||
return 3 # Literal double
|
||||
elif line.lower().startswith('};'):
|
||||
return 4 # Jump Table bottom
|
||||
elif line.lower().startswith('const u32 '):
|
||||
return 5 # Jump Table top
|
||||
else:
|
||||
return 0
|
||||
|
||||
def getAsmFileReferenceForLabel(filename, label):
|
||||
possibleFiles = []
|
||||
search = os.popen('fgrep -r "' + label + '"').read().split('\n')
|
||||
for line in search:
|
||||
if ASM_DIRECTORY in line:
|
||||
asmfilePath = line.split(':')[0]
|
||||
if asmfilePath not in possibleFiles:
|
||||
possibleFiles.append(asmfilePath)
|
||||
if len(possibleFiles) > 1:
|
||||
raise Exception('Cannot determine asm file for label: ' + label)
|
||||
elif len(possibleFiles) == 0:
|
||||
raise Exception(label + ' has no asm files associated with it!')
|
||||
return possibleFiles[0]
|
||||
|
||||
def adjustAsmFile(asmFilepath, refs):
|
||||
with open(asmFilepath, 'r') as inFile:
|
||||
asmFile = inFile.read()
|
||||
if '.late_rodata' in asmFile:
|
||||
raise Exception('Cannot add to file "' + asmFilepath + '" since it already has .late_rodata in it!')
|
||||
asmLines = asmFile.split('\n')
|
||||
replaces = []
|
||||
|
||||
rodata = ''
|
||||
lateRodata = ''
|
||||
|
||||
for index, ref in enumerate(refs):
|
||||
if ref[0] == 'asciz':
|
||||
rodata += 'glabel ' + ref[1] + '\n'
|
||||
rodata += '.' + ref[0] + ' ' + ref[2] + '\n'
|
||||
strEnd = int(ref[1][-1:], 16) + len(ref[2].replace('\\', '')) - 2 + 1
|
||||
if strEnd % 4 != 0:
|
||||
rodata += '.ascii "' + ('\\0' * (4 - (strEnd % 4))) + '" # padding\n'
|
||||
elif ref[0] == 'double' or ref[0] == 'float':
|
||||
lateRodata += 'glabel ' + ref[1] + '\n'
|
||||
lateRodata += '.' + ref[0] + ' ' + ref[2] + '\n'
|
||||
elif ref[0] == 'table':
|
||||
replaces.append((ref[1], 'jpt' + ref[1][1:]))
|
||||
tblWords = '.word L' + ref[2][0]
|
||||
for word in ref[2][1:]:
|
||||
tblWords += ', L' + word
|
||||
doneWords = []
|
||||
for word in ref[2]:
|
||||
if word in doneWords:
|
||||
continue
|
||||
for i in range(0, len(asmLines)):
|
||||
line = asmLines[i]
|
||||
if line.startswith('/* '):
|
||||
lineParts = line.split(' ')
|
||||
if lineParts[2] == word:
|
||||
asmLines.insert(i, 'glabel L' + word)
|
||||
break
|
||||
doneWords.append(word)
|
||||
lateRodata += 'glabel ' + ref[1] + '\n'
|
||||
lateRodata += tblWords + '\n'
|
||||
if index < len(refs) - 1 and ref[0] != 'double' and refs[index + 1][0] == 'double':
|
||||
addPadding = False
|
||||
if ref[0] == 'float':
|
||||
addPadding = ref[1].endswith('0') or ref[1].endswith('8')
|
||||
elif ref[0] == 'table':
|
||||
tableSizeAlignment = len(ref[2]) % 4
|
||||
if ref[1].endswith('0') or ref[1].endswith('8'):
|
||||
addPadding = tableSizeAlignment == 1 or tableSizeAlignment == 3
|
||||
else:
|
||||
addPadding = tableSizeAlignment == 0 or tableSizeAlignment == 2
|
||||
if addPadding:
|
||||
lateRodata += '.word 0 # Padding\n' # Insert padding before the next double.
|
||||
outData = ''
|
||||
if len(rodata) > 0:
|
||||
outData += '.rdata\n' + rodata + '\n'
|
||||
if len(lateRodata) > 0:
|
||||
outData += '.late_rodata\n' + lateRodata + '\n'
|
||||
asmLines.insert(0, outData + '.text');
|
||||
asmOut = '\n'.join(asmLines)
|
||||
for rep in replaces:
|
||||
asmOut = asmOut.replace(rep[0], rep[1])
|
||||
# print(asmOut)
|
||||
if not NO_OUTPUT:
|
||||
with open(asmFilepath, 'w') as outFile:
|
||||
outFile.write(asmOut)
|
||||
#print(asmFilepath)
|
||||
#print(refs)
|
||||
|
||||
def convertFile(cFilepath):
|
||||
filename = cFilepath[cFilepath.rindex('/') + 1:-2]
|
||||
print(filename)
|
||||
with open(cFilepath, 'r') as inFile:
|
||||
cFile = inFile.read()
|
||||
match = getMatches(cFile, C_RODATA_REGEX)
|
||||
if len(match) == 0:
|
||||
print('No .rodata in file "' + cFilepath + '"')
|
||||
return
|
||||
rodataPart = match[0][0]
|
||||
rodataSection = match[0][2]
|
||||
rodataLines = rodataPart.split('\n')
|
||||
tableBottom = -1
|
||||
hasProcessed = False
|
||||
stoppedPremature = False
|
||||
curLine = len(rodataLines) - 1
|
||||
bottomLine = curLine
|
||||
asmData = []
|
||||
for line in reversed(rodataLines):
|
||||
lineType = getLineType(line)
|
||||
if lineType == 0:
|
||||
pass
|
||||
elif lineType == 1: # Literal string
|
||||
try:
|
||||
strMatch = getMatches(line, C_STRING_REGEX)[0]
|
||||
strLabel = strMatch[1][0]
|
||||
strValue = strMatch[1][1]
|
||||
# Check if the string has a reference somewhere
|
||||
getAsmFileReferenceForLabel(filename, strLabel)
|
||||
hasProcessed = True
|
||||
bottomLine = curLine
|
||||
asmData.insert(0, ('asciz', strLabel, strValue, bottomLine))
|
||||
except:
|
||||
print('Currently cannot process literal string: ')
|
||||
print(line)
|
||||
stoppedPremature = True
|
||||
break
|
||||
elif lineType == 2: # Literal float
|
||||
floatMatch = getMatches(line, C_FLOAT_REGEX)[0]
|
||||
floatLabel = floatMatch[1][0]
|
||||
floatValue = floatMatch[1][1]
|
||||
if floatValue == '0.0':
|
||||
print('File boundary hit!')
|
||||
stoppedPremature = True
|
||||
break
|
||||
hasProcessed = True
|
||||
bottomLine = curLine
|
||||
asmData.insert(0, ('float', floatLabel, floatValue, bottomLine))
|
||||
elif lineType == 3: # Literal double
|
||||
floatMatch = getMatches(line, C_FLOAT_REGEX)[0]
|
||||
floatLabel = floatMatch[1][0]
|
||||
floatValue = floatMatch[1][1]
|
||||
if floatValue == '0.0':
|
||||
print('File boundary hit!')
|
||||
stoppedPremature = True
|
||||
break
|
||||
hasProcessed = True
|
||||
bottomLine = curLine
|
||||
asmData.insert(0, ('double', floatLabel, floatValue, bottomLine))
|
||||
elif lineType == 4: # Jump table bottom
|
||||
tableBottom = curLine
|
||||
elif lineType == 5: # Jump table top
|
||||
tblMatch = getMatches('\n'.join(rodataLines[curLine:tableBottom+1]), C_JMPTABLE_REGEX)
|
||||
tblLabel = tblMatch[0][0]
|
||||
tblValues = []
|
||||
for val in range(1, len(tblMatch)):
|
||||
tblValues.append(tblMatch[val][0][2:])
|
||||
hasProcessed = True
|
||||
bottomLine = curLine
|
||||
asmData.insert(0, ('table', tblLabel, tblValues, bottomLine))
|
||||
tableBottom = -1
|
||||
else:
|
||||
print(line)
|
||||
curLine -= 1
|
||||
asmRefs = {}
|
||||
asmBottomLine = 999999
|
||||
for i in reversed(range(0, len(asmData))):
|
||||
data = asmData[i]
|
||||
try:
|
||||
ref = getAsmFileReferenceForLabel(filename, data[1])
|
||||
if ref not in asmRefs:
|
||||
asmRefs[ref] = []
|
||||
asmRefs[ref].append(data)
|
||||
asmBottomLine = min(asmBottomLine, data[3])
|
||||
except Exception as e:
|
||||
# raise e
|
||||
print(e)
|
||||
if asmBottomLine != 999999:
|
||||
print(bottomLine, asmBottomLine)
|
||||
bottomLine = asmBottomLine
|
||||
stoppedPremature = True
|
||||
break
|
||||
for asmFilepath in asmRefs:
|
||||
print('Processing: ' + asmFilepath)
|
||||
asmRefs[asmFilepath].reverse()
|
||||
asmBottomLine = 999999
|
||||
adjustAsmFile(asmFilepath, asmRefs[asmFilepath])
|
||||
for ref in asmRefs[asmFilepath]:
|
||||
asmBottomLine = min(asmBottomLine, ref[3])
|
||||
if ONLY_THE_FIRST_ASM_FILE:
|
||||
print(bottomLine, asmBottomLine)
|
||||
bottomLine = asmBottomLine
|
||||
stoppedPremature = True
|
||||
break
|
||||
# print(asmRefs)
|
||||
if hasProcessed and not NO_OUTPUT:
|
||||
newRodata = '\n'.join(rodataLines[0:bottomLine] + rodataLines[-2:])
|
||||
newFileText = cFile[:rodataSection[0]] + (newRodata if stoppedPremature else '') + cFile[rodataSection[1]:]
|
||||
with open(cFilepath, 'w') as outFile:
|
||||
outFile.write(newFileText)
|
||||
print('Changes were made to ' + cFilepath)
|
||||
else:
|
||||
print('No changes were made to ' + cFilepath)
|
||||
|
||||
|
||||
parser = argparse.ArgumentParser(description="")
|
||||
parser.add_argument("file", help=".c file that contains rodata to move.")
|
||||
parser.add_argument("-s", "--single", help="Only do one ASM function in the file", action='store_true')
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.single is not None and args.single:
|
||||
ONLY_THE_FIRST_ASM_FILE = True
|
||||
|
||||
if args.file.endswith('.c'):
|
||||
convertFile(args.file)
|
||||
else:
|
||||
raise Exception('The file must be a .c file.')
|
||||
|
||||
# files = FileUtil.get_filenames_from_directory('src', ('.c',))
|
||||
#
|
||||
@@ -1,58 +0,0 @@
|
||||
import hashlib
|
||||
|
||||
from file_util import FileUtil
|
||||
|
||||
class ROM:
|
||||
def __init__(self, filename):
|
||||
with open(filename, 'rb') as romFile:
|
||||
self.fixedRomEndianess = False
|
||||
self.bytearray = romFile.read()
|
||||
self.bytes = list(self.bytearray)
|
||||
self.size = len(self.bytes)
|
||||
self._test_endianness()
|
||||
self.md5 = hashlib.md5(self.bytearray).hexdigest()
|
||||
if self.fixedRomEndianess:
|
||||
FileUtil.delete_file(filename)
|
||||
# Save the ROM as big-endian
|
||||
with open(filename[:-4] + '.z64', 'wb') as romFile:
|
||||
romFile.write(self.bytearray)
|
||||
|
||||
def get_bytes_from_range(self, start, end):
|
||||
return self.bytes[start:end]
|
||||
|
||||
# Check and correct mixed/little endian ROMs
|
||||
def _test_endianness(self):
|
||||
if self.bytes[0] == 0x80 and self.bytes[1] == 0x37:
|
||||
self.endianness = 'big'
|
||||
elif self.bytes[0] == 0x37 and self.bytes[1] == 0x80:
|
||||
self.endianness = 'mixed'
|
||||
elif self.bytes[0] == 0x40 and self.bytes[1] == 0x12:
|
||||
self.endianness = 'little'
|
||||
|
||||
# Convert mixed/little endian to big endian.
|
||||
# This is kinda slow, and should be avoided if possible.
|
||||
if self.endianness == 'mixed':
|
||||
print('Converting mixed-endian (byte-swapped) to big-endian...')
|
||||
for i in range(0, len(self.bytes), 2):
|
||||
temp = self.bytes[i]
|
||||
self.bytes[i] = self.bytes[i + 1]
|
||||
self.bytes[i + 1] = temp
|
||||
self.bytearray = bytearray(self.bytes)
|
||||
self.endianness = 'big'
|
||||
self.fixedRomEndianess = True
|
||||
pass
|
||||
elif self.endianness == 'little':
|
||||
print('Converting little-endian to big-endian...')
|
||||
temp = [0, 0, 0, 0]
|
||||
for i in range(0, len(self.bytes), 4):
|
||||
temp[0] = self.bytes[i]
|
||||
temp[1] = self.bytes[i + 1]
|
||||
temp[2] = self.bytes[i + 2]
|
||||
temp[3] = self.bytes[i + 3]
|
||||
self.bytes[i] = temp[3]
|
||||
self.bytes[i + 1] = temp[2]
|
||||
self.bytes[i + 2] = temp[1]
|
||||
self.bytes[i + 3] = temp[0]
|
||||
self.bytearray = bytearray(self.bytes)
|
||||
self.endianness = 'big'
|
||||
self.fixedRomEndianess = True
|
||||
+30
-14
@@ -8,13 +8,20 @@ from score_display import ScoreDisplay
|
||||
|
||||
ASM_FOLDERS = [
|
||||
'./asm',
|
||||
'./lib/asm',
|
||||
'./src/hasm',
|
||||
'./libultra/src/gu',
|
||||
'./libultra/src/libc',
|
||||
'./libultra/src/os',
|
||||
]
|
||||
|
||||
BLACKLIST = [
|
||||
'/non_matchings/',
|
||||
'/nonmatchings/',
|
||||
'/assets/',
|
||||
'/boot/'
|
||||
'/boot/',
|
||||
'/data/',
|
||||
'/header.s',
|
||||
'/llmuldiv_gcc.s',
|
||||
'/libm_vals.s'
|
||||
]
|
||||
|
||||
BLACKLIST_C = [
|
||||
@@ -38,7 +45,7 @@ for asmDir in ASM_FOLDERS:
|
||||
|
||||
# These will automatically be added to the adventure one percentage.
|
||||
ASM_LABELS = []
|
||||
GLABEL_REGEX = r'glabel ([0-9A-Za-z_]+)'
|
||||
GLABEL_REGEX = r'glabel|leaf ([0-9A-Za-z_]+)'
|
||||
for filename in filelist:
|
||||
with open(filename, 'r') as asmFile:
|
||||
text = asmFile.read()
|
||||
@@ -48,12 +55,12 @@ for filename in filelist:
|
||||
if not glabel in ASM_LABELS:
|
||||
ASM_LABELS.append(glabel)
|
||||
|
||||
BUILD_DIRECTORY = './build/us_1.0'
|
||||
BUILD_DIRECTORY = './build'
|
||||
SRC_DIRECTORY = './src'
|
||||
LIB_SRC_DIRECTORY = './lib/src'
|
||||
FUNCTION_REGEX = r'^(?<!static\s)(?:(\/[*][*!][*]*\n(?:[^/]*\n)+?\s*[*]\/\n)(?:\s*)*?)?(?:\s*UNUSED\s+)?([^\s]+)\s(?:\s|[*])*?([0-9A-Za-z_]+)\s*[(][^)]*[)]\s*{'
|
||||
GLOBAL_ASM_REGEX = r'GLOBAL_ASM[(]".*(?=\/)\/([^.]+).s"[)]'
|
||||
WIP_REGEX = r'#ifdef\s+(?:NON_MATCHING|NON_EQUIVALENT)(?:.|\n)*?#else\s*(GLOBAL_ASM[(][^)]*[)])(.|\n)*?#endif'
|
||||
LIB_SRC_DIRECTORY = './libultra/src'
|
||||
FUNCTION_REGEX = r'^(?<!static\s)(?:(\/[*][*!][*]*\n(?:[^\/]*\n)+?\s*[*]\/\n)(?:\s*)*?)?(?:\s*UNUSED\s+)?([^\s]+)\s(?:\s|[*])*?([0-9A-Za-z_]+)\s*[(][^)]*[)]\s*{'
|
||||
GLOBAL_ASM_REGEX = r'\#pragma\sGLOBAL_ASM[(]".*(?=\/)\/([^.]+).s"[)]'
|
||||
WIP_REGEX = r'ifdef\s+(?:NON_MATCHING|NON_EQUIVALENT)(?:.|\n)*?\#else\s*(\#pragma\sGLOBAL_ASM[(][^)]*[)])(.|\n)*?'
|
||||
NON_MATCHING_REGEX = re.compile(r'^#ifdef[ ]+NON_MATCHING(?:.|\n)*?(?:\s*UNUSED\s+)?(?:[^\s]+)\s(?:\s|[*])*?([0-9A-Za-z_]+)\s*[(][^)]*[)]\s*{', re.MULTILINE)
|
||||
NON_EQUVIALENT_REGEX = re.compile(r'^#ifdef[ ]+NON_EQUIVALENT', re.MULTILINE)
|
||||
|
||||
@@ -64,15 +71,20 @@ CODE_SIZE = CODE_END - CODE_START
|
||||
class DkrMapFile:
|
||||
def __init__(self):
|
||||
try:
|
||||
with open(BUILD_DIRECTORY + '/dkr.map', 'r') as mapFile:
|
||||
with open(BUILD_DIRECTORY + '/dkr.us.v77.map', 'r') as mapFile:
|
||||
self.functionSizes = {}
|
||||
functions = []
|
||||
lines = mapFile.read().split('\n')
|
||||
for line in lines:
|
||||
if line.startswith(' 0x8'):
|
||||
lineSet = 0
|
||||
if line.startswith(' 0x00000000'):
|
||||
lineSet = 26
|
||||
elif line.startswith(' 0x8'):
|
||||
lineSet = 18
|
||||
if (lineSet != 0):
|
||||
if '=' in line:
|
||||
line = line[0:line.find('=')-1]
|
||||
address = int(line[18:18+8], 16)
|
||||
address = int(line[lineSet:lineSet+8], 16)
|
||||
if address >= CODE_START and address < CODE_END:
|
||||
symbol = line[line.rfind(' ')+1:]
|
||||
if (not symbol.startswith(".L") and not symbol.startswith("L800")
|
||||
@@ -89,14 +101,18 @@ class DkrMapFile:
|
||||
|
||||
|
||||
def contains_forbidden_func(self, string):
|
||||
for forbidden in ['__FUNC_RAM_START', 'cosf', 'sinf']:
|
||||
for forbidden in ['__FUNC_RAM_START', 'cosf', 'sinf', 'main_VRAM', 'main_TEXT_START', '.']:
|
||||
if forbidden in string:
|
||||
return True
|
||||
return False
|
||||
|
||||
MAP_FILE = DkrMapFile()
|
||||
|
||||
NOT_FUNCTION_NAMES = ['if', 'else', 'switch', 'while', 'for']
|
||||
# Adding regional and other version functions here to ignore since we're only scoring US_1.0 for now.
|
||||
NOT_FUNCTION_NAMES = ['if', 'else', 'switch', 'while', 'for', 'dmacopy_internal', 'func_80082BC8_837C8', 'rumble_enable',
|
||||
'func_800C6464_C7064', 'func_800C663C_C723C', 'func_800C67F4_C73F4', 'func_800C6870_C7470',
|
||||
'func_800C68CC_C74CC', 'func_800C6DD4_C79D4', 'func_800C7744_C8344', 'func_800C7804_C8404',
|
||||
'func_800C7864_C8464', 'func_800C78E0_C84E0']
|
||||
|
||||
class ScoreFileMatch:
|
||||
def __init__(self, comment, functionName):
|
||||
|
||||
@@ -1,141 +0,0 @@
|
||||
import os
|
||||
import re
|
||||
from bisect import bisect
|
||||
from collections import OrderedDict
|
||||
|
||||
from file_util import FileUtil
|
||||
|
||||
DATA_FILE_PATH = 'data/dkr.data.s'
|
||||
GLABEL_REGEX = r'D_[0-9A-F]{8}'
|
||||
GLABEL_DEF_REGEX = r'glabel (%s)' % GLABEL_REGEX
|
||||
RODATA_START = 'D_800E49DC' # i.e. the end of .data
|
||||
BSS_START = 'D_800E98D0' # i.e. the end of .rodata
|
||||
# List of labels that are not used in the file they are defined in.
|
||||
# This throws off the splitter algorithm, so the troublesome ones
|
||||
# must be individually blacklisted for now.
|
||||
IGNORE_GLABELS = ['D_800E0001', 'D_800E63E0', 'D_800E94D0']
|
||||
|
||||
def _rom_offset(vaddr):
|
||||
"""
|
||||
Returns the ROM offset of the corresponding virtual address given.
|
||||
Parameters:
|
||||
vaddr: can be a string or integer. If string, it is assumed to be in
|
||||
hex.
|
||||
"""
|
||||
if type(vaddr) == str:
|
||||
vaddr = int(vaddr, 16)
|
||||
return vaddr - 0x7FFFF400
|
||||
|
||||
def _get_glabels():
|
||||
"""
|
||||
Returns all the glabel definitions in the data file, split into .data,
|
||||
.rodata, and .bss.
|
||||
"""
|
||||
data_file = FileUtil.get_text_from_file(DATA_FILE_PATH)
|
||||
glabels = re.findall(GLABEL_DEF_REGEX, data_file)
|
||||
glabels = [glabel for glabel in glabels if glabel not in IGNORE_GLABELS]
|
||||
rodata_idx = glabels.index(RODATA_START)
|
||||
bss_idx = glabels.index(BSS_START)
|
||||
return glabels[:rodata_idx], glabels[rodata_idx:bss_idx], glabels[bss_idx:]
|
||||
|
||||
def _get_file_offset(file, contents):
|
||||
"""
|
||||
Returns the ROM offset of the given file. Throws exception upon error.
|
||||
Parameters:
|
||||
file: filename. Must be a .c or .s file.
|
||||
contents: the contents of file.
|
||||
"""
|
||||
if file.endswith('.c'):
|
||||
return _rom_offset(re.search('/\* RAM_POS: 0x([0-9A-F]{8}) \*/', contents)[1])
|
||||
elif file.endswith('.s'):
|
||||
return int(re.search('/\* ([0-9A-F]{6}) [0-9A-F]{8} [0-9A-F]{8} \*/', contents)[1], 16)
|
||||
else:
|
||||
raise exception('cannot find offset for file ' + file)
|
||||
|
||||
def _log_glabel_usage(glabels):
|
||||
"""
|
||||
Returns:
|
||||
usage: A sorted map from glabel names to a sorted list of all the ROM
|
||||
addresses it is accessed from.
|
||||
c_file_offsets: A list of (filename, ROM offset) tuples from all the c
|
||||
files used.
|
||||
Parameters:
|
||||
glabels: output from _get_glabels.
|
||||
"""
|
||||
usage = OrderedDict([(glabel, set()) for glabel in glabels])
|
||||
files = FileUtil.get_filenames_from_directory_recursive('.', ('.c', '.s'))
|
||||
c_file_offsets = []
|
||||
for file in files:
|
||||
contents = FileUtil.get_text_from_file(file)
|
||||
try:
|
||||
offset = _get_file_offset(file, contents)
|
||||
if file.endswith('.c'):
|
||||
c_file_offsets.append((file, offset))
|
||||
matches = re.findall(GLABEL_REGEX, contents)
|
||||
for glabel in matches:
|
||||
if glabel in usage:
|
||||
usage[glabel].add(offset)
|
||||
except:
|
||||
pass
|
||||
for glabel in usage:
|
||||
usage[glabel] = sorted(list(usage[glabel]))
|
||||
c_file_offsets.sort(key=lambda f: f[1])
|
||||
return usage, c_file_offsets
|
||||
|
||||
def _filter_glabel_usage(glabel_usage):
|
||||
"""
|
||||
Returns a sorted (by ROM offset) list of (glabel name, ROM offset), where
|
||||
the ROM offset is the estimated location the glabel is defined at. Note
|
||||
that this is an estimate; the algorithm used is greedy and may
|
||||
overpredict.
|
||||
Parameters:
|
||||
glabel_usage: output from _log_glabel_usage.
|
||||
"""
|
||||
filtered_usage = []
|
||||
cur_offset = min(glabel_usage[next(iter(glabel_usage))])
|
||||
for glabel in glabel_usage:
|
||||
usage = glabel_usage[glabel]
|
||||
valid_offsets = usage[bisect(usage, cur_offset):]
|
||||
if len(valid_offsets) > 0:
|
||||
cur_offset = valid_offsets[0]
|
||||
filtered_usage.append((glabel, cur_offset))
|
||||
return filtered_usage
|
||||
|
||||
def _split_glabel_files(glabel_usage, c_file_offsets):
|
||||
"""
|
||||
Returns a sorted (by file offset) list of (file name, file offset, glabel name)
|
||||
for every file, where glabel name is the name of the first glabel that
|
||||
lives within the ROM address domain of the corresponding file.
|
||||
Parameters:
|
||||
glabel_usage: output from _filter_glabel_usage.
|
||||
c_file_offsets: output from _log_glabel_usage.
|
||||
"""
|
||||
file_splits = []
|
||||
glabel_idx = 0
|
||||
for i in range(len(c_file_offsets)):
|
||||
file = c_file_offsets[i]
|
||||
while glabel_idx < len(glabel_usage) and glabel_usage[glabel_idx][1] < file[1]:
|
||||
glabel_idx += 1
|
||||
if glabel_idx < len(glabel_usage) and i < len(c_file_offsets) - 1:
|
||||
glabel = glabel_usage[glabel_idx]
|
||||
glabel_name = glabel[0] if glabel[1] < c_file_offsets[i + 1][1] else None
|
||||
else:
|
||||
glabel_name = None
|
||||
file_splits.append((file[0], file[1], glabel_name))
|
||||
return file_splits
|
||||
|
||||
def main():
|
||||
FileUtil.set_working_dir_to_project_base()
|
||||
data_glabels, rodata_glabels, bss_glabels = _get_glabels()
|
||||
for section in [('.data', data_glabels), ('.rodata', rodata_glabels), ('.bss', bss_glabels)]:
|
||||
glabels = section[1]
|
||||
usage, c_file_offsets = _log_glabel_usage(glabels)
|
||||
filtered_usage = _filter_glabel_usage(usage)
|
||||
file_splits = _split_glabel_files(filtered_usage, c_file_offsets)
|
||||
print('File splits for %s:' % section[0])
|
||||
for split in file_splits:
|
||||
print('%s (%06X): %s' % split)
|
||||
print()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -1,110 +0,0 @@
|
||||
import subprocess
|
||||
import sys
|
||||
import re
|
||||
import subprocess
|
||||
import argparse
|
||||
|
||||
def fixBadLabel():
|
||||
parser = argparse.ArgumentParser(description="")
|
||||
parser.add_argument("baseLabel", help="Base label")
|
||||
parser.add_argument("badLabel", help="Bad label")
|
||||
args = parser.parse_args()
|
||||
|
||||
baseAddress = int(args.baseLabel[-8:], 16)
|
||||
badAddress = int(args.badLabel[-8:], 16)
|
||||
|
||||
if baseAddress >= badAddress:
|
||||
raise Exception('baseLabel should be smaller! base = ' + hex(baseAddress) + ', bad = ' + hex(badAddress))
|
||||
|
||||
newLabel = args.baseLabel + '+' + str(badAddress-baseAddress)
|
||||
|
||||
subprocess.run(['./rename_sym.sh', args.badLabel, newLabel])
|
||||
|
||||
def load_syms():
|
||||
with open('undefined_syms.txt', 'r') as inFile:
|
||||
return inFile.read().split('\n')
|
||||
|
||||
def save_syms(syms):
|
||||
with open('undefined_syms.txt', 'w') as outFile:
|
||||
outFile.write('\n'.join(syms))
|
||||
|
||||
def check_is_double(symbol, address):
|
||||
command = subprocess.run(['fgrep', '-ri', symbol], capture_output=True)
|
||||
outLines = command.stdout.decode("utf-8").split('\n')
|
||||
for outLine in outLines:
|
||||
if 'lwc1' in outLine:
|
||||
newSym = 'D_' + hex(address - 4)[2:].upper() + ' + 4'
|
||||
subprocess.run(['./rename_sym.sh', symbol, newSym])
|
||||
return True
|
||||
return False
|
||||
|
||||
def fixDoubleLoads():
|
||||
syms = load_syms()
|
||||
target = len(syms)
|
||||
i = 0
|
||||
while i < target:
|
||||
sym = syms[i]
|
||||
if sym.startswith('D_'):
|
||||
symSplit = sym.split(' = ')
|
||||
symbol = symSplit[0]
|
||||
address = int(symSplit[1][2:-1], 16)
|
||||
if address >= 0x800E4BFC and address < 0x800E9BA0 and ((address & 4) != 0):
|
||||
if check_is_double(symbol, address):
|
||||
syms.pop(i)
|
||||
target -= 1
|
||||
continue
|
||||
i += 1
|
||||
save_syms(syms)
|
||||
|
||||
def getMatches(string, regex):
|
||||
out = []
|
||||
matches = re.finditer(regex, string, re.MULTILINE)
|
||||
for matchNum, match in enumerate(matches, start=1):
|
||||
start = match.start()
|
||||
end = match.end()
|
||||
out.append((string[start:end], match.groups(), (start, end)))
|
||||
return out
|
||||
|
||||
FILE_REGEX = r"(?:lib/)?(?:(?:src/)|(?:asm/non_matchings/))([^/.]*)"
|
||||
def getFiles(symbol):
|
||||
command = subprocess.run(['fgrep', '-ri', symbol], capture_output=True)
|
||||
outLines = command.stdout.decode("utf-8").split('\n')
|
||||
fileNames = []
|
||||
for outLine in outLines:
|
||||
if 'src' in outLine or 'asm' in outLine:
|
||||
match = getMatches(outLine, FILE_REGEX)[0][1][0]
|
||||
if match not in fileNames:
|
||||
fileNames.append(match)
|
||||
return fileNames
|
||||
|
||||
def getBss():
|
||||
currentFile = ''
|
||||
out = []
|
||||
syms = load_syms()
|
||||
for sym in syms:
|
||||
if ' = ' in sym:
|
||||
symSplit = sym.split(' = ')
|
||||
symbol = symSplit[0]
|
||||
address = int(symSplit[1][2:-1], 16)
|
||||
if address >= 0x80115CE8 and address <= 0x8012D3F0:
|
||||
try:
|
||||
files = getFiles(symbol)
|
||||
if currentFile not in files:
|
||||
if len(files) == 1:
|
||||
if currentFile != '':
|
||||
out.append('\n')
|
||||
currentFile = files[0]
|
||||
out.append(currentFile)
|
||||
else:
|
||||
print(symbol, files, 'Cannot tell which file to use!')
|
||||
except IndexError:
|
||||
print('Error with symbol: ' + symbol)
|
||||
out.append(sym)
|
||||
with open('bss.txt', 'w') as outFile:
|
||||
outFile.write('\n'.join(out))
|
||||
|
||||
|
||||
fixBadLabel()
|
||||
#fixDoubleLoads()
|
||||
#getBss()
|
||||
|
||||
Reference in New Issue
Block a user