Added Files

tools/python/.gitignore

@@ -0,0 +1 @@
+__pycache__

tools/python/extract.py

@@ -0,0 +1,143 @@
+from os import makedirs
+import shutil
+
+from file_util import FileUtil
+from extract_config import Config
+from rom import ROM
+
+##################################################################################
+
+CONFIGS_DIRECTORY = './extract-ver'
+CONFIGS_EXTENSION = 'extract-config'
+
+DEFAULT_ROM_DIRECTORY = './baseroms'
+ACCEPTED_ROM_EXTENSIONS = ('.z64', '.v64', '.n64')
+
+ASSETS_DIRECTORY = './assets'
+ASM_ASSETS_DIRECTORY = './asm/assets'
+
+RESET_DIRECTORIES = ['/bin', '/text', '/ucode', '/level', '/object']
+
+##################################################################################
+
+romsDirectory = DEFAULT_ROM_DIRECTORY
+
+roms = []
+configs = []
+
+##################################################################################
+
+def main():
+    global roms, configs
+    print('Removing existing directories...')
+    for dir in RESET_DIRECTORIES:
+        try:
+            shutil.rmtree(ASSETS_DIRECTORY + dir)
+        except FileNotFoundError as error:
+            pass
+    get_roms()
+    if(len(roms) > 0):
+        print('Preparing Configs...')
+        get_configs()
+        if(len(configs) > 0):
+            list_extraction_options()
+        else:
+            print('No proper configs found in: "' + CONFIGS_DIRECTORY + '"')
+            print('The valid config file extension is: ' + CONFIGS_EXTENSION)
+    else:
+        print('No ROMs found in the directory: "' + romsDirectory + '"')
+        print('The valid ROM file extensions are: ' + str(ACCEPTED_ROM_EXTENSIONS))
+    
+def get_configs():
+    global configs
+    configsFilenames = FileUtil.get_filenames_from_directory(CONFIGS_DIRECTORY)
+    for configFilename in configsFilenames:
+        with open(CONFIGS_DIRECTORY + '/' + configFilename, 'r') as configFile:
+            configs.append(Config(configFile.read()))
+    if(len(configs) > 0):
+        print('Configs loaded!')
+    
+def get_roms():
+    global roms
+    print('Preparing ROMs...')
+    romFilenames = FileUtil.get_filenames_from_directory(romsDirectory, ACCEPTED_ROM_EXTENSIONS)
+    for romFilename in romFilenames:
+        roms.append(ROM(romsDirectory + '/' + romFilename))
+    if(len(roms) > 0):
+        print('ROMs loaded!')
+
+def ends_with_n64_extension(filename):
+    return filename.endswith()
+        
+def list_extraction_options():
+    for config in configs:
+        configChecksum = config.md5
+        foundROM = None
+        for rom in roms:
+            if rom.md5 == configChecksum:
+                foundROM = rom
+                break
+        if foundROM is not None:
+            print('Found ROM file for config "' + config.name + '"')
+            extract_assets_from_rom(config, foundROM)
+    print('Done!')
+            
+def check_if_config_sizes_are_valid(config, rom):
+    romOffset = 0
+    for romRange in config.ranges:
+        romOffset += romRange.size
+    if romOffset != rom.size:
+        raise Exception('Error: Config does not add up to ROM size.\nROM size is ' + hex(rom.size) + '; Config ends at ' + hex(romOffset))
+            
+def extract_assets_from_rom(config, rom):
+    if(config.notSupported):
+        print('This version of the game is currently not supported.')
+        return
+    check_if_config_sizes_are_valid(config, rom)
+    with open(ASM_ASSETS_DIRECTORY + '/assets.s', 'w') as assetsImportFile:
+        assetsImportText = '# This file was generated from extract.py\n';
+        assetsImportText += '# TODO: Add if/elif/else for other versions of dkr\n\n';
+        romOffset = 0
+        for romRange in config.ranges:
+            rangeSize = romRange.size
+            rangeStart = romOffset
+            rangeEnd = romOffset + rangeSize
+            # TODO: Add more types
+            if romRange.type == 'binary':
+                binaryName = romRange.properties[0]
+                binaryExtractLocation = romRange.properties[1]
+                outputFilename = binaryName + '.' + "{:06x}".format(rangeStart) + '.bin'
+                outputDirectory = ASSETS_DIRECTORY + '/'
+                if len(config.subfolder) > 0:
+                    outputDirectory += config.subfolder + '/'
+                outputDirectory += binaryExtractLocation + '/'
+                data = rom.get_bytes_from_range(rangeStart, rangeEnd)
+                write_data_to_file(romRange, rangeStart, outputDirectory, outputFilename, 'wb', data)
+                if rangeStart > 0x1000: # Exclude boot.000040.bin from assets.s
+                    assetsImportText += '.incbin "' + outputDirectory + outputFilename + '"\n'
+            elif romRange.type == 'noextract':
+                pass
+            else:
+                raise Exception('Invalid range type: "' + romRange.type + '"')
+            romOffset += rangeSize
+        assetsImportFile.write(assetsImportText)
+
+def write_data_to_file(range, rangeStart, directory, filename, flags, data):
+    try: 
+        makedirs(directory)
+    except OSError as error: 
+        pass
+    with open(directory + filename, flags) as outFile:
+        if flags == 'wb':
+            outFile.write(bytearray(data))
+        elif  flags == 'w':
+            outFile.write(data)
+        print('Extracted ' + range.get_range_string(rangeStart) + ' to ' + directory + filename)
+        
+
+def _bytes_to_int32(arr, offset):
+    return int.from_bytes(arr[offset:offset+4], byteorder='big')
+    
+##################################################################################
+
+main()

tools/python/extract_config.py

@@ -0,0 +1,83 @@
+import re
+
+class ConfigRange:
+    def __init__(self, size, type, properties):
+        self.size = size
+        self.type = type.lower()
+        self.properties = properties
+        
+    def get_range_string(self, start):
+        return "{:06x}".format(start) + '-' + "{:06x}".format(start + self.size)
+    
+    def __repr__(self):
+        return "{:06x}".format(self.size) + ', ' + self.type + ', ' + str(self.properties)
+
+class Config:
+    def __init__(self, text):
+        self.text = text
+        self.ranges = []
+        self.name = ''
+        self.md5 = ''
+        self.subfolder = ''
+        self.notSupported = False
+        if not self._parse():
+            raise Exception('Error: ' + self.parseError)
+        #print(self.name)
+        #print(self.md5)
+        #print(str(self.ranges))
+        
+    def _parse(self):
+        regex_property = r'^\s*([0-9a-zA-Z\-]+)\s*:\s*["]([^"]*)["]\s*$'
+        regex_range = r'^\s*\[\s*(0x[0-9a-fA-F]+)\s*\]\s*:\s*(.*)$'
+        for line in self.text.split('\n'):
+            line = line.strip() # remove leading and trailing whitespace
+            line = line.partition('#')[0] # remove comments
+            if len(line) > 0:
+                matches = self._parse_test_re(regex_property, line)
+                if matches is not None:
+                    self._parse_property(matches)
+                    continue
+                matches = self._parse_test_re(regex_range, line)
+                if matches is not None:
+                    #print(matches)
+                    self._parse_range(matches)
+                    continue
+                self.parseError = 'Invalid line "' + line + '"'
+                return False
+        return True
+    
+    def _parse_property(self, matches):
+        propertyName = matches[0]
+        propertyValue = matches[1]
+        if propertyName == 'config-name':
+            self.name = propertyValue
+        elif propertyName == 'subfolder':
+            self.subfolder = propertyValue
+        elif propertyName == 'checksum-md5':
+            self.md5 = propertyValue
+        elif propertyName == 'not-supported':
+            self.notSupported = (propertyValue.lower() == 'true')
+        else:
+            print('Unknown property "' + propertyName + '" with value "' + propertyValue + '"')
+            
+    def _parse_range(self, matches):
+        rangeSize = int(matches[0], 0)
+        rangeProperties = matches[1].split(',')
+        
+        for i in range(0, len(rangeProperties)):
+            rangeProperties[i] = rangeProperties[i].strip() # remove leading and trailing whitespace
+            rangeProperties[i] = self._parse_range_property(rangeProperties[i])
+            
+        self.ranges.append(ConfigRange(rangeSize, rangeProperties[0], rangeProperties[1:]))
+            
+    def _parse_range_property(self, property):
+        if property.startswith('"'): # string
+            return property[1:-1]
+        else:
+            return int(property, 0)
+              
+    def _parse_test_re(self, regex, line):
+        matches = re.match(regex, line)
+        if matches is not None:
+            return matches.groups()
+        return None

tools/python/file_util.py

@@ -0,0 +1,25 @@
+from os import listdir, remove
+from os.path import isfile, join
+
+class FileUtil:
+    @staticmethod
+    def get_filenames_from_directory(directory, extensions=None):
+        if extensions is None:
+            return [f for f in listdir(directory) if isfile(join(directory, f))]
+        else:
+            return [f for f in listdir(directory) if isfile(join(directory, f)) and f.endswith(extensions)]
+    
+    @staticmethod
+    def get_text_from_file(filename):
+        with open(filename, 'r') as inFile:
+            return inFile.read()
+            
+    @staticmethod
+    def write_text_to_file(filename, text):
+        with open(filename, 'w') as outFile:
+            outFile.write(text)
+            
+    @staticmethod
+    def delete_file(filename):
+        remove(filename)
+    

tools/python/generate_ld.py

@@ -0,0 +1,135 @@
+import re
+
+from file_util import FileUtil
+
+LD_NAME = 'dkr.ld'
+ASM_DIR = './asm'
+
+class LD:
+    def __init__(self, file):
+        print('Generating linker file...')
+        self.files = self.get_asm_files()
+        self.indentLevel = 0
+        self.file = file
+        self.gen_comment('linker script generated by generate_ld.py')
+        self.gen_newline()
+        self.gen_line('OUTPUT_ARCH (mips)')
+        self.gen_newline()
+        #self.gen_macros()
+        self.gen_sections()
+        print('New linker file created!')
+    
+    def gen_macros(self):
+        self.gen_line('#define BEGIN_SEG(name, addr) \\')
+        self.increase_indent()
+        self.gen_line('_##name##SegmentStart = ADDR(.name); \\')
+        self.gen_line('_##name##SegmentRomStart = __romPos; \\')
+        self.gen_line('.name addr : AT(__romPos)')
+        self.decrease_indent()
+        self.gen_newline()
+        self.gen_line('#define END_SEG(name) \\')
+        self.increase_indent()
+        self.gen_line('_##name##SegmentEnd = ADDR(.name) + SIZEOF(.name); \\')
+        self.gen_line('_##name##SegmentRomEnd = __romPos + SIZEOF(.name); \\')
+        self.gen_line('__romPos += SIZEOF(.name);')
+        self.decrease_indent()
+        self.gen_newline()
+        
+    def gen_sections(self):
+        self.gen_line('SECTIONS')
+        self.gen_open_block()
+        self.gen_line('romPos = 0x0;')
+        self.gen_boot_section()
+        self.gen_main_section()
+        self.gen_assets_section()
+        self.gen_discard()
+        self.gen_close_block()
+    
+    def gen_boot_section(self):
+        self.gen_line('.boot 0 : AT(romPos)')
+        self.gen_open_block()
+        self.gen_line('build/asm/boot/rom_header.o(.text);')
+        self.gen_line('build/asm/boot/rom_boot.o(.text);')
+        self.gen_close_block()
+        self.gen_line('romPos += SIZEOF(.boot);')
+        self.gen_newline()
+    
+    def gen_main_section(self):
+        self.gen_line('.main 0x80000400 : AT(romPos) SUBALIGN(16)')
+        self.gen_open_block()
+        self.files.sort(key = lambda x: x[2]) # Sort tuples by RAM address
+        for asmFile in self.files:
+            self.gen_line(asmFile[0] + '.o(.text);')
+        self.gen_close_block()
+        self.gen_line('romPos += SIZEOF(.main);')
+        self.gen_newline()
+        
+    def gen_assets_section(self):
+        self.gen_line('.assets 0 : AT(romPos)')
+        self.gen_open_block()
+        self.gen_line('build/asm/assets/assets.o(.text);')
+        self.gen_close_block()
+        self.gen_line('romPos += SIZEOF(.assets);')
+        self.gen_newline()
+        
+    def gen_discard(self):
+        self.gen_comment('Discard everything not specifically mentioned above.')
+        self.gen_line('/DISCARD/ :')
+        self.gen_open_block()
+        self.gen_line('*(*);')
+        self.gen_close_block()
+    
+    def increase_indent(self):
+        self.indentLevel += 1
+    
+    def decrease_indent(self):
+        self.indentLevel -= 1
+        
+    def gen_open_block(self):
+        self.gen_line('{')
+        self.increase_indent()
+        
+    def gen_close_block(self):
+        self.decrease_indent()
+        self.gen_line('}')
+    
+    def gen_line(self, text):
+        spaces = 4 * self.indentLevel
+        while spaces > 0:
+            self.file.write(' ')
+            spaces -= 1
+        self.file.write(text)
+        self.gen_newline()
+        
+    def gen_comment(self, text):
+        spaces = 4 * self.indentLevel
+        while spaces > 0:
+            self.file.write(' ')
+            spaces -= 1
+        self.file.write('/* ' + text + ' */')
+        self.gen_newline()
+        
+    def gen_newline(self):
+        self.file.write('\n')
+        
+    def get_asm_files(self):
+        asmFiles = []
+        asmFilenames = FileUtil.get_filenames_from_directory(ASM_DIR, ('.s',))
+        regex = r'[\/][*]\s*([0-9A-F]{6})\s*([0-9A-F]{8})\s*([0-9A-F]{8})\s*[*][\/]'
+        for filename in asmFilenames:
+            with open(ASM_DIR + '/' + filename, 'r') as asmFile:
+                notDone = True
+                line = asmFile.readline()
+                while line:
+                    matches = re.match(regex, line)
+                    if matches is None:
+                        line = asmFile.readline()
+                        continue
+                    matchedGroups = matches.groups()
+                    asmFiles.append(('build/asm/' + filename[:-2], matchedGroups[0], matchedGroups[1]))
+                    break
+        return asmFiles
+            
+        
+with open(LD_NAME, 'w') as ldFile:
+    LD(ldFile)

tools/python/get_symbol.py

@@ -0,0 +1,38 @@
+import sys
+import re
+
+from file_util import FileUtil
+
+SYMBOLS_TEXT_FILENAME = 'undefined_syms.txt'
+
+symbol_define_regex = r'([_0-9A-Za-z]*)\s*=\s*([_0-9A-Za-z]*)\s*;'
+
+def main():
+    if len(sys.argv) != 2:
+        show_help()
+        return
+    print_symbol(int(sys.argv[1], 16))
+
+def print_symbol(ramAddress):
+    lines = FileUtil.get_text_from_file(SYMBOLS_TEXT_FILENAME).split('\n')
+    foundSymbol = False
+    for line in lines:
+        matches = re.match(symbol_define_regex, line)
+        if matches is None:
+            continue
+        address = int(matches[2], 16)
+        if address == ramAddress:
+            symbol = matches[1]
+            print(matches[2] + ' = ' + matches[1])
+            foundSymbol = True
+            break
+    if not foundSymbol:
+        print('No symbol was found for the address "' + hex(ramAddress) + '"')
+        
+    
+def show_help():
+    print("Usage: ./get_symbol <RAM Address>")
+    
+##################################################################################
+
+main()

tools/python/rename_symbol.py

@@ -0,0 +1,48 @@
+import sys
+import re
+
+from file_util import FileUtil
+
+ASM_DIR = './asm'
+SYMBOL_MIN_LENGTH = 4
+SYMBOLS_TEXT_FILENAME = 'undefined_syms.txt'
+
+def main():
+    if len(sys.argv) != 3:
+        show_help()
+        return
+    old_symbol = sys.argv[1]
+    new_symbol = sys.argv[2]
+    if len(old_symbol) < SYMBOL_MIN_LENGTH or len(new_symbol) < SYMBOL_MIN_LENGTH:
+        show_help()
+        return
+    replace_symbol(old_symbol, new_symbol)
+
+def show_help():
+    print("Usage: ./rename_symbol <Old Symbol> <New Symbol>")
+    print("The symbols must be at-least {0} characters long.".format(SYMBOL_MIN_LENGTH))
+    
+def get_regex(symbol):
+    return r'(?<=[^_0-9A-Za-z])' + symbol + r'(?=[^_0-9A-Za-z])'
+    
+def replace_symbol(old, new):
+    replaceRegex = get_regex(old)
+    
+    # Replace symbol in undefined_syms.txt
+    undefined_symbols = FileUtil.get_text_from_file(SYMBOLS_TEXT_FILENAME)
+    undefined_symbols = re.sub(replaceRegex, new, undefined_symbols)
+    FileUtil.write_text_to_file(SYMBOLS_TEXT_FILENAME, undefined_symbols)
+    
+    # Replace symbol in .s files in the ./asm directory
+    asmFilenames = FileUtil.get_filenames_from_directory(ASM_DIR, ('.s',))
+    for asmFilename in asmFilenames:
+        asm = FileUtil.get_text_from_file(ASM_DIR + '/' + asmFilename)
+        asm = re.sub(replaceRegex, new, asm)
+        FileUtil.write_text_to_file(ASM_DIR + '/' + asmFilename, asm)
+    
+    print('Successfully replaced symbol "' + old + '" with "' + new + '"')
+    
+    
+##################################################################################
+
+main()

tools/python/rom.py

@@ -0,0 +1,58 @@
+import hashlib
+
+from file_util import FileUtil
+
+class ROM:
+    def __init__(self, filename):
+        with open(filename, 'rb') as romFile:
+            self.fixedRomEndianess = False
+            self.bytearray = romFile.read()
+            self.bytes = list(self.bytearray)
+            self.size = len(self.bytes)
+            self._test_endianness()
+            self.md5 = hashlib.md5(self.bytearray).hexdigest()
+        if self.fixedRomEndianess: 
+            FileUtil.delete_file(filename)
+            # Save the ROM as big-endian
+            with open(filename[:-4] + '.z64', 'wb') as romFile:
+                romFile.write(self.bytearray)
+            
+    def get_bytes_from_range(self, start, end):
+        return self.bytes[start:end]
+            
+    # Check and correct mixed/little endian ROMs
+    def _test_endianness(self):
+        if self.bytes[0] == 0x80 and self.bytes[1] == 0x37:
+            self.endianness = 'big'
+        elif self.bytes[0] == 0x37 and self.bytes[1] == 0x80:
+            self.endianness = 'mixed'
+        elif self.bytes[0] == 0x40 and self.bytes[1] == 0x12:
+            self.endianness = 'little'
+        
+        # Convert mixed/little endian to big endian.
+        # This is kinda slow, and should be avoided if possible.
+        if self.endianness is 'mixed':
+            print('Converting mixed-endian (byte-swapped) to big-endian...')
+            for i in range(0, len(self.bytes), 2):
+                temp = self.bytes[i]
+                self.bytes[i] = self.bytes[i + 1]
+                self.bytes[i + 1] = temp
+            self.bytearray = bytearray(self.bytes)
+            self.endianness = 'big'
+            self.fixedRomEndianess = True
+            pass
+        elif self.endianness is 'little':
+            print('Converting little-endian to big-endian...')
+            temp = [0, 0, 0, 0]
+            for i in range(0, len(self.bytes), 4):
+                temp[0] = self.bytes[i]
+                temp[1] = self.bytes[i + 1]
+                temp[2] = self.bytes[i + 2]
+                temp[3] = self.bytes[i + 3]
+                self.bytes[i]     = temp[3]
+                self.bytes[i + 1] = temp[2]
+                self.bytes[i + 2] = temp[1]
+                self.bytes[i + 3] = temp[0]
+            self.bytearray = bytearray(self.bytes)
+            self.endianness = 'big'
+            self.fixedRomEndianess = True