Add matching yaz0 compression

Makefile

@@ -49,15 +49,15 @@ C_FILES := $(wildcard src/libultra/*) \
            $(wildcard src/boot_O2_g3/*) \
            $(wildcard src/boot_O1/*)
 C_O_FILES = $(C_FILES:src/%.c=build/src/%.o)
-O_FILES := $(BASEROM_O_FILES) $(S_O_FILES)
+ROM_FILES := $(shell cat makerom_files.txt)
 
 
 ROM := rom.z64
-ELF := build/rom.elf
 
 # make build directories
 $(shell mkdir -p build/asm)
 $(shell mkdir -p build/baserom)
+$(shell mkdir -p build/comp)
 $(shell mkdir -p build/src)
 $(shell mkdir -p build/src/libultra)
 $(shell mkdir -p build/src/libultra/os)
@@ -71,11 +71,8 @@ $(shell mkdir -p build/src/boot_O1)
 check: $(ROM) code.bin boot.bin
 	@md5sum -c checksum.md5
 
-$(ROM): $(ELF)
-	@python3 elf2rom.py
-
-$(ELF): $(O_FILES) ldscript.txt
-	$(LD) -T ldscript.txt --no-check-sections --accept-unknown-input-arch -o $@
+$(ROM): $(ROM_FILES)
+	@python3 makerom.py
 
 boot.bin: code.elf
 	$(MIPS_BINUTILS)objcopy --dump-section boot=$@ $<
@@ -90,12 +87,12 @@ test.txt: build/src/test.o
 	$(MIPS_BINUTILS)objdump -d -z --adjust-vma=0x80080790 $< > test.txt
 
 clean:
-	rm $(ROM) $(ELF) code.elf code.bin boot.bin -r build
+	rm $(ROM) code.elf code.bin boot.bin -r build
 
 # Recipes
 
-build/baserom/%.o: baserom/%
-	$(MIPS_BINUTILS)objcopy -I binary -O elf32-big $< $@
+build/baserom/%: baserom/%
+	cp $< $@
 
 build/asm/%.o: asm/%.asm
 	$(AS) $(ASFLAGS) $^ -o $@
@@ -103,3 +100,6 @@ build/asm/%.o: asm/%.asm
 build/src/%.o: src/%.c include/*
 	$(CC) -c $(CFLAGS) $(MIPS_VERSION) $(OPTIMIZATION) -Iinclude -o $@ $<
 
+build/comp/%.yaz0: decomp/%
+	python3 yaz0.py -i $< -o $@
+

elf2rom.py

@@ -1,36 +0,0 @@
-import os
-import struct
-
-FILENAME = 'build/rom.elf'
-OUT = 'rom.z64'
-
-try:
-    with open(FILENAME, 'rb') as f:
-        fileData = f.read()
-except IOError:
-    print('failed to read file ' + FILENAME)
-
-def read_uint32_be(offset):
-    return struct.unpack('>I', fileData[offset:offset+4])[0]
-
-def read_uint16_be(offset):
-    return struct.unpack('>H', fileData[offset:offset+2])[0]
-
-with open(OUT, 'wb') as w:
-    sht_off = read_uint32_be(0x20)
-    sh_size = read_uint16_be(0x2E)
-    num_sections = read_uint16_be(0x30)
-
-    total_size = 0
-    for i in range(0, num_sections):
-        type_ = read_uint32_be(sht_off + i*sh_size + 0x04)
-        offset = read_uint32_be(sht_off + i*sh_size + 0x10)
-        size = read_uint32_be(sht_off + i*sh_size + 0x14)
-        if type_ == 1:# SHT_PROGBITS
-            total_size += size
-            w.write(fileData[offset:offset+size])
-
-    while total_size < 0x2000000:
-        w.write((total_size % 256).to_bytes(1,"big"))
-        total_size += 1
-    

extract_rom.py

@@ -1,9 +1,10 @@
 import struct;
 import os;
+import sys
 
 from libyaz0 import decompress
 
-ROM_FILE_NAME = 'Legend of Zelda, The - Majoras Mask (E) (Prototype).z64'
+ROM_FILE_NAME = 'baserom.z64'
 FILE_TABLE_OFFSET = 0x1A500 # 0x1C110 for JP1.0, 0x1C050 for JP1.1, 0x24F60 for debug
 
 FILE_NAMES = {

makerom.py

@@ -0,0 +1,32 @@
+import os
+import struct
+import sys
+
+FILENAME = 'makerom_files.txt'
+OUT = 'rom.z64'
+
+def read_uint32_be(offset):
+    return struct.unpack('>I', fileData[offset:offset+4])[0]
+
+def read_uint16_be(offset):
+    return struct.unpack('>H', fileData[offset:offset+2])[0]
+
+with open(OUT, 'wb') as w, open(FILENAME, 'rt') as f:
+    file_name = f.readline().strip()
+    total_size = 0
+
+    while file_name:
+        try:
+            with open(file_name, 'rb') as current_file:
+                file_data = current_file.read()
+                w.write(file_data)
+                total_size += len(file_data)
+        except:
+            print('Could not open file ' + file_name)
+            sys.exit(1)
+        file_name = f.readline().strip()
+
+    while total_size < 0x2000000:
+        w.write((total_size % 256).to_bytes(1,"big"))
+        total_size += 1
+

yaz0.py

@@ -0,0 +1,192 @@
+import os
+import sys
+import getopt
+
+def read_file(name):
+    file_data=[]
+
+    try:
+        with open(name, 'rb') as f:
+            file_data = f.read()
+    except IOError:
+        print('failed to read file ' + name)
+        sys.exit(2)
+    return file_data
+
+
+def write_file(name, file_data):
+    try:
+        with open(name, 'wb') as f:
+            f.write(file_data)
+    except IOError:
+        print('failed to write file ' + name)
+        sys.exit(2)
+
+
+def yaz0_decompress(input):
+    output = bytearray()
+
+    return output
+
+
+max_len = 0xFF + 0x12
+def back_seach(input, size, start_pos):
+    best_len = 1
+    match_pos = 0
+    search_pos = max(start_pos - 0x1000, 0)
+    end_pos = min(size, start_pos + max_len)
+
+    # Seach for substrings that are at least 3 bytes long (the smallest size resulting in a compressed chunk)
+    token_end_pos = min(start_pos + 3, size)
+    seatch_len = token_end_pos - start_pos
+    token = input[start_pos:token_end_pos]
+
+    while search_pos < start_pos:
+        search_pos = input.find(token, search_pos, start_pos + seatch_len - 1)
+        if search_pos == -1:
+            break
+
+        pos1 = search_pos + seatch_len
+        pos2 = start_pos + seatch_len
+
+        # Find how many more bytes match
+        while pos2 < end_pos and input[pos1] == input[pos2]:
+            pos1 += 1
+            pos2 += 1
+
+        found_len = pos2 - start_pos
+
+        if found_len > best_len:
+            best_len = found_len
+            seatch_len = found_len
+            match_pos = search_pos
+
+            if best_len == max_len:
+                break
+
+            token_end_pos = start_pos + seatch_len
+            token = input[start_pos:start_pos + seatch_len]
+
+        search_pos += 1
+
+    return best_len, match_pos
+
+
+prev_flag = False
+prev_len = 0
+prev_pos = 0
+def cached_encode(input, size, pos):
+    global prev_flag
+    global prev_len
+    global prev_pos
+
+    # If a previous search found that it was better to have an uncompressed byte, return the position and length that we already found
+    if prev_flag:
+        prev_flag = False
+        return prev_len, prev_pos
+
+    comp_len, comp_pos = back_seach(input, size, pos)
+
+    # Check that it wouldn't be better to have an uncompressed byte then compressing the following data
+    if comp_len >= 3:
+        prev_len, prev_pos = back_seach(input, size, pos + 1)
+        if prev_len >= comp_len + 2: # +2 to account for the uncompressed byte plus 1 more to see if it's better compression
+            comp_len = 1
+            prev_flag = True
+
+    return comp_len, comp_pos
+
+
+def write_yaz0_header(output, size):
+    output += 'Yaz0'.encode()
+
+    output.append((size & 0xFF000000) >> 24)
+    output.append((size & 0x00FF0000) >> 16)
+    output.append((size & 0x0000FF00) >> 8)
+    output.append( size & 0x000000FF)
+
+    output += '\0\0\0\0\0\0\0\0'.encode()
+
+
+def yaz0_compress(input):
+    output = bytearray()
+
+    decompressed_size = len(input)
+
+    write_yaz0_header(output, decompressed_size)
+
+    curr_pos = 0
+    chunk_bits = 0
+    chunk_num_bits = 0
+    chunk_data = bytearray()
+    while curr_pos < decompressed_size:
+        num_bytes, match_pos = cached_encode(input, decompressed_size, curr_pos)
+
+        if num_bytes < 3:
+            chunk_data.append(input[curr_pos])
+            curr_pos += 1
+            chunk_bits |= (0x80 >> chunk_num_bits)
+        else:
+            dist = curr_pos - match_pos - 1
+
+            if num_bytes >= 0x12:
+                chunk_data.append(dist >> 8)
+                chunk_data.append(dist & 0xFF)
+                chunk_data.append(num_bytes - 0x12)
+            else:
+                chunk_data.append(((num_bytes - 2) << 4) | (dist >> 8))
+                chunk_data.append(dist & 0xFF)
+
+            curr_pos += num_bytes
+
+        chunk_num_bits += 1
+
+        if chunk_num_bits == 8:
+            output.append(chunk_bits)
+            output += chunk_data
+
+            chunk_bits = 0
+            chunk_num_bits = 0
+            chunk_data = bytearray()
+
+    if chunk_num_bits > 0:
+        output.append(chunk_bits)
+        output += chunk_data
+
+    output_size = len(output)
+    output_padding_amount = ((output_size + 15) // 16) * 16 - output_size
+    for i in range(output_padding_amount):
+        output.append(0)
+
+    return output
+
+
+def main(argv):
+    inputfile = ''
+    outputfile = ''
+    decompress = False
+    try:
+        opts, args = getopt.getopt(argv, 'i:o:d')
+    except getopt.GetoptError:
+        print('getopt Error') # todo errors
+        sys.exit(2)
+    for opt, arg in opts:
+        if opt == '-i':
+            inputfile = arg
+        elif opt == '-o':
+            outputfile = arg
+        elif opt == '-d':
+            decompress = True
+
+    input_data = read_file(inputfile)
+
+    if decompress:
+        output_data = yaz0_decompress(input_data)
+    else:
+        output_data = yaz0_compress(input_data)
+
+    write_file(outputfile, output_data)
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])