Add yar (Yaz0 ARchives) support to build system (#1255)

* extract_archives.py

* autogenerated xmls

* build OK

* cleanup

* Use symbols in gItemIcons

* change how archives are generated

* makeyar in C

* Rename segments

* rename .archive.o to .yar.o

* Explanation at the top of programs

* fix progress calculation

* Ignore archives in progress script

* merge

* Update tools/buildtools/makeyar.c

Co-authored-by: engineer124 <47598039+engineer124@users.noreply.github.com>

* Update tools/buildtools/makeyar.c

Co-authored-by: engineer124 <47598039+engineer124@users.noreply.github.com>

* Update tools/buildtools/makeyar.c

Co-authored-by: engineer124 <47598039+engineer124@users.noreply.github.com>

* Update tools/buildtools/makeyar.c

Co-authored-by: engineer124 <47598039+engineer124@users.noreply.github.com>

---------

Co-authored-by: engineer124 <47598039+engineer124@users.noreply.github.com>

tools/buildtools/.gitignore

@@ -5,3 +5,4 @@ makeromfs
 mkldscript
 reloc_prereq
 yaz0
+makeyar

tools/buildtools/Makefile

@@ -1,6 +1,5 @@
-CC := gcc
-CFLAGS := -Wall -Wextra -pedantic -std=c99 -g -O2
-PROGRAMS := elf2rom makeromfs mkldscript reloc_prereq yaz0
+CFLAGS := -Wall -Wextra -Wpedantic -std=c99 -g -Os
+PROGRAMS := elf2rom makeromfs mkldscript reloc_prereq yaz0 makeyar
 
 ifeq ($(shell command -v clang >/dev/null 2>&1; echo $$?),0)
   CC := clang
@@ -8,15 +7,6 @@ else
   CC := gcc
 endif
 
-LLD ?= 0
-
-ifeq ($(shell command -v ld.lld >/dev/null 2>&1; echo $$?),0)
-  LLD := 1
-endif
-
-ifneq ($(LLD),0)
-  CFLAGS += -fuse-ld=lld
-endif
 all: $(PROGRAMS)
 
 clean:
@@ -27,6 +17,7 @@ makeromfs_SOURCES    := makeromfs.c n64chksum.c util.c
 mkldscript_SOURCES   := mkldscript.c spec.c util.c
 reloc_prereq_SOURCES := reloc_prereq.c spec.c util.c
 yaz0_SOURCES         := yaz0tool.c yaz0.c util.c
+makeyar_SOURCES      := makeyar.c elf32.c yaz0.c util.c
 
 define COMPILE =
 $(1): $($1_SOURCES)

tools/buildtools/elf32.c

@@ -138,6 +138,7 @@ bool elf32_get_section(struct Elf32* e, struct Elf32_Section* sec, int secnum) {
     sec->flags = e->read32(sechdr + 0x08);
     sec->addr = e->read32(sechdr + 0x0C);
     sec->offset = e->read32(sechdr + 0x10);
+    sec->size = e->read32(sechdr + 0x14);
     sec->addralign = e->read32(sechdr + 0x20);
     sec->entsize = e->read32(sechdr + 0x24);
     return true;
@@ -162,5 +163,8 @@ bool elf32_get_symbol(struct Elf32* e, struct Elf32_Symbol* sym, int symnum) {
 
     sym->name = strings + e->read32(symtab + symnum * 0x10);
     sym->value = e->read32(symtab + symnum * 0x10 + 4);
+    sym->size = e->read32(symtab + symnum * 0x10 + 8);
+    sym->st_type = symtab[symnum * 0x10 + 0xC] & 0xF;
+    sym->shndx = e->read16(symtab + symnum * 0x10 + 0xE);
     return true;
 }

tools/buildtools/elf32.h

@@ -3,6 +3,7 @@
 
 #include <stddef.h>
 #include <stdint.h>
+#include <stdbool.h>
 
 enum {
     ELF_MACHINE_NONE = 0,
@@ -52,14 +53,31 @@ struct Elf32_Section {
     uint32_t type;
     uint32_t flags;
     uint32_t addr;
+    uint32_t size;
     uint32_t offset;
     uint32_t addralign;
     uint32_t entsize;
 };
 
+#define SHN_UNDEF   0
+#define SHN_ABS     0xFFF1
+#define SHN_COMMON  0xFFF2
+
+#define STT_NOTYPE  0
+#define STT_OBJECT  1
+#define STT_FUNC    2
+#define STT_SECTION 3
+#define STT_FILE    4
+#define STT_COMMON  5
+#define STT_TLS     6
+#define STT_NUM     7
+
 struct Elf32_Symbol {
     const char* name;
     uint32_t value;
+    uint32_t size;
+    uint8_t st_type;
+    uint16_t shndx;
 };
 
 bool elf32_init(struct Elf32* e, const void* data, size_t size);

tools/buildtools/makeyar.c

@@ -0,0 +1,267 @@
+/* SPDX-FileCopyrightText: © 2023 ZeldaRET */
+/* SPDX-License-Identifier: MIT */
+
+/**
+ * Program to generate compressed yar (Yaz0 ARchive) files.
+ *
+ * The program expects an .o elf file and outputs a raw yar binary file and a
+ * "symbols" elf.
+ *
+ * A yar file consists of multiple Yaz0 files compressed individually. The
+ * archive begins with a header of non-fixed size, which describes the
+ * location of each individual Yaz0 block within the archive itself. This
+ * header is followed by each Yaz0 file.
+ *
+ * The first word (a 4 byte group) of the header indicates the size in bytes of
+ * the header itself (also describes the offset of the first Yaz0 block). The
+ * rest of the header consists of words describing the offsets of each Yaz0
+ * block relative to the end of the header, because the first Yaz0
+ * block is omitted from the offsets in the header.
+ *
+ * Each Yaz0 block is 0xFF-padded to a multiple of 0x10 in size.
+ *
+ * The entire archive is 0-padded to a multiple of 0x10 in size.
+ *
+ * The program works by compressing each .data symbol in the input elf file as
+ * its own Yaz0 compressed file, appending them in order for the generated
+ * archive. Other elf sections are ignored for the resulting yar file.
+ *
+ * The program also outputs an elf file that's identical to the elf input,
+ * but with its .data section zero'ed out completely. This "symbols" elf can be
+ * used for referencing each symbol as the whole file were completely
+ * uncompressed.
+ */
+
+
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+
+#include "elf32.h"
+#include "yaz0.h"
+#include "util.h"
+
+
+typedef struct Bytearray {
+    uint8_t *bytes;
+    size_t size;
+} Bytearray;
+
+typedef struct SymbolList {
+    struct Elf32_Symbol *symbols;
+    size_t size; // allocated size
+    size_t len; // elements in the list
+} SymbolList;
+
+typedef struct DataSection {
+    Bytearray data;
+    uint32_t dataOffset;
+    SymbolList symbols;
+} DataSection;
+
+
+void Bytearray_Init(Bytearray *bytearr, const uint8_t *bytes, size_t size) {
+    bytearr->bytes = malloc(size);
+    if (bytearr->bytes == NULL) {
+        util_fatal_error("memory error");
+    }
+
+    memcpy(bytearr->bytes, bytes, size);
+    bytearr->size = size;
+}
+
+void Bytearray_InitValue(Bytearray *bytearr, uint8_t val, size_t count) {
+    bytearr->bytes = malloc(count * sizeof(uint8_t));
+    if (bytearr->bytes == NULL) {
+        util_fatal_error("memory error");
+    }
+
+    memset(bytearr->bytes, val, count);
+    bytearr->size = count;
+}
+
+void Bytearray_ExtendValue(Bytearray *bytearr, uint8_t val, size_t count) {
+    size_t newSize = bytearr->size + count;
+
+    bytearr->bytes = realloc(bytearr->bytes, newSize);
+    if (bytearr->bytes == NULL) {
+        util_fatal_error("memory error");
+    }
+
+    memset(&bytearr->bytes[bytearr->size], val, count);
+    bytearr->size = newSize;
+}
+
+void Bytearray_Extend(Bytearray *bytearr, const uint8_t *bytes, size_t size) {
+    size_t newSize = bytearr->size + size;
+
+    bytearr->bytes = realloc(bytearr->bytes, newSize);
+    if (bytearr->bytes == NULL) {
+        util_fatal_error("memory error");
+    }
+
+    memcpy(&bytearr->bytes[bytearr->size], bytes, size);
+    bytearr->size = newSize;
+}
+
+void Bytearray_Destroy(Bytearray *bytearr) {
+    free(bytearr->bytes);
+}
+
+void SymbolList_Init(SymbolList *list, size_t initialAmount) {
+    list->symbols = malloc(initialAmount * sizeof(struct Elf32_Symbol));
+    if (list->symbols == NULL) {
+        util_fatal_error("memory error");
+    }
+    list->size = initialAmount;
+    list->len = 0;
+}
+
+void SymbolList_Destroy(SymbolList *list) {
+    free(list->symbols);
+}
+
+void DataSection_FromElf(DataSection *dst, const Bytearray *elfBytes){
+    struct Elf32 elf;
+    size_t i;
+    int symIndex;
+    size_t dataShndx = 0;
+
+    if (!elf32_init(&elf, elfBytes->bytes, elfBytes->size) || (elf.machine != ELF_MACHINE_MIPS)) {
+        util_fatal_error("not a valid 32-bit MIPS ELF file");
+    }
+
+    for (i = 0; i < elf.shnum; i++) {
+        struct Elf32_Section sec;
+
+        if (!elf32_get_section(&elf, &sec, i)) {
+            util_fatal_error("invalid or corrupt ELF file");
+        }
+
+        if (strcmp(sec.name, ".data") == 0) {
+            dst->dataOffset = sec.offset;
+            Bytearray_Init(&dst->data, &elfBytes->bytes[sec.offset], sec.size);
+            dataShndx = i;
+            break;
+        }
+    }
+
+    SymbolList_Init(&dst->symbols, elf.numsymbols);
+    for (symIndex = 0; symIndex < elf.numsymbols; symIndex++) {
+        struct Elf32_Symbol sym;
+
+        if (!elf32_get_symbol(&elf, &sym, symIndex)) {
+            util_fatal_error("invalid or corrupt ELF file");
+        }
+
+        if (sym.shndx != dataShndx) {
+            continue;
+        }
+        if (sym.st_type != STT_OBJECT) {
+            continue;
+        }
+
+        dst->symbols.symbols[dst->symbols.len++] = sym;
+    }
+}
+
+void DataSection_Destroy(DataSection *dataSect) {
+    Bytearray_Destroy(&dataSect->data);
+    SymbolList_Destroy(&dataSect->symbols);
+}
+
+#define ALIGN16(val) (((val) + 0xF) & ~0xF)
+
+void createArchive(Bytearray *archive, const DataSection *dataSect) {
+    uint32_t firstEntryOffset = (dataSect->symbols.len + 1) * sizeof(uint32_t);
+    size_t i;
+    size_t offset;
+
+    // Fill with zeroes until the compressed data start
+    Bytearray_InitValue(archive, 0, firstEntryOffset);
+
+    util_write_uint32_be(&archive->bytes[0], firstEntryOffset);
+
+    offset = firstEntryOffset;
+    for (i = 0; i < dataSect->symbols.len; i++) {
+        const struct Elf32_Symbol *sym = &dataSect->symbols.symbols[i];
+        size_t uncompressedSize = sym->size;
+        uint8_t *output = malloc(uncompressedSize * sizeof(uint8_t)); // assume compressed shouldn't be bigger than uncompressed
+        size_t compressedSize;
+
+        output[0] = 'Y';
+        output[1] = 'a';
+        output[2] = 'z';
+        output[3] = '0';
+        util_write_uint32_be(&output[4], uncompressedSize);
+        memset(&output[8], 0, 8);
+        compressedSize = 0x10;
+
+        assert(sym->value + uncompressedSize <= dataSect->data.size);
+        compressedSize += yaz0_encode(&dataSect->data.bytes[sym->value], &output[0x10], uncompressedSize);
+
+        // Pad to 0x10
+        while (compressedSize % 0x10 != 0) {
+            output[compressedSize++] = 0xFF;
+        }
+
+        Bytearray_Extend(archive, output, compressedSize);
+
+        if (i > 0) {
+            util_write_uint32_be(&archive->bytes[i * sizeof(uint32_t)], offset - firstEntryOffset);
+        }
+
+        offset += compressedSize;
+        free(output);
+    }
+
+    util_write_uint32_be(&archive->bytes[i * sizeof(uint32_t)], offset - firstEntryOffset);
+
+    if (archive->size % 16 != 0) {
+        size_t extraPad = ALIGN16(archive->size) - archive->size;
+
+        Bytearray_ExtendValue(archive, 0, extraPad);
+    }
+}
+
+
+int main(int argc, char *argv[]) {
+    const char *inPath;
+    const char *binPath;
+    const char *symPath;
+    Bytearray elfBytes;
+    DataSection dataSect;
+    Bytearray archive;
+
+    if (argc != 4) {
+        fprintf(stderr, "%s in_file out_bin out_sym\n", argv[0]);
+        exit(1);
+    }
+
+    inPath = argv[1];
+    binPath = argv[2];
+    symPath = argv[3];
+
+    elfBytes.bytes = util_read_whole_file(inPath, &elfBytes.size);
+
+    DataSection_FromElf(&dataSect, &elfBytes);
+
+    createArchive(&archive, &dataSect);
+
+    // Write the compressed archive file as a raw binary
+    util_write_whole_file(binPath, archive.bytes, archive.size);
+
+    // Zero out data
+    memset(&elfBytes.bytes[dataSect.dataOffset], 0, dataSect.data.size);
+
+    util_write_whole_file(symPath, elfBytes.bytes, elfBytes.size);
+
+    Bytearray_Destroy(&archive);
+    DataSection_Destroy(&dataSect);
+    Bytearray_Destroy(&elfBytes);
+
+    return 0;
+}

tools/buildtools/yaz0.h

@@ -1,8 +1,6 @@
 #ifndef YAZ0_H
 #define YAZ0_H
 
-int yaz0_encode2(uint8_t* src, uint8_t* dest, int uncompressedSize);
-
 void yaz0_decode(uint8_t* src, uint8_t* dst, int uncompressedSize);
 
 int yaz0_encode(uint8_t* src, uint8_t* dest, int srcSize);

tools/decompress_yars.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+
+# SPDX-FileCopyrightText: © 2023 ZeldaRET
+# SPDX-License-Identifier: MIT
+
+# yar (Yaz0 ARchive) decompressor
+#
+# This program decompresses every raw yar binary file listed in
+# `tools/filelists/{version}/archives.csv` to a corresponding
+# `baserom/{file}.unarchive` raw file.
+#
+# It works by decompressing every Yaz0 block and appending them one by one into
+# a new raw binary file so it can be processed normally by other tools.
+
+
+from __future__ import annotations
+
+import argparse
+import dataclasses
+import libyaz0
+from pathlib import Path
+import struct
+
+
+PRINT_XML = False
+
+@dataclasses.dataclass
+class ArchiveMeta:
+    start: int
+    end: int
+
+
+def readFileAsBytearray(filepath: Path) -> bytearray:
+    with filepath.open(mode="rb") as f:
+        return bytearray(f.read())
+
+
+def readBytesAsWord(array_of_bytes: bytearray, offset: int) -> int:
+    return struct.unpack_from(f">I", array_of_bytes, offset)[0]
+
+
+def getOffsetsList(archiveBytes: bytearray) -> list[ArchiveMeta]:
+    archivesOffsets: list[ArchiveMeta] = []
+
+    firstEntryOffset = readBytesAsWord(archiveBytes, 0)
+    firstEntrySize = readBytesAsWord(archiveBytes, 4)
+
+    archivesOffsets.append(ArchiveMeta(firstEntryOffset, firstEntryOffset + firstEntrySize))
+
+    offset = 4
+    while offset < firstEntryOffset - 4:
+        entry = readBytesAsWord(archiveBytes, offset)
+        nextEntry = readBytesAsWord(archiveBytes, offset + 4)
+        entryStart = entry + firstEntryOffset
+        entryEnd = nextEntry + firstEntryOffset
+        archivesOffsets.append(ArchiveMeta(entryStart, entryEnd))
+
+        offset += 4
+
+    return archivesOffsets
+
+
+def extractArchive(archivePath: Path, outPath: Path):
+    archiveBytes = readFileAsBytearray(archivePath)
+
+    if readBytesAsWord(archiveBytes, 0) == 0:
+        # Empty file, ignore it
+        return
+
+    print(f"Extracting '{archivePath}' -> '{outPath}'")
+    archivesOffsets = getOffsetsList(archiveBytes)
+
+    if PRINT_XML:
+        print('<Root>')
+        print(f'    <File Name="{outPath.stem}">')
+
+    with outPath.open("wb") as out:
+        currentOffset = 0
+        for meta in archivesOffsets:
+            decompressedBytes = libyaz0.decompress(archiveBytes[meta.start:meta.end])
+            decompressedSize = len(decompressedBytes)
+            out.write(decompressedBytes)
+
+            if PRINT_XML:
+                print(f'        <Blob Name="{archivePath.stem}_Blob_{currentOffset:06X}" Size="0x{decompressedSize:04X}" Offset="0x{currentOffset:X}" />')
+
+            currentOffset += decompressedSize
+
+    if PRINT_XML:
+        print(f'    </File>')
+        print('</Root>')
+
+
+def main():
+    parser = argparse.ArgumentParser(description="MM archives extractor")
+    parser.add_argument("-v", "--version", help="version to process", default="mm.us.rev1")
+    parser.add_argument("--xml", help="Generate xml to stdout", action="store_true")
+
+    args = parser.parse_args()
+
+    global PRINT_XML
+    PRINT_XML = args.xml
+
+    archivesCsvPath = Path(f"tools/filelists/{args.version}/archives.csv")
+
+    with archivesCsvPath.open() as f:
+        for line in f:
+            archiveName = line.strip().split(",")[1]
+            archivePath = Path(f"baserom/{archiveName}")
+
+            extractedPath = Path(str(archivePath) + ".unarchive")
+            extractArchive(archivePath, extractedPath)
+
+if __name__ == "__main__":
+    main()

tools/disasm/dma_filenames.txt

@@ -7,8 +7,8 @@
     'Audiotable',
     'kanji',
     'link_animetion',
-    'icon_item_static_old',
-    'icon_item_24_static_old',
+    'icon_item_static_syms',
+    'icon_item_24_static_syms',
     'icon_item_field_static',
     'icon_item_dungeon_static',
     'icon_item_gameover_static',
@@ -18,10 +18,10 @@
     'map_grand_static',
     'item_name_static',
     'map_name_static',
-    'icon_item_static_test',
-    'icon_item_24_static_test',
-    'schedule_dma_static_old',
-    'schedule_dma_static_test',
+    'icon_item_static_yar',
+    'icon_item_24_static_yar',
+    'schedule_dma_static_syms',
+    'schedule_dma_static_yar',
     'schedule_static',
     'story_static',
     'do_action_static',

tools/extract_baserom.py

@@ -14,8 +14,8 @@ FILE_NAMES = [
     'Audiotable',
     'kanji',
     'link_animetion',
-    'icon_item_static_old',
-    'icon_item_24_static_old',
+    'icon_item_static_syms',
+    'icon_item_24_static_syms',
     'icon_item_field_static',
     'icon_item_dungeon_static',
     'icon_item_gameover_static',
@@ -25,10 +25,10 @@ FILE_NAMES = [
     'map_grand_static',
     'item_name_static',
     'map_name_static',
-    'icon_item_static_test',
-    'icon_item_24_static_test',
-    'schedule_dma_static_old',
-    'schedule_dma_static_test',
+    'icon_item_static_yar',
+    'icon_item_24_static_yar',
+    'schedule_dma_static_syms',
+    'schedule_dma_static_yar',
     'schedule_static',
     'story_static',
     'do_action_static',

tools/filelists/mm.us.rev1/all.csv

@@ -6,8 +6,8 @@
 5,Audiotable
 6,kanji
 7,link_animetion
-8,icon_item_static_old
-9,icon_item_24_static_old
+8,icon_item_static_syms
+9,icon_item_24_static_syms
 10,icon_item_field_static
 11,icon_item_dungeon_static
 12,icon_item_gameover_static
@@ -17,10 +17,10 @@
 16,map_grand_static
 17,item_name_static
 18,map_name_static
-19,icon_item_static_test
-20,icon_item_24_static_test
-21,schedule_dma_static_old
-22,schedule_dma_static_test
+19,icon_item_static_yar
+20,icon_item_24_static_yar
+21,schedule_dma_static_syms
+22,schedule_dma_static_yar
 23,schedule_static
 24,story_static
 25,do_action_static

tools/filelists/mm.us.rev1/archives.csv

@@ -2,6 +2,6 @@
 16,map_grand_static
 17,item_name_static
 18,map_name_static
-19,icon_item_static_test
-20,icon_item_24_static_test
-22,schedule_dma_static_test
+19,icon_item_static_yar
+20,icon_item_24_static_yar
+22,schedule_dma_static_yar

tools/filelists/mm.us.rev1/deleted.csv

@@ -1,3 +1,3 @@
-8,icon_item_static_old
-9,icon_item_24_static_old
-21,schedule_dma_static_old
+8,icon_item_static_syms
+9,icon_item_24_static_syms
+21,schedule_dma_static_syms

tools/progress.py

@@ -136,7 +136,6 @@ if not args.matching:
 # The order of this list should not change to prevent breaking the graph of the website
 # New stuff shall be appended at the end of the list
 assetsCategories = [
-    "archives",
     "audio",
     "interface",
     "misc",
@@ -148,6 +147,11 @@ assetsCategories = [
 ]
 assetsTracker = dict()
 
+# Assets that we don't have a proper way of tracking right now
+ignoredAssets = {
+    "archives",
+}
+
 # Manual fixer for files that would be counted in wrong categories
 # "filename": "correctSection"
 fileSectionFixer = {
@@ -203,12 +207,12 @@ for line in map_file:
         file_size = int(line_split[2], 16)
         obj_file = line_split[3].strip()
         objFileSplit = obj_file.split("/")
+        objFileName = objFileSplit[-1].split(".o")[0]
 
         fileData = {"name": obj_file, "vram": obj_vram, "size": file_size, "section": section, "symbols": []}
         mapFileList.append(fileData)
 
         if (section == ".text"):
-            objFileName = objFileSplit[-1].split(".o")[0]
             srcCat = obj_file.split("/")[2]
             if srcCat in srcCategoriesFixer:
                 srcCat = srcCategoriesFixer[srcCat]
@@ -229,6 +233,8 @@ for line in map_file:
                 assetCat = obj_file.split("/")[2]
                 if assetCat in assetsTracker:
                     assetsTracker[assetCat]["currentSize"] += file_size
+                elif assetCat in ignoredAssets:
+                    pass
                 else:
                     eprint(f"Found file '{obj_file}' in unknown asset category '{assetCat}'")
                     eprint("I'll ignore this for now, but please fix it!")