Assembly modding support & objdiff + decomp.me integration (#15)

* Add initial asm build support

* WIP decomp.me and links

README.example.md

@@ -73,7 +73,7 @@ Linux:
 ------
 - Install [ninja](https://github.com/ninja-build/ninja/wiki/Pre-built-Ninja-packages).
 - For non-x86(_64) platforms: Install wine from your package manager.
-  - For x86(_64), [WiBo](https://github.com/decompals/WiBo), a minimal 32-bit Windows binary wrapper, will be automatically downloaded and used.
+  - For x86(_64), [wibo](https://github.com/decompals/wibo), a minimal 32-bit Windows binary wrapper, will be automatically downloaded and used.
 
 Building
 ========

config/GAMEID/config.example.yml

@@ -63,6 +63,11 @@ symbols_known: false
 # this can be set to false.
 fill_gaps: true
 
+# (optional) By default, emitted objects will "export" all symbols (force active).
+# This is useful to prevent the linker from removing any symbols.
+# Individual symbols can be excluded using `noexport` in the symbols file.
+export_all: true
+
 # (optional) Custom template for `ldscript.lcf`. Avoid unless necessary.
 # See https://github.com/encounter/decomp-toolkit/blob/main/assets/ldscript.lcf
 ldscript_template: config/GAMEID/module/ldscript.tpl
@@ -99,7 +104,14 @@ modules:
   # See https://github.com/encounter/decomp-toolkit/blob/main/assets/ldscript_partial.lcf
   ldscript_template: config/GAMEID/module/ldscript.tpl
 
+  # (optional) By default, every REL is linked with every other REL.
+  # Some games link RELs individually, so the module IDs are not unique.
+  # To support this, `links` overrides which other modules are included in this module's analysis.
+  # The DOL is always included, and does not need to be specified.
+  links: [module2] # This module will be linked with the DOL and "module2".
+
 # (optional) Configuration for asset extraction.
+# For modules, this goes in the module's configuration above.
 extract:
 
 - # The symbol name to extract.

configure.py

@@ -56,6 +56,12 @@ parser.add_argument(
     default=Path("build"),
     help="base build directory (default: build)",
 )
+parser.add_argument(
+    "--binutils",
+    dest="binutils",
+    type=Path,
+    help="path to binutils (optional)",
+)
 parser.add_argument(
     "--compilers",
     dest="compilers",
@@ -110,6 +116,7 @@ version_num = VERSIONS.index(config.version)
 # Apply arguments
 config.build_dir = args.build_dir
 config.build_dtk_path = args.build_dtk
+config.binutils_path = args.binutils
 config.compilers_path = args.compilers
 config.debug = args.debug
 config.generate_map = args.map
@@ -118,14 +125,22 @@ if not is_windows():
     config.wrapper = args.wrapper
 
 # Tool versions
+config.binutils_tag = "2.42-1"
 config.compilers_tag = "20231018"
-config.dtk_tag = "v0.6.2"
+config.dtk_tag = "v0.7.4"
 config.sjiswrap_tag = "v1.1.1"
-config.wibo_tag = "0.6.9"
+config.wibo_tag = "0.6.11"
 
 # Project
 config.config_path = Path("config") / config.version / "config.yml"
 config.check_sha_path = Path("config") / config.version / "build.sha1"
+config.asflags = [
+    "-mgekko",
+    "--strip-local-absolute",
+    "-I include",
+    f"-I build/{config.version}/include",
+    f"--defsym version={version_num}",
+]
 config.ldflags = [
     "-fp hardware",
     "-nodefaults",

docs/dependencies.md

@@ -1,6 +1,8 @@
-# Dependencies
+Dependencies
+============
 
-## Windows:
+Windows:
+--------
 
 On Windows, it's **highly recommended** to use native tooling. WSL or msys2 are **not** required.  
 When running under WSL, [objdiff](#diffing) is unable to get filesystem notifications for automatic rebuilds.
@@ -10,8 +12,8 @@ When running under WSL, [objdiff](#diffing) is unable to get filesystem notifica
 - Download [ninja](https://github.com/ninja-build/ninja/releases) and add it to `%PATH%`.
   - Quick install via pip: `pip install ninja`
 
-## macOS:
-
+macOS:
+------
 - Install [ninja](https://github.com/ninja-build/ninja/wiki/Pre-built-Ninja-packages):
   ```
   brew install ninja
@@ -26,8 +28,8 @@ After OS upgrades, if macOS complains about `Wine Crossover.app` being unverifie
 sudo xattr -rd com.apple.quarantine '/Applications/Wine Crossover.app'
 ```
 
-## Linux:
-
+Linux:
+------
 - Install [ninja](https://github.com/ninja-build/ninja/wiki/Pre-built-Ninja-packages).
 - For non-x86(_64) platforms: Install wine from your package manager.
-  - For x86(_64), [WiBo](https://github.com/decompals/WiBo), a minimal 32-bit Windows binary wrapper, will be automatically downloaded and used.
+  - For x86(_64), [wibo](https://github.com/decompals/wibo), a minimal 32-bit Windows binary wrapper, will be automatically downloaded and used.

docs/symbols.md

@@ -32,4 +32,6 @@ All attributes are optional, and are separated by spaces.
 - `data:` The data type used when writing disassembly. `byte`, `2byte`, `4byte`, `8byte`, `float`, `double`, `string`, `wstring`, `string_table`, or `wstring_table`.
 - `hidden` Marked as "hidden" in the generated object. (Only used for extab)
 - `force_active` Marked as ["exported"](comment_section.md) in the generated object, and added to `FORCEACTIVE` in the generated `ldscript.lcf`. Prevents the symbol from being deadstripped.
-- `noreloc` Prevents the _contents_ of the symbol from being interpreted as addresses. Used for objects containing data that look like pointers, but aren't.
+- `noreloc` Prevents the _contents_ of the symbol from being interpreted as addresses. Used for objects containing data that look like pointers, but aren't.
+- `noexport` When `export_all` is enabled, this excludes the symbol from being exported. Used for symbols that are affected by linker `-code_merging`.
+- `stripped` Indicates a symbol that was stripped by the linker. Used for symbols that affect the [common BSS inflation bug](common_bss.md), despite not existing in the final binary.

tools/decompctx.py

@@ -13,45 +13,50 @@
 import argparse
 import os
 import re
-from typing import List, Set
+from typing import List
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
 root_dir = os.path.abspath(os.path.join(script_dir, ".."))
 src_dir = os.path.join(root_dir, "src")
-include_dir = os.path.join(root_dir, "include")
+include_dirs = [
+    os.path.join(root_dir, "include"),
+    # Add additional include directories here
+]
 
 include_pattern = re.compile(r'^#include\s*[<"](.+?)[>"]$')
 guard_pattern = re.compile(r"^#ifndef\s+(.*)$")
 
-defines: Set[str] = set()
+defines = set()
 
 
-def import_h_file(in_file: str, r_path: str) -> str:
+def import_h_file(in_file: str, r_path: str, deps: List[str]) -> str:
     rel_path = os.path.join(root_dir, r_path, in_file)
-    inc_path = os.path.join(include_dir, in_file)
     if os.path.exists(rel_path):
-        return import_c_file(rel_path)
-    elif os.path.exists(inc_path):
-        return import_c_file(inc_path)
+        return import_c_file(rel_path, deps)
+    for include_dir in include_dirs:
+        inc_path = os.path.join(include_dir, in_file)
+        if os.path.exists(inc_path):
+            return import_c_file(inc_path, deps)
     else:
         print("Failed to locate", in_file)
-        exit(1)
+        return ""
 
 
-def import_c_file(in_file: str) -> str:
+def import_c_file(in_file: str, deps: List[str]) -> str:
     in_file = os.path.relpath(in_file, root_dir)
+    deps.append(in_file)
     out_text = ""
 
     try:
         with open(in_file, encoding="utf-8") as file:
-            out_text += process_file(in_file, list(file))
+            out_text += process_file(in_file, list(file), deps)
     except Exception:
         with open(in_file) as file:
-            out_text += process_file(in_file, list(file))
+            out_text += process_file(in_file, list(file), deps)
     return out_text
 
 
-def process_file(in_file: str, lines: List[str]) -> str:
+def process_file(in_file: str, lines: List[str], deps: List[str]) -> str:
     out_text = ""
     for idx, line in enumerate(lines):
         guard_match = guard_pattern.match(line.strip())
@@ -64,7 +69,7 @@ def process_file(in_file: str, lines: List[str]) -> str:
         include_match = include_pattern.match(line.strip())
         if include_match and not include_match[1].endswith(".s"):
             out_text += f'/* "{in_file}" line {idx} "{include_match[1]}" */\n'
-            out_text += import_h_file(include_match[1], os.path.dirname(in_file))
+            out_text += import_h_file(include_match[1], os.path.dirname(in_file), deps)
             out_text += f'/* end "{include_match[1]}" */\n'
         else:
             out_text += line
@@ -72,7 +77,11 @@ def process_file(in_file: str, lines: List[str]) -> str:
     return out_text
 
 
-def main() -> None:
+def sanitize_path(path: str) -> str:
+    return path.replace("\\", "/").replace(" ", "\\ ")
+
+
+def main():
     parser = argparse.ArgumentParser(
         description="""Create a context file which can be used for decomp.me"""
     )
@@ -80,13 +89,32 @@ def main() -> None:
         "c_file",
         help="""File from which to create context""",
     )
+    parser.add_argument(
+        "-o",
+        "--output",
+        help="""Output file""",
+        default="ctx.c",
+    )
+    parser.add_argument(
+        "-d",
+        "--depfile",
+        help="""Dependency file""",
+    )
     args = parser.parse_args()
 
-    output = import_c_file(args.c_file)
+    deps = []
+    output = import_c_file(args.c_file, deps)
 
-    with open(os.path.join(root_dir, "ctx.c"), "w", encoding="utf-8") as f:
+    with open(os.path.join(root_dir, args.output), "w", encoding="utf-8") as f:
         f.write(output)
 
+    if args.depfile:
+        with open(os.path.join(root_dir, args.depfile), "w", encoding="utf-8") as f:
+            f.write(sanitize_path(args.output) + ":")
+            for dep in deps:
+                path = sanitize_path(dep)
+                f.write(f" \\\n\t{path}")
+
 
 if __name__ == "__main__":
     main()

tools/download_tool.py

@@ -22,6 +22,24 @@ from typing import Callable, Dict
 from pathlib import Path
 
 
+def binutils_url(tag):
+    uname = platform.uname()
+    system = uname.system.lower()
+    arch = uname.machine.lower()
+    if system == "darwin":
+        system = "macos"
+        arch = "universal"
+    elif arch == "amd64":
+        arch = "x86_64"
+
+    repo = "https://github.com/encounter/gc-wii-binutils"
+    return f"{repo}/releases/download/{tag}/{system}-{arch}.zip"
+
+
+def compilers_url(tag: str) -> str:
+    return f"https://files.decomp.dev/compilers_{tag}.zip"
+
+
 def dtk_url(tag: str) -> str:
     uname = platform.uname()
     suffix = ""
@@ -48,15 +66,12 @@ def wibo_url(tag: str) -> str:
     return f"{repo}/releases/download/{tag}/wibo"
 
 
-def compilers_url(tag: str) -> str:
-    return f"https://files.decomp.dev/compilers_{tag}.zip"
-
-
 TOOLS: Dict[str, Callable[[str], str]] = {
+    "binutils": binutils_url,
+    "compilers": compilers_url,
     "dtk": dtk_url,
     "sjiswrap": sjiswrap_url,
     "wibo": wibo_url,
-    "compilers": compilers_url,
 }
 
 
@@ -77,7 +92,11 @@ def main() -> None:
             data = io.BytesIO(response.read())
             with zipfile.ZipFile(data) as f:
                 f.extractall(output)
-            output.touch(mode=0o755)
+            # Make all files executable
+            for root, _, files in os.walk(output):
+                for name in files:
+                    os.chmod(os.path.join(root, name), 0o755)
+            output.touch(mode=0o755)  # Update dir modtime
         else:
             with open(output, "wb") as f:
                 shutil.copyfileobj(response, f)