From 898b3a375337ecf333343357b32096f0dd3de2d5 Mon Sep 17 00:00:00 2001 From: Henrique Gemignani Passos Lima Date: Wed, 21 Jan 2026 13:04:18 +0200 Subject: [PATCH] Update to latest dtk-template --- .github/workflows/build.yml | 35 +-- configure.py | 101 +++++--- tools/changes_fmt.py | 162 ++++++++++++ tools/decompctx.py | 48 +++- tools/download_tool.py | 59 +++-- tools/project.py | 474 ++++++++++++++++++++++++++---------- tools/upload_progress.py | 79 ------ 7 files changed, 675 insertions(+), 283 deletions(-) create mode 100755 tools/changes_fmt.py mode change 100755 => 100644 tools/decompctx.py delete mode 100755 tools/upload_progress.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cb3fae20..746063f8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,35 +19,40 @@ jobs: - name: Checkout uses: actions/checkout@v4 with: + fetch-depth: 0 submodules: recursive # Set Git config - name: Git config run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + # Normalize file mod times + - name: Restore timestamps + run: | + uv run https://raw.githubusercontent.com/MestreLion/git-tools/refs/tags/v2022.12/git-restore-mtime \ + --merge --commit-time + # Copy the original files to the workspace - name: Prepare - run: cp -R /orig . + run: cp -a /orig . + + # Restore cached files + - name: Cache build + uses: actions/cache@v4 + with: + path: | + build + .ninja_deps + .ninja_log + key: ${{ runner.os }}-${{ matrix.version }}-${{ github.sha }} + restore-keys: ${{ runner.os }}-${{ matrix.version }}- # Build the project - name: Build run: | python configure.py --map --version ${{ matrix.version }} \ --binutils /binutils --compilers /compilers - ninja all_source build/${{ matrix.version }}/progress.json \ - build/${{ matrix.version }}/report.json - - # Upload progress if we're on the main branch - - name: Upload progress - if: github.ref == 'refs/heads/main' - continue-on-error: true - env: - PROGRESS_SLUG: prime - PROGRESS_API_KEY: ${{ secrets.PROGRESS_API_KEY }} - run: | - python tools/upload_progress.py -b https://progress.decomp.club/ \ - -p $PROGRESS_SLUG -v ${{ matrix.version }} \ - build/${{ matrix.version }}/progress.json + ninja all_source progress build/${{ matrix.version }}/report.json # Upload map files - name: Upload map diff --git a/configure.py b/configure.py index 89573255..42d1093c 100755 --- a/configure.py +++ b/configure.py @@ -15,10 +15,9 @@ import argparse import sys from pathlib import Path -from typing import List, Sequence, Union +from typing import Any, Dict, List from tools.project import ( - BuildConfigUnit, Object, ProgressCategory, ProjectConfig, @@ -122,6 +121,12 @@ parser.add_argument( type=Path, help="path to sjiswrap.exe (optional)", ) +parser.add_argument( + "--ninja", + metavar="BINARY", + type=Path, + help="path to ninja binary (optional)", +) parser.add_argument( "--verbose", action="store_true", @@ -133,6 +138,13 @@ parser.add_argument( action="store_true", help="builds equivalent (but non-matching) or modded objects", ) +parser.add_argument( + "--warn", + dest="warn", + type=str, + choices=["all", "off", "error"], + help="how to handle warnings", +) parser.add_argument( "--no-progress", dest="progress", @@ -159,6 +171,7 @@ config.compilers_path = args.compilers config.generate_map = args.map config.non_matching = args.non_matching config.sjiswrap_path = args.sjiswrap +config.ninja_path = args.ninja config.progress = args.progress if not is_windows(): config.wrapper = args.wrapper @@ -168,11 +181,11 @@ if not config.non_matching: # Tool versions config.binutils_tag = "2.42-1" -config.compilers_tag = "20240706" +config.compilers_tag = "20251118" config.dtk_tag = "v1.4.0" -config.objdiff_tag = "v2.7.1" -config.sjiswrap_tag = "v1.2.0" -config.wibo_tag = "0.6.11" +config.objdiff_tag = "v3.5.1" +config.sjiswrap_tag = "v1.2.2" +config.wibo_tag = "1.0.0" # Project config.config_path = Path("config") / config.version / "config.yml" @@ -182,18 +195,24 @@ config.asflags = [ "--strip-local-absolute", "-I include", f"-I build/{config.version}/include", - f"--defsym version={version_num}", + f"--defsym BUILD_VERSION={version_num}", ] config.ldflags = [ "-fp hardware", "-nodefaults", ] if args.debug: - config.ldflags.append("-g") + config.ldflags.append("-g") # Or -gdwarf-2 for Wii linkers if args.map: config.ldflags.append("-mapunused") + # config.ldflags.append("-listclosure") # For Wii linkers -config.build_rels = False +# Use for any additional files that should cause a re-configure when modified +config.reconfig_deps = [] + +# Optional numeric ID for decomp.me preset +# Can be overridden in libraries or objects +config.scratch_preset_id = None # Base flags, common to most GC/Wii games. # Generally leave untouched, with overrides added below. @@ -224,10 +243,19 @@ cflags_base = [ # Debug flags if args.debug: + # Or -sym dwarf-2 for Wii compilers cflags_base.extend(["-sym on", "-DDEBUG=1"]) else: cflags_base.append("-DNDEBUG=1") +# Warning flags +if args.warn == "all": + cflags_base.append("-W all") +elif args.warn == "off": + cflags_base.append("-W off") +elif args.warn == "error": + cflags_base.append("-W error") + # Dolphin flags cflags_dolphin = [ *cflags_base, @@ -342,18 +370,17 @@ config.linker_version = "GC/1.3.2" # Helper function for Dolphin libraries -def DolphinLib(lib_name, objects): +def DolphinLib(lib_name: str, objects: List[Object]) -> Dict[str, Any]: return { - "lib": lib_name + "D" if args.debug else "", + "lib": lib_name, "mw_version": "GC/1.2.5n", - "cflags": cflags_dolphin, - "host": False, + "cflags": cflags_base, "progress_category": "sdk", "objects": objects, - "shift_jis": True, } + def TrkLib(lib_name, objects): return { "lib": lib_name + "D" if args.debug else "", @@ -417,7 +444,7 @@ def MusyX( # Helper function for REL script objects -def Rel(lib_name, objects): +def Rel(lib_name: str, objects: List[Object]) -> Dict[str, Any]: return { "lib": lib_name, "mw_version": "GC/1.3.2", @@ -429,11 +456,9 @@ def Rel(lib_name, objects): } -Matching = True # Object matches and should be linked -NonMatching = False # Object does not match and should not be linked -Equivalent = ( - config.non_matching -) # Object should be linked when configured with --non-matching +Matching = True # Object matches and should be linked +NonMatching = False # Object does not match and should not be linked +Equivalent = config.non_matching # Object should be linked when configured with --non-matching # Object is only matching for specific versions @@ -2481,34 +2506,36 @@ config.libs = [ ), ] -# Disable missing return type warnings for incomplete objects -for lib in config.libs: - for obj in lib["objects"]: - if not obj.completed: - obj.options["extra_clang_flags"].append("-Wno-return-type") + +# Optional callback to adjust link order. This can be used to add, remove, or reorder objects. +# This is called once per module, with the module ID and the current link order. +# +# For example, this adds "dummy.c" to the end of the DOL link order if configured with --non-matching. +# "dummy.c" *must* be configured as a Matching (or Equivalent) object in order to be linked. +def link_order_callback(module_id: int, objects: List[str]) -> List[str]: + # Don't modify the link order for matching builds + return objects -def link_order_callback( - module_id: int, units: List[str] -) -> Sequence[Union[str, BuildConfigUnit]]: - # if module_id == 0: # DOL - # return units + [ - # {"object": "dummy.o", "name": "dummy.c", "autogenerated": False} - # ] - return units +# Uncomment to enable the link order callback. +# config.link_order_callback = link_order_callback -config.link_order_callback = link_order_callback - # Optional extra categories for progress tracking +# Adjust as desired for your project config.progress_categories = [ ProgressCategory("game", "Game"), ProgressCategory("core", "Core Engine (Kyoto)"), ProgressCategory("sdk", "SDK"), ProgressCategory("third_party", "Third Party"), ] -config.progress_all = True config.progress_each_module = args.verbose +# Optional extra arguments to `objdiff-cli report generate` +config.progress_report_args = [ + # Marks relocations as mismatching if the target value is different + # Default is "functionRelocDiffs=none", which is most lenient + # "--config functionRelocDiffs=data_value", +] config.progress_modules = False config.progress_use_fancy = True config.progress_code_fancy_frac = 1499 @@ -2520,7 +2547,7 @@ if args.mode == "configure": # Write build.ninja and objdiff.json generate_build(config) elif args.mode == "progress": - # Print progress and write progress.json + # Print progress information calculate_progress(config) else: sys.exit("Unknown mode: " + args.mode) diff --git a/tools/changes_fmt.py b/tools/changes_fmt.py new file mode 100755 index 00000000..153f40f6 --- /dev/null +++ b/tools/changes_fmt.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 + +from argparse import ArgumentParser +import os +import json +from pathlib import Path +from typing import Optional, Tuple + +script_dir = os.path.dirname(os.path.realpath(__file__)) +root_dir = os.path.abspath(os.path.join(script_dir, "..")) + + +UNIT_KEYS_TO_DIFF = [ + "fuzzy_match_percent", + "matched_code_percent", + "matched_data_percent", + "complete_code_percent", + "complete_data_percent", +] + +FUNCTION_KEYS_TO_DIFF = [ + "fuzzy_match_percent", +] + +Change = Tuple[str, str, float, float] + + +def format_float(value: float) -> str: + if value < 100.0 and value > 99.99: + value = 99.99 + return "%6.2f" % value + + +def get_changes(changes_file: str) -> Tuple[list[Change], list[Change]]: + changes_file = os.path.relpath(changes_file, root_dir) + with open(changes_file, "r") as f: + changes_json = json.load(f) + + regressions = [] + progressions = [] + + def diff_key(object_name: Optional[str], object: dict, key: str): + from_value = object.get("from", {}).get(key, 0.0) + to_value = object.get("to", {}).get(key, 0.0) + key = key.removesuffix("_percent") + change = (object_name, key, from_value, to_value) + if from_value > to_value: + regressions.append(change) + elif to_value > from_value: + progressions.append(change) + + for key in UNIT_KEYS_TO_DIFF: + diff_key(None, changes_json, key) + + for unit in changes_json.get("units", []): + unit_name = unit["name"] + for key in UNIT_KEYS_TO_DIFF: + diff_key(unit_name, unit, key) + # Ignore sections + for func in unit.get("functions", []): + func_name = func["name"] + for key in FUNCTION_KEYS_TO_DIFF: + diff_key(func_name, func, key) + + return regressions, progressions + + +def generate_changes_plaintext(changes: list[Change]) -> str: + if len(changes) == 0: + return "" + + table_total_width = 136 + percents_max_len = 7 + 4 + 7 + key_max_len = max(len(key) for _, key, _, _ in changes) + name_max_len = max(len(name or "Total") for name, _, _, _ in changes) + max_width_for_name_col = table_total_width - 3 - key_max_len - 3 - percents_max_len + name_max_len = min(max_width_for_name_col, name_max_len) + + out_lines = [] + for name, key, from_value, to_value in changes: + if name is None: + name = "Total" + if len(name) > name_max_len: + name = name[: name_max_len - len("[...]")] + "[...]" + out_lines.append( + f"{name:>{name_max_len}} | {key:<{key_max_len}} | {format_float(from_value)}% -> {format_float(to_value)}%" + ) + + return "\n".join(out_lines) + + +def generate_changes_markdown(changes: list[Change], description: str) -> str: + if len(changes) == 0: + return "" + + out_lines = [] + name_max_len = 100 + + out_lines.append("
") + out_lines.append( + f"Detected {len(changes)} {description} compared to the base:" + ) + out_lines.append("") # Must include a blank line before a table + out_lines.append("| Name | Type | Before | After |") + out_lines.append("| ---- | ---- | ------ | ----- |") + + for name, key, from_value, to_value in changes: + if name is None: + name = "Total" + else: + if len(name) > name_max_len: + name = name[: name_max_len - len("...")] + "..." + name = f"`{name}`" # Surround with backticks + key = key.replace("_", " ").capitalize() + out_lines.append( + f"| {name} | {key} | {format_float(from_value)}% | {format_float(to_value)}% |" + ) + + out_lines.append("
") + + return "\n".join(out_lines) + + +def main(): + parser = ArgumentParser(description="Format objdiff-cli report changes.") + parser.add_argument( + "report_changes_file", + type=Path, + help="""path to the JSON file containing the changes, generated by objdiff-cli.""", + ) + parser.add_argument( + "-o", + "--output", + type=Path, + help="""Output file (prints to console if unspecified)""", + ) + parser.add_argument( + "--all", + action="store_true", + help="""Includes progressions as well.""", + ) + args = parser.parse_args() + + regressions, progressions = get_changes(args.report_changes_file) + + if args.output: + markdown_output = generate_changes_markdown(regressions, "regressions") + if args.all: + markdown_output += generate_changes_markdown(progressions, "progressions") + with open(args.output, "w", encoding="utf-8") as f: + f.write(markdown_output) + else: + if args.all: + changes = progressions + regressions + else: + changes = regressions + text_output = generate_changes_plaintext(changes) + print(text_output) + + +if __name__ == "__main__": + main() diff --git a/tools/decompctx.py b/tools/decompctx.py old mode 100755 new mode 100644 index f2f31dfa..02eb2450 --- a/tools/decompctx.py +++ b/tools/decompctx.py @@ -11,6 +11,7 @@ ### import argparse +import fnmatch import os import re from typing import List @@ -19,6 +20,7 @@ script_dir = os.path.dirname(os.path.realpath(__file__)) root_dir = os.path.abspath(os.path.join(script_dir, "..")) src_dir = os.path.join(root_dir, "src") include_dirs: List[str] = [] # Set with -I flag +exclude_globs: List[str] = [] # Set with -x flag include_pattern = re.compile(r'^#\s*include\s*[<"](.+?)[>"]') guard_pattern = re.compile(r"^#\s*ifndef\s+(.*)$") @@ -28,6 +30,23 @@ defines = set() deps = [] +def generate_prelude(defines) -> str: + if len(defines) == 0: + return "" + + out_text = "/* decompctx prelude */\n" + for define in defines: + parts = define.split("=", 1) + if len(parts) == 2: + macro_name, macro_val = parts + out_text += f"#define {macro_name} {macro_val}\n" + else: + out_text += f"#define {parts[0]}\n" + out_text += "/* end decompctx prelude */\n\n" + + return out_text + + def import_h_file(in_file: str, r_path: str) -> str: rel_path = os.path.join(root_dir, r_path, in_file) if os.path.exists(rel_path): @@ -73,8 +92,17 @@ def process_file(in_file: str, lines: List[str]) -> str: print("Processing file", in_file) include_match = include_pattern.match(line.strip()) if include_match and not include_match[1].endswith(".s"): + excluded = False + for glob in exclude_globs: + if fnmatch.fnmatch(include_match[1], glob): + excluded = True + break + out_text += f'/* "{in_file}" line {idx} "{include_match[1]}" */\n' - out_text += import_h_file(include_match[1], os.path.dirname(in_file)) + if excluded: + out_text += "/* Skipped excluded file */\n" + else: + out_text += import_h_file(include_match[1], os.path.dirname(in_file)) out_text += f'/* end "{include_match[1]}" */\n' else: out_text += line @@ -111,13 +139,29 @@ def main(): help="""Include directory""", action="append", ) + parser.add_argument( + "-x", + "--exclude", + help="""Excluded file name glob""", + action="append", + ) + parser.add_argument( + "-D", + "--define", + help="""Macro definition""", + action="append", + ) args = parser.parse_args() if args.include is None: exit("No include directories specified") global include_dirs include_dirs = args.include - output = import_c_file(args.c_file) + global exclude_globs + exclude_globs = args.exclude or [] + prelude_defines = args.define or [] + output = generate_prelude(prelude_defines) + output += import_c_file(args.c_file) with open(os.path.join(root_dir, args.output), "w", encoding="utf-8") as f: f.write(output) diff --git a/tools/download_tool.py b/tools/download_tool.py index f4512d01..22465267 100644 --- a/tools/download_tool.py +++ b/tools/download_tool.py @@ -78,8 +78,14 @@ def sjiswrap_url(tag: str) -> str: def wibo_url(tag: str) -> str: + uname = platform.uname() + arch = uname.machine.lower() + system = uname.system.lower() + if system == "darwin": + arch = "macos" + repo = "https://github.com/decompals/wibo" - return f"{repo}/releases/download/{tag}/wibo" + return f"{repo}/releases/download/{tag}/wibo-{arch}" TOOLS: Dict[str, Callable[[str], str]] = { @@ -92,6 +98,23 @@ TOOLS: Dict[str, Callable[[str], str]] = { } +def download(url, response, output) -> None: + if url.endswith(".zip"): + data = io.BytesIO(response.read()) + with zipfile.ZipFile(data) as f: + f.extractall(output) + # Make all files executable + for root, _, files in os.walk(output): + for name in files: + os.chmod(os.path.join(root, name), 0o755) + output.touch(mode=0o755) # Update dir modtime + else: + with open(output, "wb") as f: + shutil.copyfileobj(response, f) + st = os.stat(output) + os.chmod(output, st.st_mode | stat.S_IEXEC) + + def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("tool", help="Tool name") @@ -104,21 +127,25 @@ def main() -> None: print(f"Downloading {url} to {output}") req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"}) - with urllib.request.urlopen(req) as response: - if url.endswith(".zip"): - data = io.BytesIO(response.read()) - with zipfile.ZipFile(data) as f: - f.extractall(output) - # Make all files executable - for root, _, files in os.walk(output): - for name in files: - os.chmod(os.path.join(root, name), 0o755) - output.touch(mode=0o755) # Update dir modtime - else: - with open(output, "wb") as f: - shutil.copyfileobj(response, f) - st = os.stat(output) - os.chmod(output, st.st_mode | stat.S_IEXEC) + try: + with urllib.request.urlopen(req) as response: + download(url, response, output) + except urllib.error.URLError as e: + if str(e).find("CERTIFICATE_VERIFY_FAILED") == -1: + raise e + try: + import certifi + import ssl + except ImportError: + print( + '"certifi" module not found. Please install it using "python -m pip install certifi".' + ) + return + + with urllib.request.urlopen( + req, context=ssl.create_default_context(cafile=certifi.where()) + ) as response: + download(url, response, output) if __name__ == "__main__": diff --git a/tools/project.py b/tools/project.py index d50adab7..b1417ae6 100644 --- a/tools/project.py +++ b/tools/project.py @@ -18,12 +18,10 @@ import platform import sys from pathlib import Path from typing import ( + IO, Any, Callable, - Sequence, - cast, Dict, - IO, Iterable, List, Optional, @@ -31,6 +29,7 @@ from typing import ( Tuple, TypedDict, Union, + cast, ) from . import ninja_syntax @@ -46,6 +45,9 @@ if sys.platform == "cygwin": Library = Dict[str, Any] +PrecompiledHeader = Dict[str, Any] + + class Object: def __init__(self, completed: bool, name: str, **options: Any) -> None: self.name = name @@ -55,10 +57,10 @@ class Object: "asflags": None, "asm_dir": None, "cflags": None, + "extab_padding": None, "extra_asflags": [], "extra_cflags": [], "extra_clang_flags": [], - "host": None, "lib": None, "mw_version": None, "progress_category": None, @@ -74,7 +76,6 @@ class Object: self.asm_path: Optional[Path] = None self.src_obj_path: Optional[Path] = None self.asm_obj_path: Optional[Path] = None - self.host_obj_path: Optional[Path] = None self.ctx_path: Optional[Path] = None def resolve(self, config: "ProjectConfig", lib: Library) -> "Object": @@ -92,7 +93,7 @@ class Object: set_default("add_to_all", True) set_default("asflags", config.asflags) set_default("asm_dir", config.asm_dir) - set_default("host", False) + set_default("extab_padding", None) set_default("mw_version", config.linker_version) set_default("scratch_preset_id", config.scratch_preset_id) set_default("shift_jis", config.shift_jis) @@ -122,7 +123,6 @@ class Object: base_name = Path(self.name).with_suffix("") obj.src_obj_path = build_dir / "src" / f"{base_name}.o" obj.asm_obj_path = build_dir / "mod" / f"{base_name}.o" - obj.host_obj_path = build_dir / "host" / f"{base_name}.o" obj.ctx_path = build_dir / "src" / f"{base_name}.ctx" return obj @@ -154,6 +154,7 @@ class ProjectConfig: self.wrapper: Optional[Path] = None # If None, download wibo on Linux self.sjiswrap_tag: Optional[str] = None # Git tag self.sjiswrap_path: Optional[Path] = None # If None, download + self.ninja_path: Optional[Path] = None # If None, use system PATH self.objdiff_tag: Optional[str] = None # Git tag self.objdiff_path: Optional[Path] = None # If None, download @@ -166,6 +167,9 @@ class ProjectConfig: self.asflags: Optional[List[str]] = None # Assembler flags self.ldflags: Optional[List[str]] = None # Linker flags self.libs: Optional[List[Library]] = None # List of libraries + self.precompiled_headers: Optional[List[PrecompiledHeader]] = ( + None # List of precompiled headers + ) self.linker_version: Optional[str] = None # mwld version self.version: Optional[str] = None # Version name self.warn_missing_config: bool = False # Warn on missing unit configuration @@ -193,13 +197,18 @@ class ProjectConfig: self.scratch_preset_id: Optional[int] = ( None # Default decomp.me preset ID for scratches ) - self.link_order_callback: Optional[ - Callable[[int, List[str]], Sequence[Union[str, BuildConfigUnit]]] - ] = None # Callback to add/remove/reorder units within a module + self.link_order_callback: Optional[Callable[[int, List[str]], List[str]]] = ( + None # Callback to add/remove/reorder units within a module + ) + self.context_exclude_globs: List[ + str + ] = [] # Globs to exclude from context files + self.context_defines: List[ + str + ] = [] # Macros to define at the top of context files - # Progress output, progress.json and report.json config + # Progress output and report.json config self.progress = True # Enable report.json generation and CLI progress output - self.progress_all: bool = True # Include combined "all" category self.progress_modules: bool = True # Include combined "modules" category self.progress_each_module: bool = ( False # Include individual modules, disable for large numbers of modules @@ -208,6 +217,9 @@ class ProjectConfig: self.print_progress_categories: Union[bool, List[str]] = ( True # Print additional progress categories in the CLI progress output ) + self.progress_report_args: Optional[List[str]] = ( + None # Flags to `objdiff-cli report generate` + ) # Progress fancy printing self.progress_use_fancy: bool = False @@ -273,8 +285,8 @@ class ProjectConfig: def use_wibo(self) -> bool: return ( self.wibo_tag is not None - and sys.platform == "linux" - and platform.machine() in ("i386", "x86_64") + and (sys.platform == "linux" or sys.platform == "darwin") + and platform.machine() in ("i386", "x86_64", "aarch64", "arm64") and self.wrapper is None ) @@ -298,19 +310,62 @@ def file_is_c(path: Path) -> bool: def file_is_cpp(path: Path) -> bool: - return path.suffix.lower() in (".cc", ".cp", ".cpp", ".cxx") + return path.suffix.lower() in (".cc", ".cp", ".cpp", ".cxx", ".pch++") def file_is_c_cpp(path: Path) -> bool: return file_is_c(path) or file_is_cpp(path) +_listdir_cache = {} + + +def check_path_case(path: Path): + parts = path.parts + if path.is_absolute(): + curr = Path(parts[0]) + start = 1 + else: + curr = Path(".") + start = 0 + + for part in parts[start:]: + if curr in _listdir_cache: + entries = _listdir_cache[curr] + else: + try: + entries = os.listdir(curr) + except (FileNotFoundError, PermissionError): + sys.exit(f"Cannot access: {curr}") + _listdir_cache[curr] = entries + + for entry in entries: + if entry.lower() == part.lower(): + curr = curr / entry + break + else: + sys.exit(f"Cannot resolve: {path}") + + if path != curr: + print(f"⚠️ Case mismatch: expected={path} actual={curr}") + + def make_flags_str(flags: Optional[List[str]]) -> str: if flags is None: return "" return " ".join(flags) +def get_pch_out_name(config: ProjectConfig, pch: PrecompiledHeader) -> str: + pch_rel_path = Path(pch["source"]) + pch_out_name = pch_rel_path.with_suffix(".mch") + # Use absolute path as a workaround to allow this target to be matched with absolute paths in depfiles. + # + # Without this any object which includes the PCH would depend on the .mch filesystem entry but not the + # corresponding Ninja task, so the MCH would not be implicitly rebuilt when the PCH is modified. + return os.path.abspath(config.out_path() / "include" / pch_out_name) + + # Unit configuration class BuildConfigUnit(TypedDict): object: Optional[str] @@ -372,19 +427,16 @@ def load_build_config( modules: List[BuildConfigModule] = [build_config, *build_config["modules"]] for module in modules: unit_names = list(map(lambda u: u["name"], module["units"])) - new_units = config.link_order_callback(module["module_id"], unit_names) + unit_names = config.link_order_callback(module["module_id"], unit_names) units: List[BuildConfigUnit] = [] - for new_unit in new_units: - if isinstance(new_unit, str): - units.append( - # Find existing unit or create a new one - next( - (u for u in module["units"] if u["name"] == new_unit), - {"object": None, "name": new_unit, "autogenerated": False}, - ) + for unit_name in unit_names: + units.append( + # Find existing unit or create a new one + next( + (u for u in module["units"] if u["name"] == unit_name), + {"object": None, "name": unit_name, "autogenerated": False}, ) - else: - units.append(new_unit) + ) module["units"] = units return build_config @@ -427,6 +479,7 @@ def generate_build_ninja( if config.linker_version is None: sys.exit("ProjectConfig.linker_version missing") n.variable("mw_version", Path(config.linker_version)) + n.variable("objdiff_report_args", make_flags_str(config.progress_report_args)) n.newline() ### @@ -435,7 +488,6 @@ def generate_build_ninja( n.comment("Tooling") build_path = config.out_path() - progress_path = build_path / "progress.json" report_path = build_path / "report.json" build_tools_path = config.build_dir / "tools" download_tool = config.tools_dir / "download_tool.py" @@ -448,7 +500,7 @@ def generate_build_ninja( decompctx = config.tools_dir / "decompctx.py" n.rule( name="decompctx", - command=f"$python {decompctx} $in -o $out -d $out.d $includes", + command=f"$python {decompctx} $in -o $out -d $out.d $includes $excludes $defines", description="CTX $in", depfile="$out.d", deps="gcc", @@ -545,10 +597,7 @@ def generate_build_ninja( sys.exit("ProjectConfig.sjiswrap_tag missing") wrapper = config.compiler_wrapper() - # Only add an implicit dependency on wibo if we download it - wrapper_implicit: Optional[Path] = None if wrapper is not None and config.use_wibo(): - wrapper_implicit = wrapper n.build( outputs=wrapper, rule="download_tool", @@ -558,6 +607,11 @@ def generate_build_ninja( "tag": config.wibo_tag, }, ) + + wrapper_implicit: Optional[Path] = None + if wrapper is not None and (wrapper.exists() or config.use_wibo()): + wrapper_implicit = wrapper + wrapper_cmd = f"{wrapper} " if wrapper else "" compilers = config.compilers() @@ -619,6 +673,22 @@ def generate_build_ninja( mwcc_sjis_cmd = f"{wrapper_cmd}{sjiswrap} {mwcc} $cflags -MMD -c $in -o $basedir" mwcc_sjis_implicit: List[Optional[Path]] = [*mwcc_implicit, sjiswrap] + # MWCC for precompiled headers + mwcc_pch_cmd = f"{wrapper_cmd}{mwcc} $cflags -MMD -c $in -o $basedir -precompile $basefilestem.mch" + mwcc_pch_implicit: List[Optional[Path]] = [*mwcc_implicit] + + # MWCC for precompiled headers with UTF-8 to Shift JIS wrapper + mwcc_pch_sjis_cmd = f"{wrapper_cmd}{sjiswrap} {mwcc} $cflags -MMD -c $in -o $basedir -precompile $basefilestem.mch" + mwcc_pch_sjis_implicit: List[Optional[Path]] = [*mwcc_implicit, sjiswrap] + + # MWCC with extab post-processing + mwcc_extab_cmd = ( + f'{CHAIN}{mwcc_cmd} && {dtk} extab clean --padding "$extab_padding" $out $out' + ) + mwcc_extab_implicit: List[Optional[Path]] = [*mwcc_implicit, dtk] + mwcc_sjis_extab_cmd = f'{CHAIN}{mwcc_sjis_cmd} && {dtk} extab clean --padding "$extab_padding" $out $out' + mwcc_sjis_extab_implicit: List[Optional[Path]] = [*mwcc_sjis_implicit, dtk] + # MWLD mwld = compiler_path / "mwldeppc.exe" mwld_cmd = f"{wrapper_cmd}{mwld} $ldflags -o $out @$out.rsp" @@ -627,17 +697,27 @@ def generate_build_ninja( # GNU as gnu_as = binutils / f"powerpc-eabi-as{EXE}" gnu_as_cmd = ( - f"{CHAIN}{gnu_as} $asflags -o $out $in -MD $out.d" - + f" && {dtk} elf fixup $out $out" + f"{CHAIN}{gnu_as} $asflags -o $out $in" + f" && {dtk} elf fixup $out $out" ) gnu_as_implicit = [binutils_implicit or gnu_as, dtk] + # As a workaround for https://github.com/encounter/dtk-template/issues/51 + # include macros.inc directly as an implicit dependency + gnu_as_implicit.append(build_path / "include" / "macros.inc") if os.name != "nt": transform_dep = config.tools_dir / "transform_dep.py" mwcc_cmd += f" && $python {transform_dep} $basefile.d $basefile.d" mwcc_sjis_cmd += f" && $python {transform_dep} $basefile.d $basefile.d" + mwcc_pch_cmd += f" && $python {transform_dep} $basefile.d $basefile.d" + mwcc_pch_sjis_cmd += f" && $python {transform_dep} $basefile.d $basefile.d" + mwcc_extab_cmd += f" && $python {transform_dep} $basefile.d $basefile.d" + mwcc_sjis_extab_cmd += f" && $python {transform_dep} $basefile.d $basefile.d" mwcc_implicit.append(transform_dep) mwcc_sjis_implicit.append(transform_dep) + mwcc_pch_implicit.append(transform_dep) + mwcc_pch_sjis_implicit.append(transform_dep) + mwcc_extab_implicit.append(transform_dep) + mwcc_sjis_extab_implicit.append(transform_dep) n.comment("Link ELF file") n.rule( @@ -677,12 +757,52 @@ def generate_build_ninja( ) n.newline() + n.comment("MWCC build (with extab post-processing)") + n.rule( + name="mwcc_extab", + command=mwcc_extab_cmd, + description="MWCC $out", + depfile="$basefile.d", + deps="gcc", + ) + n.newline() + + n.comment("MWCC build (with UTF-8 to Shift JIS wrapper and extab post-processing)") + n.rule( + name="mwcc_sjis_extab", + command=mwcc_sjis_extab_cmd, + description="MWCC $out", + depfile="$basefile.d", + deps="gcc", + ) + n.comment("Assemble asm") n.rule( name="as", command=gnu_as_cmd, description="AS $out", - depfile="$out.d", + # See https://github.com/encounter/dtk-template/issues/51 + # depfile="$out.d", + # deps="gcc", + ) + n.newline() + + n.comment("Build precompiled header") + n.rule( + name="mwcc_pch", + command=mwcc_pch_cmd, + description="PCH $out", + depfile="$basefile.d", + deps="gcc", + ) + n.newline() + + n.comment("Build precompiled header (with UTF-8 to Shift JIS wrapper)") + n.rule( + name="mwcc_pch_sjis", + command=mwcc_pch_sjis_cmd, + description="PCH $out", + depfile="$basefile.d", deps="gcc", ) n.newline() @@ -704,12 +824,16 @@ def generate_build_ninja( ) n.newline() - def write_custom_step(step: str, prev_step: Optional[str] = None) -> None: - implicit: List[str | Path] = [] + def write_custom_step( + step: str, + prev_step: Optional[str] = None, + extra_inputs: Optional[List[str]] = None, + ) -> None: + implicit: List[Union[str, Path]] = [] if config.custom_build_steps and step in config.custom_build_steps: n.comment(f"Custom build steps ({step})") for custom_step in config.custom_build_steps[step]: - outputs = cast(List[str | Path], custom_step.get("outputs")) + outputs = cast(List[Union[str, Path]], custom_step.get("outputs")) if isinstance(outputs, list): implicit.extend(outputs) @@ -728,33 +852,20 @@ def generate_build_ninja( dyndep=custom_step.get("dyndep", None), ) n.newline() + n.build( outputs=step, rule="phony", inputs=implicit, order_only=prev_step, + implicit=extra_inputs, ) - n.comment("Host build") - n.variable("host_cflags", "-I include -Wno-trigraphs") - n.variable( - "host_cppflags", - "-std=c++98 -I include -fno-exceptions -fno-rtti -D_CRT_SECURE_NO_WARNINGS -Wno-trigraphs -Wno-c++11-extensions", - ) - n.rule( - name="host_cc", - command="clang $host_cflags -c -o $out $in", - description="CC $out", - ) - n.rule( - name="host_cpp", - command="clang++ $host_cppflags -c -o $out $in", - description="CXX $out", - ) - n.newline() - # Add all build steps needed before we compile (e.g. processing assets) - write_custom_step("pre-compile") + pch_out_names = [ + get_pch_out_name(config, pch) for pch in config.precompiled_headers or [] + ] + write_custom_step("pre-compile", extra_inputs=pch_out_names) ### # Source files @@ -852,9 +963,41 @@ def generate_build_ninja( link_steps: List[LinkStep] = [] used_compiler_versions: Set[str] = set() source_inputs: List[Path] = [] - host_source_inputs: List[Path] = [] source_added: Set[Path] = set() + if config.precompiled_headers: + for pch in config.precompiled_headers: + src_path_rel_str = Path(pch["source"]) + src_path_rel = Path(src_path_rel_str) + pch_out_name = src_path_rel.with_suffix(".mch") + pch_out_abs_path = Path(get_pch_out_name(config, pch)) + # Add appropriate language flag if it doesn't exist already + cflags = pch["cflags"] + if not any(flag.startswith("-lang") for flag in cflags): + if file_is_cpp(src_path_rel): + cflags.insert(0, "-lang=c++") + else: + cflags.insert(0, "-lang=c") + + cflags_str = make_flags_str(cflags) + shift_jis = pch.get("shift_jis", config.shift_jis) + + n.comment(f"Precompiled header {pch_out_name}") + n.build( + outputs=pch_out_abs_path, + rule="mwcc_pch_sjis" if shift_jis else "mwcc_pch", + inputs=f"include/{src_path_rel_str}", + variables={ + "mw_version": Path(pch["mw_version"]), + "cflags": cflags_str, + "basedir": os.path.dirname(pch_out_abs_path), + "basefile": pch_out_abs_path.with_suffix(""), + "basefilestem": pch_out_abs_path.stem, + }, + implicit=mwcc_pch_sjis_implicit if shift_jis else mwcc_pch_implicit, + ) + n.newline() + def c_build(obj: Object, src_path: Path) -> Optional[Path]: # Avoid creating duplicate build rules if obj.src_obj_path is None or obj.src_obj_path in source_added: @@ -883,20 +1026,37 @@ def generate_build_ninja( # Add MWCC build rule lib_name = obj.options["lib"] + build_rule = "mwcc" + build_implcit = mwcc_implicit + variables = { + "mw_version": Path(obj.options["mw_version"]), + "cflags": cflags_str, + "basedir": os.path.dirname(obj.src_obj_path), + "basefile": obj.src_obj_path.with_suffix(""), + } + + if obj.options["shift_jis"] and obj.options["extab_padding"] is not None: + build_rule = "mwcc_sjis_extab" + build_implcit = mwcc_sjis_extab_implicit + variables["extab_padding"] = "".join( + f"{i:02x}" for i in obj.options["extab_padding"] + ) + elif obj.options["shift_jis"]: + build_rule = "mwcc_sjis" + build_implcit = mwcc_sjis_implicit + elif obj.options["extab_padding"] is not None: + build_rule = "mwcc_extab" + build_implcit = mwcc_extab_implicit + variables["extab_padding"] = "".join( + f"{i:02x}" for i in obj.options["extab_padding"] + ) n.comment(f"{obj.name}: {lib_name} (linked {obj.completed})") n.build( outputs=obj.src_obj_path, - rule="mwcc_sjis" if obj.options["shift_jis"] else "mwcc", + rule=build_rule, inputs=src_path, - variables={ - "mw_version": Path(obj.options["mw_version"]), - "cflags": cflags_str, - "basedir": os.path.dirname(obj.src_obj_path), - "basefile": obj.src_obj_path.with_suffix(""), - }, - implicit=( - mwcc_sjis_implicit if obj.options["shift_jis"] else mwcc_implicit - ), + variables=variables, + implicit=build_implcit, order_only="pre-compile", ) @@ -911,28 +1071,20 @@ def generate_build_ninja( ): include_dirs.append(flag[3:]) includes = " ".join([f"-I {d}" for d in include_dirs]) + excludes = " ".join([f"-x {d}" for d in config.context_exclude_globs]) + defines = " ".join([f"-D {d}" for d in config.context_defines]) + n.build( outputs=obj.ctx_path, rule="decompctx", inputs=src_path, implicit=decompctx, - variables={"includes": includes}, - ) - - # Add host build rule - if obj.options["host"] and obj.host_obj_path is not None: - n.build( - outputs=obj.host_obj_path, - rule="host_cc" if file_is_c(src_path) else "host_cpp", - inputs=src_path, variables={ - "basedir": os.path.dirname(obj.host_obj_path), - "basefile": obj.host_obj_path.with_suffix(""), + "includes": includes, + "excludes": excludes, + "defines": defines, }, - order_only="pre-compile", ) - if obj.options["add_to_all"]: - host_source_inputs.append(obj.host_obj_path) n.newline() if obj.options["add_to_all"]: @@ -986,8 +1138,9 @@ def generate_build_ninja( link_built_obj = obj.completed built_obj_path: Optional[Path] = None if obj.src_path is not None and obj.src_path.exists(): + check_path_case(obj.src_path) if file_is_c_cpp(obj.src_path): - # Add MWCC & host build rules + # Add C/C++ build rule built_obj_path = c_build(obj, obj.src_path) elif file_is_asm(obj.src_path): # Add assembler build rule @@ -1000,7 +1153,12 @@ def generate_build_ninja( link_built_obj = False # Assembly overrides - if obj.asm_path is not None and obj.asm_path.exists(): + if ( + not link_built_obj + and obj.asm_path is not None + and obj.asm_path.exists() + ): + check_path_case(obj.asm_path) link_built_obj = True built_obj_path = asm_build(obj, obj.asm_path, obj.asm_obj_path) @@ -1151,17 +1309,6 @@ def generate_build_ninja( ) n.newline() - ### - # Helper rule for building all source files, with a host compiler - ### - n.comment("Build all source files with a host compiler") - n.build( - outputs="all_source_host", - rule="phony", - inputs=host_source_inputs, - ) - n.newline() - ### # Check hash ### @@ -1192,7 +1339,7 @@ def generate_build_ninja( description="PROGRESS", ) n.build( - outputs=progress_path, + outputs="progress", rule="progress", implicit=[ ok_path, @@ -1209,16 +1356,91 @@ def generate_build_ninja( n.comment("Generate progress report") n.rule( name="report", - command=f"{objdiff} report generate -o $out", + command=f"{objdiff} report generate $objdiff_report_args -o $out", description="REPORT", ) n.build( outputs=report_path, rule="report", - implicit=[objdiff, "all_source"], + implicit=[objdiff, "objdiff.json", "all_source"], order_only="post-build", ) + n.comment("Phony edge that will always be considered dirty by ninja.") + n.comment( + "This can be used as an implicit to a target that should always be rerun, ignoring file modified times." + ) + n.build( + outputs="always", + rule="phony", + ) + n.newline() + + ### + # Regression test progress reports + ### + report_baseline_path = build_path / "baseline.json" + report_changes_path = build_path / "report_changes.json" + changes_fmt = config.tools_dir / "changes_fmt.py" + regressions_md = build_path / "regressions.md" + n.comment( + "Create a baseline progress report for later match regression testing" + ) + n.build( + outputs=report_baseline_path, + rule="report", + implicit=[objdiff, "all_source", "always"], + order_only="post-build", + ) + n.build( + outputs="baseline", + rule="phony", + inputs=report_baseline_path, + ) + n.comment("Check for any match regressions against the baseline") + n.comment("Will fail if no baseline has been created") + n.rule( + name="report_changes", + command=f"{objdiff} report changes --format json-pretty {report_baseline_path} $in -o $out", + description="CHANGES", + ) + n.build( + outputs=report_changes_path, + rule="report_changes", + inputs=report_path, + implicit=[objdiff, "always"], + ) + n.rule( + name="changes_fmt", + command=f"$python {changes_fmt} $args $in", + description="CHANGESFMT", + ) + n.build( + outputs="changes", + rule="changes_fmt", + inputs=report_changes_path, + implicit=changes_fmt, + ) + n.build( + outputs="changes_all", + rule="changes_fmt", + inputs=report_changes_path, + implicit=changes_fmt, + variables={"args": "--all"}, + ) + n.rule( + name="changes_md", + command=f"$python {changes_fmt} $in -o $out", + description="CHANGESFMT $out", + ) + n.build( + outputs=regressions_md, + rule="changes_md", + inputs=report_changes_path, + implicit=changes_fmt, + ) + n.newline() + ### # Helper tools ### @@ -1294,7 +1516,7 @@ def generate_build_ninja( description=f"RUN {configure_script}", ) n.build( - outputs="build.ninja", + outputs=["build.ninja", "objdiff.json"], rule="configure", implicit=[ build_config_path, @@ -1314,7 +1536,7 @@ def generate_build_ninja( if config.non_matching: n.default(link_outputs) elif config.progress: - n.default(progress_path) + n.default("progress") else: n.default(ok_path) else: @@ -1342,16 +1564,30 @@ def generate_objdiff_config( existing_config = json.load(r) existing_units = {unit["name"]: unit for unit in existing_config["units"]} + if config.ninja_path: + ninja = str(config.ninja_path.absolute()) + else: + ninja = "ninja" + objdiff_config: Dict[str, Any] = { "min_version": "2.0.0-beta.5", - "custom_make": "ninja", + "custom_make": ninja, "build_target": False, "watch_patterns": [ "*.c", + "*.cc", "*.cp", "*.cpp", + "*.cxx", + "*.c++", "*.h", + "*.hh", + "*.hp", "*.hpp", + "*.hxx", + "*.h++", + "*.pch", + "*.pch++", "*.inc", "*.py", "*.yml", @@ -1366,6 +1602,7 @@ def generate_objdiff_config( COMPILER_MAP = { "GC/1.0": "mwcc_233_144", "GC/1.1": "mwcc_233_159", + "GC/1.1p1": "mwcc_233_159p1", "GC/1.2.5": "mwcc_233_163", "GC/1.2.5e": "mwcc_233_163e", "GC/1.2.5n": "mwcc_233_163n", @@ -1373,6 +1610,7 @@ def generate_objdiff_config( "GC/1.3.2": "mwcc_242_81", "GC/1.3.2r": "mwcc_242_81r", "GC/2.0": "mwcc_247_92", + "GC/2.0p1": "mwcc_247_92p1", "GC/2.5": "mwcc_247_105", "GC/2.6": "mwcc_247_107", "GC/2.7": "mwcc_247_108", @@ -1471,7 +1709,7 @@ def generate_objdiff_config( "build_ctx": True, } ) - category_opt: List[str] | str = obj.options["progress_category"] + category_opt: Union[List[str], str] = obj.options["progress_category"] if isinstance(category_opt, list): progress_categories.extend(category_opt) elif category_opt is not None: @@ -1753,7 +1991,7 @@ def generate_compile_commands( json.dump(clangd_config, w, indent=2, default=default_format) -# Calculate, print and write progress to progress.json +# Print progress information from objdiff report def calculate_progress(config: ProjectConfig) -> None: config.validate() out_path = config.out_path() @@ -1845,35 +2083,3 @@ def calculate_progress(config: ProjectConfig) -> None: if summary_file: summary_file.write("```\n") summary_file.close() - - # Generate and write progress.json - progress_json: Dict[str, Any] = {} - - def add_category(id: str, measures: Dict[str, Any]) -> None: - progress_json[id] = { - "code": measures.get("complete_code", 0), - "code/total": measures.get("total_code", 0), - "data": measures.get("complete_data", 0), - "data/total": measures.get("total_data", 0), - "matched_code": measures.get("matched_code", 0), - "matched_code/total": measures.get("total_code", 0), - "matched_data": measures.get("matched_data", 0), - "matched_data/total": measures.get("total_data", 0), - "matched_functions": measures.get("matched_functions", 0), - "matched_functions/total": measures.get("total_functions", 0), - "fuzzy_match": int(measures.get("fuzzy_match_percent", 0) * 100), - "fuzzy_match/total": 10000, - "units": measures.get("complete_units", 0), - "units/total": measures.get("total_units", 0), - } - - if config.progress_all: - add_category("all", report_data["measures"]) - else: - # Support for old behavior where "dol" was the main category - add_category("dol", report_data["measures"]) - for category in report_data.get("categories", []): - add_category(category["id"], category["measures"]) - - with open(out_path / "progress.json", "w", encoding="utf-8") as w: - json.dump(progress_json, w, indent=2) diff --git a/tools/upload_progress.py b/tools/upload_progress.py deleted file mode 100755 index dc61d156..00000000 --- a/tools/upload_progress.py +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env python3 - -### -# Uploads progress information to https://github.com/decompals/frogress. -# -# Usage: -# python3 tools/upload_progress.py -b https://progress.decomp.club/ -p [project] -v [version] build/[version]/progress.json -# -# If changes are made, please submit a PR to -# https://github.com/encounter/dtk-template -### - -import argparse -import json -import os -import requests -import subprocess -import sys - - -def get_git_commit_timestamp() -> int: - return int( - subprocess.check_output(["git", "show", "-s", "--format=%ct"]) - .decode("ascii") - .rstrip() - ) - - -def get_git_commit_sha() -> str: - return subprocess.check_output(["git", "rev-parse", "HEAD"]).decode("ascii").strip() - - -def generate_url(args: argparse.Namespace) -> str: - url_components = [args.base_url.rstrip("/"), "data"] - - for arg in [args.project, args.version]: - if arg != "": - url_components.append(arg) - - return str.join("/", url_components) + "/" - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Upload progress information.") - parser.add_argument("-b", "--base_url", help="API base URL", required=True) - parser.add_argument("-a", "--api_key", help="API key (env var PROGRESS_API_KEY)") - parser.add_argument("-p", "--project", help="Project slug", required=True) - parser.add_argument("-v", "--version", help="Version slug", required=True) - parser.add_argument("input", help="Progress JSON input") - - args = parser.parse_args() - api_key = args.api_key or os.environ.get("PROGRESS_API_KEY") - if not api_key: - raise KeyError("API key required") - url = generate_url(args) - - entries = [] - with open(args.input, "r") as f: - data = json.load(f) - entries.append( - { - "timestamp": get_git_commit_timestamp(), - "git_hash": get_git_commit_sha(), - "categories": data, - } - ) - - print("Publishing entry to", url) - json.dump(entries[0], sys.stdout, indent=4) - print() - r = requests.post( - url, - json={ - "api_key": api_key, - "entries": entries, - }, - ) - r.raise_for_status() - print("Done!")