New assets system (#2481)

* wip: New assets system tm

Builds gc-eu-mq-dbg OK from clean after
1) make setup
2) python3 -m tools.assets.extract -j
3) replace 0x80A8E610 with sShadowTex in extracted/gc-eu-mq-dbg/assets/overlays/ovl_En_Jsjutan/sShadowMaterialDL.inc.c
4) make various symbols in extracted data like sTex static

* use variables from config.yml for gMtxClear and sShadowTex addresses

* Write source with static for overlays using `HACK_IS_STATIC_ON` hack

* gc-eu-mq-dbg OK from clean with `make setup && make`

* implement more skeleton-related types, cleanups, fixups

* fix extracted data to no longer produce compilation warnings

* implement more of RoomShapeImage types

* yeet XmlPath from ExternalFile usage

* Implement PlayerAnimationDataResource (link_animetion data)

* fix csdis CS_TIME extra arg

* dmadata file names no longer hardcoded for gc-eu-mq-dbg

* ntsc-1.0 OK

* xml fixes

* slightly improve standard output

* rm extract_assets.py

* generate and use Limb enums (TODO: check Skin skels and implement for Curve skels)

* handle dependencies between xmls

* introduce RawPointers xml attribute to ignore specific pointers and keep them raw

* add tools/extract_assets.sh

* fixups

* only extract if xmls changed or if -f (force) is used

* fixups, gc-eu OK

* all versions OK

* check attributes of xml resources elements

* Implement legacy skelanime resources

* fix ASSET_FILES_BIN_EXTRACTED/COMMITTED: look for .u8.bin specifically instead of just .bin

* implement JFIFResource

* fix png/jpg wildcards: look specifically for .u64.png .u32.png .u64.jpg

* Makefile: Add rules to build .png, .bin and .jpg in assets/ too

* start writing actual docs

* extract sTransCircleDL and sTransWipeDL

* misc cleanup/fixes, pygfxd 1.0.3

* refactor CDataExt.set_write callback args to use a dataclass

* Move {} to in-source

* misc

* more progress on spec

* fix missing braces in n64dd_error_textures.c

* finish xml spec doc

* assets xmls fixes

* some cleanup, use `gNameTex_WIDTH/HEIGHT` macros in dlists

* handle hackmode_syotes_room, fix compile

* C build_from_png

* rm tools/assets/bin2c

* rm ZAPD

* format

* remove rule to generate dmadata_table.py

* CC0 license (and some import cleanup)

* dont try to build zapd (rmd)

* simplify palettes with single user (ci images with a non-shared palette)

* add docs on how images are handled

* bss

* allow -j N

* fix n64texconv python bindings memory management

* move -j at the end of calling extraction script

* with -j, update last_extracts.json as each job completes rather than only if all complete

* make interrupting less jank by making child processes ignore sigint

* use enum names in `SCENE_CMD_SKYBOX_SETTINGS`

* `multiprocessing.get_context("fork")`

* import rich, except ImportError s

* fix optional rich usage

* .bss

* .bss

* .bss

* assets extraction: -j -> -j$(N_THREADS)

* .bss

* change LIMB_NONE/MAX defaults to be FILE_OFFSET instead of SKELNAME

* 0XHEX -> 0xHEX

* fix bss

* Proper includes for assets

mostly proper, some includes like dlists resources always causing a sys_matrix.h include (when not every dlist references gIdentityMtx) could be done better

* rm z64.h

* rm z64.h take two

* bss

* Make .u64 suffix for pngs optional

* fixup: rm .u64 suffix from n64dd image paths

* Remove elemtype suffixes from .bin and .jpg files

* Update images.md

* some build_from_png cleanup, more error handling, comments

* Handle skybox textures

Introduce "sub-format" suffix for pngs, with sub-formats split_lo and split_hi being used for skybox textures

* fixup for older python

* improve collision output some

* fully use SURFACETYPE[01] macros in writing extracted surface types

* use WATERBOX_PROPERTIES in extracted waterboxes

* some SceneCommandsResource cleanup

* format EnvLightSettingsList output

Makefile

@@ -791,9 +791,9 @@ setup: venv
 	$(MAKE) -C tools
 	$(PYTHON) tools/decompress_baserom.py $(VERSION)
 	$(PYTHON) tools/extract_baserom.py $(BASEROM_DIR)/baserom-decompressed.z64 $(EXTRACTED_DIR)/baserom -v $(VERSION)
+	$(PYTHON) -m tools.assets.extract $(EXTRACTED_DIR)/baserom $(EXTRACTED_DIR) -v $(VERSION) -j$(N_THREADS)
 	$(PYTHON) tools/extract_incbins.py $(EXTRACTED_DIR)/baserom $(EXTRACTED_DIR)/incbin -v $(VERSION)
 	$(PYTHON) tools/extract_text.py $(EXTRACTED_DIR)/baserom $(EXTRACTED_DIR)/text -v $(VERSION)
-	$(PYTHON) tools/extract_assets.py $(EXTRACTED_DIR)/baserom $(EXTRACTED_DIR)/assets -v $(VERSION) -j$(N_THREADS)
 	$(PYTHON) tools/extract_audio.py -b $(EXTRACTED_DIR)/baserom -o $(EXTRACTED_DIR) -v $(VERSION) --read-xml
 
 disasm:
@@ -979,21 +979,25 @@ $(BUILD_DIR)/src/overlays/%_reloc.o: $(BUILD_DIR)/spec
 	$(POSTPROCESS_OBJ) $(@:.o=.s)
 	$(AS) $(ASFLAGS) $(@:.o=.s) -o $@
 
-$(BUILD_DIR)/assets/%.inc.c: assets/%.png
-	$(N64TEXCONV) $(subst .,,$(suffix $*)) "$(findstring u32,$(subst .,,$(suffix $(basename $*))))" $< $@ $(@:.inc.c=.pal.inc.c)
+# Assets from assets/
 
-$(BUILD_DIR)/assets/%.inc.c: $(EXTRACTED_DIR)/assets/%.png
-	$(N64TEXCONV) $(subst .,,$(suffix $*)) "$(findstring u32,$(subst .,,$(suffix $(basename $*))))" $< $@ $(@:.inc.c=.pal.inc.c)
+$(BUILD_DIR)/assets/%.inc.c: assets/%.png
+	tools/assets/build_from_png/build_from_png $< $(dir $@) assets/$(dir $*) $(wildcard $(EXTRACTED_DIR)/assets/$(dir $*))
 
 $(BUILD_DIR)/assets/%.bin.inc.c: assets/%.bin
 	$(BIN2C) -t 1 $< $@
 
-$(BUILD_DIR)/assets/%.bin.inc.c: $(EXTRACTED_DIR)/assets/%.bin
-	$(BIN2C) -t 1 $< $@
-
 $(BUILD_DIR)/assets/%.jpg.inc.c: assets/%.jpg
 	$(N64TEXCONV) JFIF "" $< $@
 
+# Assets from extracted/
+
+$(BUILD_DIR)/assets/%.inc.c: $(EXTRACTED_DIR)/assets/%.png
+	tools/assets/build_from_png/build_from_png $< $(dir $@) $(wildcard assets/$(dir $*)) $(EXTRACTED_DIR)/assets/$(dir $*)
+
+$(BUILD_DIR)/assets/%.bin.inc.c: $(EXTRACTED_DIR)/assets/%.bin
+	$(BIN2C) -t 1 $< $@
+
 $(BUILD_DIR)/assets/%.jpg.inc.c: $(EXTRACTED_DIR)/assets/%.jpg
 	$(N64TEXCONV) JFIF "" $< $@
 

assets/xml/code/fbdemo_circle.xml

@@ -7,7 +7,6 @@
         <Array Name="sTransCircleVtx" Count="34" Offset="0x1000">
             <Vtx/>
         </Array>
-        <!-- ZAPD isn't finding symbols correctly, instead finding them much later in `code`-->
-        <!--<DList Name="sTransCircleDL" Offset="0x1220"/>-->
+        <DList Name="sTransCircleDL" Offset="0x1220"/>
     </File>
 </Root>

assets/xml/code/fbdemo_wipe1.xml

@@ -4,7 +4,6 @@
             <Vtx/>
         </Array>
         <Texture Name="sTransWipeTex" OutName="trans_wipe" Format="i4" Width="64" Height="64" Offset="0x190"/>
-        <!-- ZAPD isn't finding symbols correctly, instead finding them much later in `code`-->
-        <!-- <DList Name="sTransWipeDL" Offset="0x990"/> -->
+        <DList Name="sTransWipeDL" Offset="0x990"/>
     </File>
 </Root>

assets/xml/objects/gameplay_keep.xml

@@ -1,5 +1,5 @@
 <Root>
-    <ExternalFile XmlPath="misc/link_animetion.xml" OutPath="assets/misc/link_animetion/"/>
+    <ExternalFile OutPath="assets/misc/link_animetion/"/>
     <File Name="gameplay_keep" Segment="4">
         <Texture Name="gHilite1Tex" OutName="hilite_1" Format="rgba16" Width="16" Height="16" Offset="0x0"/>
         <Texture Name="gHilite2Tex" OutName="hilite_2" Format="rgba16" Width="16" Height="16" Offset="0x200"/>

assets/xml/objects/gameplay_keep_pal.xml

@@ -1,5 +1,5 @@
 <Root>
-    <ExternalFile XmlPath="misc/link_animetion.xml" OutPath="assets/misc/link_animetion/"/>
+    <ExternalFile OutPath="assets/misc/link_animetion/"/>
     <File Name="gameplay_keep" Segment="4">
         <Texture Name="gHilite1Tex" OutName="hilite_1" Format="rgba16" Width="16" Height="16" Offset="0x0"/>
         <Texture Name="gHilite2Tex" OutName="hilite_2" Format="rgba16" Width="16" Height="16" Offset="0x200"/>

assets/xml/objects/object_bdoor.xml

@@ -1,5 +1,5 @@
 <Root>
-    <ExternalFile XmlPath="objects/gameplay_dangeon_keep.xml" OutPath="assets/objects/gameplay_dangeon_keep/"/>
+    <ExternalFile OutPath="assets/objects/gameplay_dangeon_keep/"/>
     <File Name="object_bdoor" Segment="6">
         <!-- One of the Boss Door Textures -->
         <Texture Name="gBossDoorGanonsCastleTex" OutName="boss_door_ganons_castle" Format="rgba16" Width="32" Height="64" Offset="0x0"/>

assets/xml/objects/object_demo_6k.xml

@@ -1,4 +1,5 @@
 <Root>
+    <ExternalFile OutPath="assets/objects/gameplay_keep"/>
     <File Name="object_demo_6k" Segment="6">
         <Texture Name="object_demo_6k_Tex_000000" OutName="tex_00000000" Format="i8" Width="64" Height="64" Offset="0x0"/>
         <DList Name="object_demo_6k_DL_001040" Offset="0x1040"/>

assets/xml/objects/object_demo_kekkai.xml

@@ -1,4 +1,5 @@
 <Root>
+    <ExternalFile OutPath="assets/objects/gameplay_keep"/>
     <File Name="object_demo_kekkai" Segment="6">
         <!-- Demo_Kekkai -->
         <DList Name="gTowerBarrierDL" Offset="0x4930"/>

assets/xml/objects/object_efc_erupc.xml

@@ -1,4 +1,5 @@
 <Root>
+    <ExternalFile OutPath="assets/objects/gameplay_keep"/>
     <File Name="object_efc_erupc" Segment="6">
         <Texture Name="object_efc_erupc_Tex_000000" OutName="tex_00000000" Format="i8" Width="32" Height="64" Offset="0x0"/>
         <Texture Name="object_efc_erupc_Tex_000800" OutName="tex_00000800" Format="i4" Width="64" Height="64" Offset="0x800"/>

assets/xml/objects/object_fish.xml

@@ -1,4 +1,5 @@
 <Root>
+    <ExternalFile OutPath="assets/objects/gameplay_keep"/>
     <File Name="object_fish" Segment="6">
         <Animation Name="gFishingFishAnim" Offset="0x7C"/>
         <DList Name="gFishingFishDL_000940" Offset="0x940"/>

assets/xml/objects/object_fz.xml

@@ -1,4 +1,5 @@
 <Root>
+    <ExternalFile OutPath="assets/objects/gameplay_keep"/>
     <File Name="object_fz" Segment="6">
         <DList Name="gFreezardIntactDL" Offset="0x1130"/>
         <DList Name="gFreezardTopRightHornChippedDL" Offset="0x21A0"/>

assets/xml/objects/object_gi_arrow.xml

@@ -1,4 +1,5 @@
 <Root>
+    <ExternalFile OutPath="assets/objects/gameplay_keep"/>
     <File Name="object_gi_arrow" Segment="6">
         <DList Name="gGiArrowSmallDL" Offset="0x0340"/>
         <DList Name="gGiArrowMediumDL" Offset="0x0B90"/>

assets/xml/objects/object_gi_arrowcase.xml

@@ -1,4 +1,5 @@
 <Root>
+    <ExternalFile OutPath="assets/objects/gameplay_keep"/>
     <File Name="object_gi_arrowcase" Segment="6">
         <DList Name="gGiQuiver30InnerColorDL" Offset="0x08D0"/>
         <DList Name="gGiQuiver40InnerColorDL" Offset="0x08F0"/>

assets/xml/objects/object_gi_bean.xml

@@ -1,4 +1,5 @@
 <Root>
+    <ExternalFile OutPath="assets/objects/gameplay_keep"/>
     <File Name="object_gi_bean" Segment="6">
         <DList Name="gGiBeanDL" Offset="0x0580"/>
     </File>

assets/xml/objects/object_gi_bomb_1.xml

@@ -1,4 +1,5 @@
 <Root>
+    <ExternalFile OutPath="assets/objects/gameplay_keep"/>
     <File Name="object_gi_bomb_1" Segment="6">
         <DList Name="gGiBombDL" Offset="0x09A0"/>
     </File>

assets/xml/objects/object_gi_bomb_2.xml

@@ -1,4 +1,5 @@
 <Root>
+    <ExternalFile OutPath="assets/objects/gameplay_keep"/>
     <File Name="object_gi_bomb_2" Segment="6">
         <DList Name="gGiBombchuDL" Offset="0x04B0"/>
     </File>

assets/xml/objects/object_gi_bombpouch.xml

@@ -1,4 +1,5 @@
 <Root>
+    <ExternalFile OutPath="assets/objects/gameplay_keep"/>
     <File Name="object_gi_bombpouch" Segment="6">
         <DList Name="gGiBombBag20BagColorDL" Offset="0x0AD0"/>
         <DList Name="gGiBombBag30BagColorDL" Offset="0x0AF0"/>

assets/xml/objects/object_gi_boomerang.xml

@@ -1,4 +1,5 @@
 <Root>
+    <ExternalFile OutPath="assets/objects/gameplay_keep"/>
     <File Name="object_gi_boomerang" Segment="6">
         <DList Name="gGiBoomerangDL" Offset="0x0A30"/>
     </File>

assets/xml/objects/object_gi_boots_2.xml

@@ -1,4 +1,5 @@
 <Root>
+    <ExternalFile OutPath="assets/objects/gameplay_keep"/>
     <File Name="object_gi_boots_2" Segment="6">
         <DList Name="gGiIronBootsDL" Offset="0x1630"/>
         <DList Name="gGiIronBootsRivetsDL" Offset="0x1A98"/>

assets/xml/objects/object_gi_bosskey.xml

@@ -1,4 +1,5 @@
 <Root>
+    <ExternalFile OutPath="assets/objects/gameplay_keep"/>
     <File Name="object_gi_bosskey" Segment="6">
         <DList Name="gGiBossKeyDL" Offset="0x0CA0"/>
         <DList Name="gGiBossKeyGemDL" Offset="0x0F08"/>