menu stuff

big ugly commit, cuz idc

pythonlib/formats/pak.py

@@ -0,0 +1,185 @@
+from dataclasses import dataclass
+import struct
+from typing import Optional
+from ..formats.FileIO import FileIO
+from ..utils import comptolib
+
+
+@dataclass
+class pak_file():
+    is_compressed: bool
+    type: int
+    data: bytes
+
+
+class Pak():
+    def __init__(self) -> None:
+        self.type = -1
+        self.align = False
+        self.files = []
+
+    
+    @staticmethod
+    def from_path(path, type) -> 'Pak':
+        with FileIO(path) as f:
+            self = Pak()
+            # if type == -1:
+            #     type = Pak.get_pak_type(f)
+
+            file_amount = f.read_uint32()
+            self.files = []
+            blobs: list[bytes] = []
+            offsets: list[int] = []
+            sizes: list[int] = []  
+
+            # Pak0
+            if type == 0:
+                for _ in range(file_amount):
+                    sizes.append(f.read_uint32())
+                
+                for size in sizes:
+                    blobs.append(f.read(size))
+
+            # Pak1
+            elif type == 1:
+                for _ in range(file_amount):
+                    offsets.append(f.read_uint32())
+                    sizes.append(f.read_uint32())
+
+                for offset, size in zip(offsets, sizes):
+                    f.seek(offset)
+                    blobs.append(f.read(size))
+            # Pak3
+            elif type == 3:
+                for _ in range(file_amount):
+                    offsets.append(f.read_uint32())
+                f.seek(0, 2)
+                offsets.append(f.tell())
+                for i, j in zip(offsets[::1], offsets[1::1]):
+                    sizes.append(j - i)
+                for offset, size in zip(offsets, sizes):
+                    f.seek(offset)
+                    blobs.append(f.read(size))
+
+        for blob in blobs:
+            is_compressed = comptolib.is_compressed(blob)
+            c_type = 0
+            if is_compressed:
+                c_type = blob[0]
+                blob = comptolib.decompress_data(blob)
+
+            self.files.append(pak_file(is_compressed, c_type, blob))
+
+        return self
+    
+    @staticmethod
+    def get_pak_type(data: bytes) -> Optional[int]:
+        is_aligned = False
+        
+        data_size = len(data)
+        if data_size < 0x8: return None
+    
+        files = struct.unpack("<I", data[:4])[0]
+        first_entry = struct.unpack("<I", data[4:8])[0]
+    
+        # Expectations
+        pak1_header_size = 4 + (files * 8)
+        pak3_header_size = 4 + (files * 4)
+    
+        # Check for alignment
+        if first_entry % 0x10 == 0:
+            is_aligned = True
+        
+        if pak1_header_size % 0x10 != 0:
+            pak1_check = pak1_header_size + (0x10 - (pak1_header_size % 0x10))
+        else:
+            pak1_check = pak1_header_size
+    
+        # First test pak0 
+        # (pak0 can't be aligned, so header size 
+        # would be the same as unaligned pak3)
+        if len(data) > pak3_header_size:
+            calculated_size = 0
+            for size in struct.unpack(f"<{files}I", data[4 : (files + 1) * 4]):
+                calculated_size += size
+            if calculated_size == len(data) - pak3_header_size:
+                return 0
+    
+        # Test for pak1
+        if is_aligned:
+            if pak1_check == first_entry:
+                return 1
+        elif pak1_header_size == first_entry:
+                return 1
+        
+        #Test for pak3
+        previous = 0
+        for offset in struct.unpack(f"<{files}I", data[4: (files+1)*4]):
+
+            if offset > previous and offset >= pak3_header_size:
+                previous = offset
+            else:
+                return None
+        
+        last_offset = (4*(files+1))+8
+        if data[last_offset : first_entry] == b'\x00' * (first_entry - last_offset):
+            return 3
+        return None
+    
+
+    def to_bytes(self, type=-1) -> bytes:
+        
+        compose_mode = type if type != -1 else self.type
+        if compose_mode == -1:
+            raise ValueError("Trying to compose an invalid PAK type")
+        
+        # Collect blobs
+        blobs = []
+        for blob in self.files:
+            if blob.is_compressed:
+                blobs.append(comptolib.compress_data(blob.data, version=blob.type))
+            else:
+                blobs.append(blob.data)
+
+        # Compose
+        out = struct.pack("<I", len(self.files))
+        
+        # Pak0
+        if compose_mode == 0:
+            for blob in blobs:
+                out += struct.pack("<I", len(blob))
+        # Pak1
+        elif compose_mode == 1:
+            offset = 4 + (8 * len(blobs))
+            sizes = [0] + list([len(x) for x in blobs])
+            if self.align:
+                offset = offset + (0x10 - (offset % 0x10))
+
+            for i, j in zip(sizes[::1], sizes[1::1]):
+                if self.align:
+                    i = i + (0x10 - (i % 0x10))
+                struct.pack("<I", offset + i)
+                struct.pack("<I", j)
+                offset += i
+        # Pak3
+        elif compose_mode == 3:
+            offset = 4 + (4 * len(blobs))
+            for _ in range(len(blobs)):
+                if self.align:
+                    offset = offset + (0x10 - (offset % 0x10))
+                out += struct.pack("<I", offset)
+
+        # add files
+        for blob in blobs:
+            if self.align:
+                out += b"\x00" * (0x10 - (len(out) % 0x10))
+            out += blob
+
+        return out
+    
+
+    def __getitem__(self, item):
+        return self.files[item]
+
+    def __len__(self):
+        return len(self.files)

pythonlib/formats/theirsce.py

@@ -65,11 +65,12 @@ class Theirsce(FileIO):
             yield opcode
             self.seek(pos)
 
-    def read_tag_bytes(self):
+    @staticmethod
+    def read_tag_bytes(src) -> bytes:
         data = b""
         
         while True:
-            c = self.read(1)
+            c = src.read(1)
             if c == b"\x80":
                 data += c
                 break
@@ -79,9 +80,9 @@ class Theirsce(FileIO):
             
             if opcode < 0x80: 
                 if opcode & 8 != 0: 
-                    data += self.read(2)
+                    data += src.read(2)
                 else:
-                    data += self.read(1)
+                    data += src.read(1)
                     
             elif opcode < 0xC0:
                 continue
@@ -89,30 +90,30 @@ class Theirsce(FileIO):
             elif opcode < 0xE0:
                 size_mask = (opcode >> 3) & 3
 
-                if size_mask == 1: data += self.read(1)
-                elif size_mask == 2: data += self.read(2)
-                elif size_mask == 3: data += self.read(4)
+                if size_mask == 1: data += src.read(1)
+                elif size_mask == 2: data += src.read(2)
+                elif size_mask == 3: data += src.read(4)
 
             elif opcode < 0xF0:
-                data += self.read(1)
+                data += src.read(1)
             
             elif opcode < 0xF8:
                 if (0xF2 <= opcode < 0xF5) or opcode == 0xF7:
-                    data += self.read(2)
+                    data += src.read(2)
                 elif opcode == 0xF5:
-                    data += self.read(4)
+                    data += src.read(4)
                 elif opcode == 0xF6:
-                    data += self.read(1)
+                    data += src.read(1)
                     for _ in range(data[-1]):
-                        cc = self.read_uint8()
+                        cc = src.read_uint8()
                         data += cc
                         if cc & 8 != 0:
-                            data += self.read(2) 
+                            data += src.read(2) 
                         else:
-                            data += self.read(3)
+                            data += src.read(3)
             
             elif opcode < 0xFC:
-                data += self.read(2)
+                data += src.read(2)
             
             elif opcode == 0xFE:
                 continue

pythonlib/games/ToolsNDX.py

@@ -648,7 +648,7 @@ class ToolsNDX(ToolsTales):
         self.unpack_Folder( menu_file_path)
         
                 
-    def extract_all_Menu(self):
+    def extract_all_menu(self):
         
         res = [self.prepare_Menu_File(ele) for ele in list(set([ele['Hashes_Name'] for ele in self.menu_files_json if ele['Hashes_Name'] != '']))]
         
@@ -688,7 +688,7 @@ class ToolsNDX(ToolsTales):
                 if isinstance(pointers_offset, list):
                     pointers_offset, pointers_value = self.get_Direct_Pointers(text_start, text_end, base_offset, pointers_offset, section,file_path)
                 else:
-                    pointers_offset, pointers_value = self.get_Style_Pointers( text_start, text_end, base_offset, section['Pointer_Offset_Start'], section['Style'], file_path)
+                    pointers_offset, pointers_value = self.get_style_pointers( text_start, text_end, base_offset, section['Pointer_Offset_Start'], section['Style'], file_path)
           
               
                 #Extract Text from the pointers

pythonlib/games/ToolsTOR.py

@@ -11,6 +11,8 @@ from pathlib import Path
 import lxml.etree as etree
 import pandas as pd
 from tqdm import tqdm
+from pythonlib.formats.FileIO import FileIO
+from pythonlib.formats.pak import Pak
 from pythonlib.formats.scpk import Scpk
 
 import pythonlib.utils.comptolib as comptolib
@@ -52,6 +54,7 @@ class ToolsTOR(ToolsTales):
     story_XML_new       = '../Tales-Of-Rebirth/Data/TOR/Story/'                        #Story XML files will be extracted here                      
     story_XML_patch     = '../Data/Tales-Of-Rebirth/Story/'               #Story XML files will be extracted here
     skit_XML_patch      = '../Data/Tales-Of-Rebirth/Skits/'                        #Skits XML files will be extracted here
+    menu_XML_patch      = '../Tales-Of-Rebirth/Data/TOR/Menu/'
     skit_XML_new        = '../Tales-Of-Rebirth/Data/TOR/Skits/'
     dat_archive_extract = '../Data/Tales-Of-Rebirth/DAT/' 
     # fmt: on
@@ -421,22 +424,22 @@ class ToolsTOR(ToolsTales):
         return pointers_offset, texts_offset
 
     #Convert a bytes object to text using TAGS and TBL in the json file
-    def bytes_to_text(self, theirsce: Theirsce, offset=-1, end_strings = b"\x00"):
+    def bytes_to_text(self, src: FileIO, offset=-1, end_strings = b"\x00"):
         finalText = ""
         tags = self.jsonTblTags['TAGS']
         chars = self.jsonTblTags['TBL']
 
         if (offset > 0):
-            theirsce.seek(offset, 0)
+            src.seek(offset, 0)
 
         while True:
-            b = theirsce.read(1)
+            b = src.read(1)
             if b == end_strings: break
 
             b = ord(b)
             # Custom Encoded Text
             if (0x99 <= b <= 0x9F) or (0xE0 <= b <= 0xEB):
-                c = (b << 8) | theirsce.read_uint8()
+                c = (b << 8) | src.read_uint8()
                 finalText += chars.get(c, "{%02X}{%02X}" % (c >> 8, c & 0xFF))
                 continue
             
@@ -455,7 +458,7 @@ class ToolsTOR(ToolsTales):
                 continue
 
             if b == 0x81:
-                next_b = theirsce.read(1)
+                next_b = src.read(1)
                 if next_b == b"\x40":
                     finalText += "　"
                 else:
@@ -465,13 +468,13 @@ class ToolsTOR(ToolsTales):
             
             # Simple Tags
             if 0x3 <= b <= 0xF:
-                parameter = theirsce.read_uint32()
+                parameter = src.read_uint32()
 
                 tag_name = tags.get(b, f"{b:02X}")
                 tag_param = self.jsonTblTags.get(tag_name.upper(), {}).get(parameter, None)  
 
                 if tag_param is not None:
-                    finalText += tag_param
+                    finalText += f"<{tag_param}>"
                 else:
                     finalText += f"<{tag_name}:{self.hex2(parameter)}>"
 
@@ -480,7 +483,7 @@ class ToolsTOR(ToolsTales):
             # Variable tags (same as above but using rsce bytecode as parameter)
             if 0x13 <= b <= 0x1A:
                 tag_name = f"unk{b:02X}"
-                parameter = "".join([f"{c:02X}" for c in theirsce.read_tag_bytes()])
+                parameter = "".join([f"{c:02X}" for c in Theirsce.read_tag_bytes(src)])
          
                 finalText += f"<{tag_name}:{parameter}>"
                 continue
@@ -754,6 +757,98 @@ class ToolsTOR(ToolsTales):
         
                 with open(final_path / fname, "wb") as output:
                     output.write(data)
+
+
+    def get_style_pointers(self, text_start, text_max, base_offset, start_offset, style, file: FileIO):
+        file.seek(0, 2)
+        f_size = file.tell()
+    
+        file.seek(start_offset)
+        pointers_offset = []
+        pointers_value  = []
+        split = [ele for ele in re.split(r'(P)|(\d+)', style) if ele]
+        ok = True
+        
+        while ok:
+            for step in split:
+                if step == "P":
+                    text_offset = struct.unpack("<I", file.read(4))[0] + base_offset
+
+                    if text_offset < f_size and text_offset >= text_start and text_offset < text_max:
+                        pointers_value.append(text_offset)
+                        pointers_offset.append(file.tell()-4)
+                        
+                    else:
+                        ok = False
+                else:
+                    file.read(int(step))
+        
+        return pointers_offset, pointers_value
+    
+
+    def extract_all_menu(self) -> None:
+        print("Extracting Menu Files...")
+        
+        #Prepare the menu files (Unpack PAK files and use comptoe)
+        xml_path = Path(self.menu_XML_patch) / "XML"
+        xml_path.mkdir(exist_ok=True)
+
+        for entry in tqdm(self.menu_files_json):
+            file_path = Path(entry["file_path"])
+            if entry["is_pak"]:
+                pak = Pak.from_path(file_path, int(entry["pak_type"]))
+
+                for p_file in entry["files"]:
+                    f_index = int(p_file["file"])
+                    with FileIO(pak[f_index].data, "rb") as f:
+                        xml_data = self.extract_menu_file(p_file, f)
+
+                    with open(xml_path / f"{file_path.stem}_{f_index:03d}.xml", "wb") as xmlFile:
+                        xmlFile.write(xml_data)
+
+            else:
+                with FileIO(entry["file_path"], "rb") as f:
+                    xml_data = self.extract_menu_file(entry, f)
+
+                with open(xml_path / f"{file_path.stem}.xml", "wb") as xmlFile:
+                    xmlFile.write(xml_data)
+            
+
+    def extract_menu_file(self, file_def, f: FileIO):
+        section_list = []
+        pointers_offset_list = []
+        texts_list = []
+
+        base_offset = int(file_def["base_offset"])
+        # print("BaseOffset:{}".format(base_offset))
+
+        for section in file_def['sections']:
+            
+            text_start = int(section['text_start'])
+            text_end = int(section['text_end'])
+            
+            #Extract Pointers of the file
+            # print("Extract Pointers")
+            pointers_offset, pointers_value = self.get_style_pointers(text_start, text_end, base_offset, section['pointers_start'], section['style'], f)
+            # print([hex(pv) for pv in pointers_value])
+        
+            #Extract Text from the pointers
+            # print("Extract Text")
+            texts = [ self.bytes_to_text(f, ele) for ele in pointers_value]
+            
+            #Make a list
+            section_list.extend( [section['section']] * len(texts)) 
+            pointers_offset_list.extend( pointers_offset)
+            texts_list.extend( texts )
+    
+        #Remove duplicates
+        list_informations = self.remove_duplicates(section_list, pointers_offset_list, texts_list)
+
+        #Build the XML Structure with the information
+        root = self.create_Node_XML("", list_informations, "Menu", "MenuText")
+        
+        #Write to XML file
+        return etree.tostring(root, encoding="UTF-8", pretty_print=True)
         
         
     def pack_main_archive(self):

pythonlib/games/ToolsTales.py

@@ -16,8 +16,10 @@ import pycdlib
 import pygsheets
 from googleapiclient.errors import HttpError
 from tqdm import tqdm
+from pythonlib.formats.FileIO import FileIO
 
 import pythonlib.formats.fps4 as fps4
+from pythonlib.formats.pak import Pak
 
 
 class ToolsTales:
@@ -932,7 +934,7 @@ class ToolsTales:
 
         return [pointers_offset, pointers_value]
     
-    def get_Style_Pointers(self, text_start, text_max, base_offset, start_offset, style, file_path):
+    def get_style_pointers(self, text_start, text_max, base_offset, start_offset, style, file_path):
         
         f_size = os.path.getsize(file_path)
         with open(file_path , "rb") as f:
@@ -940,7 +942,7 @@ class ToolsTales:
             f.seek(start_offset, 0)
             pointers_offset = []
             pointers_value  = []
-            split = [ele for ele in re.split(r'(P)|(\d+)', style) if ele != None and ele != '']
+            split = [ele for ele in re.split(r'(P)|(\d+)', style) if ele]
             ok = True
             
             while ok:
@@ -999,7 +1001,7 @@ class ToolsTales:
                   
                 #Extract Pointers of the file
                 print("Extract Pointers")
-                pointers_offset, pointers_value = self.get_Style_Pointers( text_start, text_end, base_offset, section['Pointer_Offset_Start'], section['Style'], file_path)
+                pointers_offset, pointers_value = self.get_style_pointers( text_start, text_end, base_offset, section['Pointer_Offset_Start'], section['Style'], file_path)
                 print([hex(pointers_value) for ele in pointers_value])
               
                 #Extract Text from the pointers

pythonlib/utils/comptolib.py

@@ -156,13 +156,19 @@ def is_compressed(data: bytes) -> bool:
     if len(data) < 0x09:
         return False
 
+    if data[0] not in (0, 1, 3):
+        return False
+
     expected_size = struct.unpack("<L", data[1:5])[0]
+    aligned_size = (expected_size + 9) + (0x100 - ((expected_size + 9) % 0x100))
     tail_data = abs(len(data) - (expected_size + 9))
 
-    if expected_size == len(data) - 9:
+    if expected_size == len(data) - 9 or aligned_size == len(data):
         return True
 
-    if tail_data <= 0x10 and data[expected_size + 9 :] == b"#" * tail_data:
+    if (tail_data <= 0x10 and data[expected_size + 9 :] == b"#" * tail_data) or (
+        tail_data <= 0x4 and data[expected_size + 9 :] == b"\x00" * tail_data
+    ):
         return True  # SCPK files have these trailing "#" bytes :(
 
     return False