diff --git a/Tales_Exe.py b/Tales_Exe.py index 82db226..6dad9cf 100644 --- a/Tales_Exe.py +++ b/Tales_Exe.py @@ -256,7 +256,7 @@ if __name__ == "__main__": tales_instance.extract_main_archive() if args.file_type == "Menu": - tales_instance.extract_all_Menu() + tales_instance.extract_all_menu() if args.file_type == "Story": tales_instance.extract_all_story() diff --git a/pythonlib/formats/pak.py b/pythonlib/formats/pak.py new file mode 100644 index 0000000..58faa24 --- /dev/null +++ b/pythonlib/formats/pak.py @@ -0,0 +1,185 @@ +from dataclasses import dataclass +import struct +from typing import Optional +from ..formats.FileIO import FileIO +from ..utils import comptolib + + +@dataclass +class pak_file(): + is_compressed: bool + type: int + data: bytes + + +class Pak(): + def __init__(self) -> None: + self.type = -1 + self.align = False + self.files = [] + + + @staticmethod + def from_path(path, type) -> 'Pak': + with FileIO(path) as f: + self = Pak() + # if type == -1: + # type = Pak.get_pak_type(f) + + file_amount = f.read_uint32() + self.files = [] + blobs: list[bytes] = [] + offsets: list[int] = [] + sizes: list[int] = [] + + # Pak0 + if type == 0: + for _ in range(file_amount): + sizes.append(f.read_uint32()) + + for size in sizes: + blobs.append(f.read(size)) + + # Pak1 + elif type == 1: + for _ in range(file_amount): + offsets.append(f.read_uint32()) + sizes.append(f.read_uint32()) + + for offset, size in zip(offsets, sizes): + f.seek(offset) + blobs.append(f.read(size)) + # Pak3 + elif type == 3: + for _ in range(file_amount): + offsets.append(f.read_uint32()) + f.seek(0, 2) + offsets.append(f.tell()) + for i, j in zip(offsets[::1], offsets[1::1]): + sizes.append(j - i) + for offset, size in zip(offsets, sizes): + f.seek(offset) + blobs.append(f.read(size)) + + for blob in blobs: + is_compressed = comptolib.is_compressed(blob) + c_type = 0 + if is_compressed: + c_type = blob[0] + blob = comptolib.decompress_data(blob) + + self.files.append(pak_file(is_compressed, c_type, blob)) + + return self + + @staticmethod + def get_pak_type(data: bytes) -> Optional[int]: + is_aligned = False + + data_size = len(data) + if data_size < 0x8: return None + + files = struct.unpack(" pak3_header_size: + calculated_size = 0 + for size in struct.unpack(f"<{files}I", data[4 : (files + 1) * 4]): + calculated_size += size + if calculated_size == len(data) - pak3_header_size: + return 0 + + # Test for pak1 + if is_aligned: + if pak1_check == first_entry: + return 1 + elif pak1_header_size == first_entry: + return 1 + + #Test for pak3 + previous = 0 + for offset in struct.unpack(f"<{files}I", data[4: (files+1)*4]): + + if offset > previous and offset >= pak3_header_size: + previous = offset + else: + return None + + last_offset = (4*(files+1))+8 + if data[last_offset : first_entry] == b'\x00' * (first_entry - last_offset): + return 3 + return None + + + def to_bytes(self, type=-1) -> bytes: + + compose_mode = type if type != -1 else self.type + if compose_mode == -1: + raise ValueError("Trying to compose an invalid PAK type") + + # Collect blobs + blobs = [] + for blob in self.files: + if blob.is_compressed: + blobs.append(comptolib.compress_data(blob.data, version=blob.type)) + else: + blobs.append(blob.data) + + # Compose + out = struct.pack(" bytes: data = b"" while True: - c = self.read(1) + c = src.read(1) if c == b"\x80": data += c break @@ -79,9 +80,9 @@ class Theirsce(FileIO): if opcode < 0x80: if opcode & 8 != 0: - data += self.read(2) + data += src.read(2) else: - data += self.read(1) + data += src.read(1) elif opcode < 0xC0: continue @@ -89,30 +90,30 @@ class Theirsce(FileIO): elif opcode < 0xE0: size_mask = (opcode >> 3) & 3 - if size_mask == 1: data += self.read(1) - elif size_mask == 2: data += self.read(2) - elif size_mask == 3: data += self.read(4) + if size_mask == 1: data += src.read(1) + elif size_mask == 2: data += src.read(2) + elif size_mask == 3: data += src.read(4) elif opcode < 0xF0: - data += self.read(1) + data += src.read(1) elif opcode < 0xF8: if (0xF2 <= opcode < 0xF5) or opcode == 0xF7: - data += self.read(2) + data += src.read(2) elif opcode == 0xF5: - data += self.read(4) + data += src.read(4) elif opcode == 0xF6: - data += self.read(1) + data += src.read(1) for _ in range(data[-1]): - cc = self.read_uint8() + cc = src.read_uint8() data += cc if cc & 8 != 0: - data += self.read(2) + data += src.read(2) else: - data += self.read(3) + data += src.read(3) elif opcode < 0xFC: - data += self.read(2) + data += src.read(2) elif opcode == 0xFE: continue diff --git a/pythonlib/games/ToolsNDX.py b/pythonlib/games/ToolsNDX.py index 82c6ea7..f613c12 100644 --- a/pythonlib/games/ToolsNDX.py +++ b/pythonlib/games/ToolsNDX.py @@ -648,7 +648,7 @@ class ToolsNDX(ToolsTales): self.unpack_Folder( menu_file_path) - def extract_all_Menu(self): + def extract_all_menu(self): res = [self.prepare_Menu_File(ele) for ele in list(set([ele['Hashes_Name'] for ele in self.menu_files_json if ele['Hashes_Name'] != '']))] @@ -688,7 +688,7 @@ class ToolsNDX(ToolsTales): if isinstance(pointers_offset, list): pointers_offset, pointers_value = self.get_Direct_Pointers(text_start, text_end, base_offset, pointers_offset, section,file_path) else: - pointers_offset, pointers_value = self.get_Style_Pointers( text_start, text_end, base_offset, section['Pointer_Offset_Start'], section['Style'], file_path) + pointers_offset, pointers_value = self.get_style_pointers( text_start, text_end, base_offset, section['Pointer_Offset_Start'], section['Style'], file_path) #Extract Text from the pointers diff --git a/pythonlib/games/ToolsTOR.py b/pythonlib/games/ToolsTOR.py index f3c664d..a5411b5 100644 --- a/pythonlib/games/ToolsTOR.py +++ b/pythonlib/games/ToolsTOR.py @@ -11,6 +11,8 @@ from pathlib import Path import lxml.etree as etree import pandas as pd from tqdm import tqdm +from pythonlib.formats.FileIO import FileIO +from pythonlib.formats.pak import Pak from pythonlib.formats.scpk import Scpk import pythonlib.utils.comptolib as comptolib @@ -52,6 +54,7 @@ class ToolsTOR(ToolsTales): story_XML_new = '../Tales-Of-Rebirth/Data/TOR/Story/' #Story XML files will be extracted here story_XML_patch = '../Data/Tales-Of-Rebirth/Story/' #Story XML files will be extracted here skit_XML_patch = '../Data/Tales-Of-Rebirth/Skits/' #Skits XML files will be extracted here + menu_XML_patch = '../Tales-Of-Rebirth/Data/TOR/Menu/' skit_XML_new = '../Tales-Of-Rebirth/Data/TOR/Skits/' dat_archive_extract = '../Data/Tales-Of-Rebirth/DAT/' # fmt: on @@ -421,22 +424,22 @@ class ToolsTOR(ToolsTales): return pointers_offset, texts_offset #Convert a bytes object to text using TAGS and TBL in the json file - def bytes_to_text(self, theirsce: Theirsce, offset=-1, end_strings = b"\x00"): + def bytes_to_text(self, src: FileIO, offset=-1, end_strings = b"\x00"): finalText = "" tags = self.jsonTblTags['TAGS'] chars = self.jsonTblTags['TBL'] if (offset > 0): - theirsce.seek(offset, 0) + src.seek(offset, 0) while True: - b = theirsce.read(1) + b = src.read(1) if b == end_strings: break b = ord(b) # Custom Encoded Text if (0x99 <= b <= 0x9F) or (0xE0 <= b <= 0xEB): - c = (b << 8) | theirsce.read_uint8() + c = (b << 8) | src.read_uint8() finalText += chars.get(c, "{%02X}{%02X}" % (c >> 8, c & 0xFF)) continue @@ -455,7 +458,7 @@ class ToolsTOR(ToolsTales): continue if b == 0x81: - next_b = theirsce.read(1) + next_b = src.read(1) if next_b == b"\x40": finalText += " " else: @@ -465,13 +468,13 @@ class ToolsTOR(ToolsTales): # Simple Tags if 0x3 <= b <= 0xF: - parameter = theirsce.read_uint32() + parameter = src.read_uint32() tag_name = tags.get(b, f"{b:02X}") tag_param = self.jsonTblTags.get(tag_name.upper(), {}).get(parameter, None) if tag_param is not None: - finalText += tag_param + finalText += f"<{tag_param}>" else: finalText += f"<{tag_name}:{self.hex2(parameter)}>" @@ -480,7 +483,7 @@ class ToolsTOR(ToolsTales): # Variable tags (same as above but using rsce bytecode as parameter) if 0x13 <= b <= 0x1A: tag_name = f"unk{b:02X}" - parameter = "".join([f"{c:02X}" for c in theirsce.read_tag_bytes()]) + parameter = "".join([f"{c:02X}" for c in Theirsce.read_tag_bytes(src)]) finalText += f"<{tag_name}:{parameter}>" continue @@ -754,6 +757,98 @@ class ToolsTOR(ToolsTales): with open(final_path / fname, "wb") as output: output.write(data) + + + def get_style_pointers(self, text_start, text_max, base_offset, start_offset, style, file: FileIO): + file.seek(0, 2) + f_size = file.tell() + + file.seek(start_offset) + pointers_offset = [] + pointers_value = [] + split = [ele for ele in re.split(r'(P)|(\d+)', style) if ele] + ok = True + + while ok: + for step in split: + if step == "P": + text_offset = struct.unpack("= text_start and text_offset < text_max: + pointers_value.append(text_offset) + pointers_offset.append(file.tell()-4) + + else: + ok = False + else: + file.read(int(step)) + + return pointers_offset, pointers_value + + + def extract_all_menu(self) -> None: + print("Extracting Menu Files...") + + #Prepare the menu files (Unpack PAK files and use comptoe) + xml_path = Path(self.menu_XML_patch) / "XML" + xml_path.mkdir(exist_ok=True) + + for entry in tqdm(self.menu_files_json): + file_path = Path(entry["file_path"]) + if entry["is_pak"]: + pak = Pak.from_path(file_path, int(entry["pak_type"])) + + for p_file in entry["files"]: + f_index = int(p_file["file"]) + with FileIO(pak[f_index].data, "rb") as f: + xml_data = self.extract_menu_file(p_file, f) + + with open(xml_path / f"{file_path.stem}_{f_index:03d}.xml", "wb") as xmlFile: + xmlFile.write(xml_data) + + else: + with FileIO(entry["file_path"], "rb") as f: + xml_data = self.extract_menu_file(entry, f) + + with open(xml_path / f"{file_path.stem}.xml", "wb") as xmlFile: + xmlFile.write(xml_data) + + + def extract_menu_file(self, file_def, f: FileIO): + section_list = [] + pointers_offset_list = [] + texts_list = [] + + base_offset = int(file_def["base_offset"]) + # print("BaseOffset:{}".format(base_offset)) + + for section in file_def['sections']: + + text_start = int(section['text_start']) + text_end = int(section['text_end']) + + #Extract Pointers of the file + # print("Extract Pointers") + pointers_offset, pointers_value = self.get_style_pointers(text_start, text_end, base_offset, section['pointers_start'], section['style'], f) + # print([hex(pv) for pv in pointers_value]) + + #Extract Text from the pointers + # print("Extract Text") + texts = [ self.bytes_to_text(f, ele) for ele in pointers_value] + + #Make a list + section_list.extend( [section['section']] * len(texts)) + pointers_offset_list.extend( pointers_offset) + texts_list.extend( texts ) + + #Remove duplicates + list_informations = self.remove_duplicates(section_list, pointers_offset_list, texts_list) + + #Build the XML Structure with the information + root = self.create_Node_XML("", list_informations, "Menu", "MenuText") + + #Write to XML file + return etree.tostring(root, encoding="UTF-8", pretty_print=True) def pack_main_archive(self): diff --git a/pythonlib/games/ToolsTales.py b/pythonlib/games/ToolsTales.py index 919c6e1..97455c2 100644 --- a/pythonlib/games/ToolsTales.py +++ b/pythonlib/games/ToolsTales.py @@ -16,8 +16,10 @@ import pycdlib import pygsheets from googleapiclient.errors import HttpError from tqdm import tqdm +from pythonlib.formats.FileIO import FileIO import pythonlib.formats.fps4 as fps4 +from pythonlib.formats.pak import Pak class ToolsTales: @@ -932,7 +934,7 @@ class ToolsTales: return [pointers_offset, pointers_value] - def get_Style_Pointers(self, text_start, text_max, base_offset, start_offset, style, file_path): + def get_style_pointers(self, text_start, text_max, base_offset, start_offset, style, file_path): f_size = os.path.getsize(file_path) with open(file_path , "rb") as f: @@ -940,7 +942,7 @@ class ToolsTales: f.seek(start_offset, 0) pointers_offset = [] pointers_value = [] - split = [ele for ele in re.split(r'(P)|(\d+)', style) if ele != None and ele != ''] + split = [ele for ele in re.split(r'(P)|(\d+)', style) if ele] ok = True while ok: @@ -999,7 +1001,7 @@ class ToolsTales: #Extract Pointers of the file print("Extract Pointers") - pointers_offset, pointers_value = self.get_Style_Pointers( text_start, text_end, base_offset, section['Pointer_Offset_Start'], section['Style'], file_path) + pointers_offset, pointers_value = self.get_style_pointers( text_start, text_end, base_offset, section['Pointer_Offset_Start'], section['Style'], file_path) print([hex(pointers_value) for ele in pointers_value]) #Extract Text from the pointers diff --git a/pythonlib/utils/comptolib.py b/pythonlib/utils/comptolib.py index 16b1f29..ccc38c3 100644 --- a/pythonlib/utils/comptolib.py +++ b/pythonlib/utils/comptolib.py @@ -156,13 +156,19 @@ def is_compressed(data: bytes) -> bool: if len(data) < 0x09: return False + if data[0] not in (0, 1, 3): + return False + expected_size = struct.unpack("