from ToolsTales import ToolsTales import subprocess from dicttoxml import dicttoxml import json import struct import shutil import os import re import io import pandas as pd import xml.etree.ElementTree as ET import lxml.etree as etree from xml.dom import minidom from pathlib import Path class ToolsTOPX(ToolsTales): def __init__(self, tbl): super().__init__("NDX", tbl, "Narikiri-Dungeon-X") with open("../{}/Data/Misc/{}".format(self.repo_name, self.tblFile), encoding="utf-8") as f: jsonRaw = json.load(f) self.jsonTblTags ={ k1:{ int(k2,16) if (k1 not in ["TBL", "NAME"]) else k2:v2 for k2,v2 in jsonRaw[k1].items()} for k1,v1 in jsonRaw.items()} self.itable = dict([[i, struct.pack(">H", int(j))] for j, i in self.jsonTblTags['TBL'].items()]) self.itags = dict([[i, j] for j, i in self.jsonTblTags['TAGS'].items()]) if "NAME" in self.jsonTblTags.keys(): self.inames = dict([[i, j] for j, i in self.jsonTblTags['NAME'].items()]) if "COLOR" in self.jsonTblTags.keys(): self.icolors = dict([[i, j] for j, i in self.jsonTblTags['COLOR'].items()]) self.id = 1 self.struct_id = 1 #Load the hash table for the files json_file = open('../Data/Narikiri-Dungeon-X/Misc/hashes.json', 'r') self.hashes = json.load(json_file) json_file.close() self.repo_name = 'Narikiri-Dungeon-X' self.misc = '../Data/{}/Misc'.format(self.repo_name) self.disc_path = '../Data/{}/Disc'.format(self.repo_name) self.story_XML_extract = '../Data/{}/Story/'.format(self.repo_name) #Files are the result of PAKCOMPOSER + Comptoe here self.story_XML_new = '../{}/Data/NDX/Story/XML'.format(self.repo_name) self.skit_extract = '../Data/{}/Skit/'.format(self.repo_name) #Files are the result of PAKCOMPOSER + Comptoe here self.all_extract = '../Data/{}/All/'.format(self.repo_name) self.all_original = '../Data/{}/Disc/Original/PSP_GAME/USRDIR/all.dat'.format(self.repo_name) self.all_new = '../Data/{}/Disc/New/PSP_GAME/USRDIR/all.dat'.format(self.repo_name) #File is all.dat self.story_struct_byte_code = b'\x18\x00\x0C\x04' self.story_string_byte_code = b'\x00\x00\x82\x02' self.make_dirs() ############################# # # Extraction of files and unpacking # ############################# # Make the basic directories for extracting all.dat def make_dirs(self): self.mkdir('../Data/{}/All'.format(self.repo_name)) self.mkdir('../Data/{}/All/battle'.format(self.repo_name)) self.mkdir('../Data/{}/All/battle/character'.format(self.repo_name)) self.mkdir('../Data/{}/All/battle/charsnd'.format(self.repo_name)) self.mkdir('../Data/{}/All/battle/data'.format(self.repo_name)) self.mkdir('../Data/{}/All/battle/effect'.format(self.repo_name)) self.mkdir('../Data/{}/All/battle/event'.format(self.repo_name)) self.mkdir('../Data/{}/All/battle/gui'.format(self.repo_name)) self.mkdir('../Data/{}/All/battle/map'.format(self.repo_name)) self.mkdir('../Data/{}/All/battle/resident'.format(self.repo_name)) self.mkdir('../Data/{}/All/battle/tutorial'.format(self.repo_name)) self.mkdir('../Data/{}/All/chat'.format(self.repo_name)) self.mkdir('../Data/{}/All/gim'.format(self.repo_name)) self.mkdir('../Data/{}/All/map'.format(self.repo_name)) self.mkdir('../Data/{}/All/map/data'.format(self.repo_name)) self.mkdir('../Data/{}/All/map/pack'.format(self.repo_name)) self.mkdir('../Data/{}/All/movie'.format(self.repo_name)) self.mkdir('../Data/{}/All/snd'.format(self.repo_name)) self.mkdir('../Data/{}/All/snd/init'.format(self.repo_name)) self.mkdir('../Data/{}/All/snd/se3'.format(self.repo_name)) self.mkdir('../Data/{}/All/snd/se3/map_mus'.format(self.repo_name)) self.mkdir('../Data/{}/All/snd/strpck'.format(self.repo_name)) self.mkdir('../Data/{}/All/sysdata'.format(self.repo_name)) # Extract each of the file from the all.dat def extract_files(self, start, size, filename): if filename in self.hashes.keys(): filename = self.hashes[filename] input_file = open( self.all_original, 'rb') input_file.seek(start, 0) data = input_file.read(size) output_file = open( os.path.join(self.all_extract, filename), 'wb') output_file.write(data) output_file.close() input_file.close() # Extract the story files def extract_All_Story(self): print("Extracting Story") path = os.path.join( self.all_extract, 'map/pack/') self.mkdir(self.story_XML_extract) for f in os.listdir( path ): if os.path.isfile( path+f) and '.cab' in f: file_name = self.story_XML_extract+'New/'+f.replace(".cab", ".pak3") self.extract_Story_File(path+f, file_name) #super().pakComposerAndComptoe(fileName, "-d", "-3") # Extract one single CAB file to the XML format def extract_Story_File(self,original_cab_file, file_name): #1) Extract CAB file to the PAK3 format #subprocess.run(['expand', original_cab_file, file_name]) #2) Decompress PAK3 to a folder #self.pakcomposer("-d", file_name, os.path.join( self.story_XML_extract, "New")) if os.path.isdir(file_name.replace(".pak3", "")): #3) Grab TSS file from PAK3 folder tss = self.get_tss_from_pak3( file_name.replace(".pak3", "")) #4) Extract TSS to XML self.extract_tss_XML(tss, original_cab_file) def get_tss_from_pak3(self, pak3_folder): if os.path.isdir(pak3_folder): folder_name = os.path.basename(pak3_folder) file_list = [os.path.dirname(pak3_folder) + "/" + folder_name + "/" + ele for ele in os.listdir(pak3_folder)] for file in file_list: with open(file, "rb") as f: data = f.read() if data[0:3] == b'TSS': print("... Extract TSS for file {} of size: {}".format(folder_name, len(data))) return io.BytesIO(data) def extract_tss_XML(self, tss, cab_file_name): root = etree.Element('SceneText') tss.read(12) strings_offset = struct.unpack(' 0): fileRead.seek(offset, 0) pos = fileRead.tell() b = fileRead.read(1) while b != end_strings: #print(hex(fileRead.tell())) b = ord(b) #Normal character if (b >= 0x80 and b <= 0x9F) or (b >= 0xE0 and b <= 0xEA): c = (b << 8) + ord(fileRead.read(1)) try: final_text += self.jsonTblTags['TBL'][c] except KeyError: b_u = (c >> 8) & 0xff b_l = c & 0xff final_text += ("{%02X}" % b_u) final_text += ("{%02X}" % b_l) #Line break elif b == 0x0A: final_text += ("\n") elif b == 0x0C: final_text += "" #Find a possible Color, Icon elif b in (0x1, 0xB): b2 = struct.unpack("') final_text += val elif chr(b) in self.PRINTABLE_CHARS: final_text += chr(b) elif b >= 0xA1 and b < 0xE0: final_text += struct.pack("B", b).decode("cp932") b = fileRead.read(1) return final_text, pos def text_to_bytes(self, text): splitLineBreak = text.split('\x0A') nb = len(splitLineBreak) bytesFinal = b'' i=0 for line in splitLineBreak: string_hex = re.split(self.HEX_TAG, line) string_hex = [sh for sh in string_hex if sh] #print(string_hex) for s in string_hex: if re.match(self.HEX_TAG, s): bytesFinal += struct.pack("B", int(s[1:3], 16)) else: s_com = re.split(self.COMMON_TAG, s) s_com = [sc for sc in s_com if sc] for c in s_com: if re.match(self.COMMON_TAG, c): if ":" in c: split = c.split(":") if split[0][1:] in self.itags.keys(): bytesFinal += struct.pack("B", self.itags[split[0][1:]]) bytesFinal += struct.pack("","") + ")" bytesFinal += c.encode("cp932") if c in self.icolors: bytesFinal += struct.pack("B", 0x5) bytesFinal += struct.pack("=2 and i 0: copy_path = os.path.join("../Data/{}/Menu/New/{}".format(self.repo_name, final_name)) Path(os.path.dirname(copy_path)).mkdir(parents=True, exist_ok=True) shutil.copy( os.path.join(self.all_extract, final_name), copy_path) order['order'].append(hash_) json.dump(order, order_json, indent = 4) order_json.close() def pack_Main_Archive(self): addrs = [] sizes = [] buffer = 0 print("Updating all.dat archive") with open( os.path.join( self.misc, 'order.json'), 'r') as order_file: order_hash = json.load(order_file) elf = open( self.elf_new, 'r+b') elf.seek(0x1FF624) #Menu files to reinsert menu_files = [ele['Hashes_Name'] for ele in self.menu_files_json if ele['Hashes_Name'] != ''] with open(self.all_new , 'wb') as all_file: for name in order_hash['order']: if name in self.hashes.keys(): name = self.hashes[name] data = b'' if os.path.dirname(name) in menu_files: with open( os.path.join( '../Data/{}/Menu/New'.format(self.repo_name), name), 'rb') as new_f: data = new_f.read() else: with open( os.path.join( self.all_extract, name), 'rb') as orig_f: data = orig_f.read() size = len(data) sizes.append(size) remainder = 0x800 - (size % 0x800) if remainder == 0x800: remainder = 0 addrs.append(buffer) buffer += size + remainder all_file.write(data) all_file.write(b'\x00' * remainder) for i in range(len(sizes)): elf.write(struct.pack('