lifebottle/PythonLib
0
0
Fork 0
mirror of https://github.com/lifebottle/PythonLib.git synced 2026-02-13 15:25:50 -08:00
Code Issues Packages Projects Releases Wiki Activity

Removing binary files and test files

Browse Source
This commit is contained in:
Stewie
2022-03-19 18:44:17 -04:00
parent 780285910c
commit b7b57bb94e
33 changed files with 457 additions and 72777 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
00013.pak3
View File

Binary file not shown.

BIN
00023_original_0002.bin
View File

Binary file not shown.

BIN
10199.theirsce
View File

Binary file not shown.

BIN
10247.scpk
View File

Binary file not shown.

BIN
Event.dat
View File

Binary file not shown.

34
NDX_Test.py Normal file
View File

@@ -0,0 +1,34 @@
import ToolsNDX
import json
import struct
import comptolib
import io
import re
import string
import pandas as pd
import json
import os
import lxml.etree as etree
import cabarchive
tool = ToolsNDX.ToolsNDX("TBL_All.json")
tool.extract_Story_File("ep_010_030_0000.unknown")
tool.extract_FPS4()
tool.extract_All_Pak3()
tool.extract_All_Cab()
tool.extract_Main_Archive()
tool.bytes_to_text_with_offset("menu.unknown", 0x30FF2, b'')
base_path = "../Data/NDX/Menu/New/BT_DATA"
list_files = os.listdir(base_path)
for file in list_files:
tool.extract_Cab(os.path.join( base_path, file), file.split(".")[0]+".bin")

10822
SLPS.xml
View File

File diff suppressed because it is too large Load Diff

BIN
SLPS_254.50
View File

Binary file not shown.

BIN
SLPS_text.xlsx
View File

Binary file not shown.

1989
TBL_All_Backup.json
View File

File diff suppressed because it is too large Load Diff

1705
TBL_PhantasiaX.txt
View File

File diff suppressed because it is too large Load Diff

1705
TOPX.tbl
View File

File diff suppressed because it is too large Load Diff

3727
TOPX_All.json
View File

File diff suppressed because it is too large Load Diff

BIN
TOPX_table.xlsx
View File

Binary file not shown.

1491
TOR_Consumable_dump_eng.txt
View File

File diff suppressed because it is too large Load Diff

5
TOR_Test.py
View File

@@ -12,7 +12,10 @@ import lxml.etree as etree
tool = ToolsTOR.ToolsTOR("TBL_All.json")
tool.extract_Main_Archive()
tool.extract_All_Menu()
tool.extract_All_Story_Files(debug=True)
t = tool.pack_Story_File("10247.scpk")

4796
ToR.tbl
View File

File diff suppressed because it is too large Load Diff

4591
ToR_utf8.tbl
View File

File diff suppressed because it is too large Load Diff

272
ToolsNDX.py
View File

@@ -14,33 +14,240 @@ from xml.dom import minidom
from pathlib import Path
import string
import io
import fps4
class ToolsNDX(ToolsTales):
POINTERS_BEGIN = 0xD76B0 # Offset to DAT.BIN pointer list start in SLPS_254.50 file
POINTERS_END = 0xE60C8 # Offset to DAT.BIN pointer list end in SLPS_254.50 file
HIGH_BITS = 0xFFFFFFC0
LOW_BITS = 0x3F
POINTERS_BEGIN = 0x1FF624 # Offset to all.dat
POINTERS_END = 0xE60C8
#Path to used
allDatOriginal = '../Data/NDX/Disc/Original/PSP_GAME/USRDIR/all.dat'
allDatNew = '../Data/NDX/Disc/New/all.dat'
elfOriginal = '../Data/NDX/MISC/ULJS00293.BIN'
elfNew = '../Data/TOR/Disc/New/ULJS00293.BIN'
storyPathArchives= '../Data/TOR/Story/SCPK/' #Story XML files will be extracted here
storyPathXML = '../Data/TOR/Story/XML/' #SCPK will be repacked here
skitPathArchives = '../Data/TOR/Skits/' #Skits XML files will be extracted here
datPathExtract = '../Data/TOR/DAT/'
allDat_Original = '../Data/NDX/Disc/Original/PSP_GAME/USRDIR/all.dat'
allDat_New = '../Data/NDX/Disc/New/all.dat'
elf_Original = '../Data/NDX/MISC/ULJS00293.BIN'
elf_New = '../Data/NDX/Disc/New/ULJS00293.BIN'
story_Path = '../Data/NDX/Story/'
skits_Path = '../Data/NDX/Skits/'
events_Path = '../Data/NDX/Events/'
all_Path_Extract = '../Data/NDX/DAT/'
def __init__(self, tbl):
super().__init__("TOR", tbl)
super().__init__("NDX", tbl)
self.struct_byte_code = b'\x18\x00\x0C\x04'
self.struct_strings_code = b'\x00\x00\x82\x02'
self.strings_byte_code = b'\x00\x00\x82\x02'
json_file = open('../Data/NDX/Misc/hashes.json', 'r')
self.hashes = json.load(json_file)
self.extract_names = True
def make_Dirs(self):
base_path = "../Data/NDX/All/"
self.mkdir(base_path)
self.mkdir(base_path+'battle')
self.mkdir(base_path+'battle/character')
self.mkdir(base_path+'battle/charsnd')
self.mkdir(base_path+'battle/data')
self.mkdir(base_path+'battle/effect')
self.mkdir(base_path+'battle/event')
self.mkdir(base_path+'battle/gui')
self.mkdir(base_path+'battle/map')
self.mkdir(base_path+'battle/resident')
self.mkdir(base_path+'battle/tutorial')
self.mkdir(base_path+'chat')
self.mkdir(base_path+'gim')
self.mkdir(base_path+'map')
self.mkdir(base_path+'map/data')
self.mkdir(base_path+'map/pack')
self.mkdir(base_path+'movie')
self.mkdir(base_path+'snd')
self.mkdir(base_path+'snd/init')
self.mkdir(base_path+'snd/se3')
self.mkdir(base_path+'snd/se3/map_mus')
self.mkdir(base_path+'snd/strpck')
self.mkdir(base_path+'sysdata')
def bytes_to_text_TOPX(self, fileRead, offset=-1, end_strings = b"\x00"):
finalText = ''
TAGS = self.jsonTblTags['TAGS']
if (offset > 0):
fileRead.seek(offset, 0)
pos = fileRead.tell()
b = fileRead.read(2)
while_condition = b''
while while_condition != b'\x00\x00':
#print(hex(fileRead.tell()))
b = ord(b)
if (b >= 0x99 and b <= 0x9F) or (b >= 0xE0 and b <= 0xEB):
c = (b << 8) + ord(fileRead.read(1))
# if str(c) not in json_data.keys():
# json_data[str(c)] = char_index[decode(c)]
try:
finalText += (self.jsonTblTags['TBL'][str(c)])
except KeyError:
b_u = (c >> 8) & 0xff
b_l = c & 0xff
finalText += ("{%02X}" % b_u)
finalText += ("{%02X}" % b_l)
elif b == 0x1:
finalText += ("\n")
elif b in (0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xB, 0xC, 0xD, 0xE, 0xF):
b2 = struct.unpack("<L", fileRead.read(4))[0]
if b in TAGS:
tag_name = TAGS.get(b)
tag_param = None
tag_search = tag_name.upper()+'S'
if (tag_search in self.jsonTblTags.keys()):
tags2 = self.jsonTblTags[tag_search]
tag_param = tags2.get(b2, None)
if tag_param != None:
finalText += tag_param
else:
finalText += ("<%s:%08X>" % (tag_name, b2))
else:
finalText += "<%02X:%08X>" % (b, b2)
elif chr(b) in self.PRINTABLE_CHARS:
finalText += chr(b)
elif b >= 0xA1 and b < 0xE0:
finalText += struct.pack("B", b).decode("cp932")
elif b in (0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19):
finalText += "{%02X}" % b
next_b = b""
while next_b != b"\x80":
next_b = fileRead.read(1)
finalText += "{%02X}" % ord(next_b)
#if next_b != b'':
#else:
# next_b = b"\x80"
elif b == 0x81:
next_b = fileRead.read(1)
if next_b == b"\x40":
finalText += "　"
else:
finalText += "{%02X}" % b
finalText += "{%02X}" % ord(next_b)
else:
finalText += "{%02X}" % b
b = fileRead.read(2)
end = fileRead.tell()
size = fileRead.tell() - pos - 1
fileRead.seek(pos)
hex_string = fileRead.read(size).hex()
hex_values = ' '.join(a+b for a,b in zip(hex_string[::2], hex_string[1::2]))
fileRead.seek(end)
return finalText, hex_values
def extract_Files(self, input_file, size, filename):
hash_name = filename
if filename in self.hashes.keys():
hash_name = self.hashes[filename]
data = input_file.read(size)
base_path = '../Data/NDX/All/'
if self.extract_names:
try:
output_file = open(base_path + hash_name, 'wb')
except:
output_file = open(base_path + filename, 'wb')
else:
output_file = open(base_path + filename, 'wb')
output_file.write(data)
output_file.close()
def extract_All_Pak3(self):
#Story
story_pak3 = [self.story_Path+"New/"+ele for ele in os.listdir(self.story_Path+"New")]
res = [ self.pakComposer_Comptoe(ele, "-d", "-3", False) for ele in story_pak3]
def extract_FPS4(self):
print("potato")
self.fps4_action("-d", "../Data/NDX/All/battle/data/bt_data.b", "../Data/NDX/All/battle/data/bt_data_battle.dat", "../Data/NDX/Menu/New")
def extract_All_Cab(self):
menuCab = {}
menuCab['File'] = []
event_path_extract = "../Data/NDX/All\map"
skits_path_extract = "../Data/NDX/All\chat"
story_path_extract = "../Data/NDX/All\map\pack"
path_exceptions = [event_path_extract, skits_path_extract, story_path_extract, "../Data/NDX/All"]
for path, subdirs, files in os.walk("../Data/NDX/All"):
print(path)
if path not in path_exceptions:
for name in files:
with open(os.path.join(path,name), "rb") as f:
data =f.read()
if data[:4] == b'MSCF':
file_name = name.split(".")[0]
self.extract_Cab( os.path.join(path, name), "../Data/NDX/Menu/New/{}.bin".format(file_name))
#Events
events_files = [file for file in os.listdir(event_path_extract) if ".bin" in file]
print(events_files)
for file in events_files:
new_file_name = self.events_Path + 'New/{}'.format(file)
self.extract_Cab( os.path.join(event_path_extract,file), new_file_name)
#Skits
skits_files = [file for file in os.listdir(skits_path_extract) if ".bin" in file]
for file in skits_files:
new_file_name = self.skits_Path+'New/{}.pak3'.format(self.get_file_name(file))
self.extract_Cab(os.path.join(skits_path_extract,file), new_file_name)
#Story
story_files = [file for file in os.listdir(story_path_extract) if ".bin" in file]
for file in story_files:
new_file_name = self.story_Path +'New/{}.bin'.format(file_name)
self.extract_Cab(os.path.join(story_path_extract,file), new_file_name)
def extract_Main_Archive(self):
print("...Unpacking all.dat")
shutil.rmtree("../Data/NDX/All")
self.make_Dirs()
order = {}
order['order'] = []
order_json = open('../Data/NDX/Misc/order.json', 'w')
eboot = open(self.elf_Original, 'rb')
eboot.seek(self.POINTERS_BEGIN)
with open(self.allDat_Original, 'rb') as all_dat_file:
while True:
file_info = struct.unpack('<3I', eboot.read(12))
if(file_info[2] == 0):
break
hash_ = "%08X" % file_info[2]
all_dat_file.seek(file_info[0])
self.extract_Files(all_dat_file, file_info[1], hash_)
order['order'].append(hash_)
json.dump(order, order_json, indent = 4)
order_json.close()
def extract_Story_Pointers(self, f):
@@ -72,7 +279,7 @@ class ToolsNDX(ToolsTales):
return structDict, stringDict, baseOffset, blockSize
def buildStructEntry(self, storyFile, f, structDict, baseOffset, blockSize, root):
def build_Struct_Entry(self, storyFile, f, structDict, baseOffset, blockSize, root):
#Extract pointers inside each structure
pathFile = os.path.join(os.path.abspath(os.path.dirname(os.getcwd())))
@@ -99,17 +306,23 @@ class ToolsNDX(ToolsTales):
unknown1Add = struct.unpack('<I', f.read(4))[0] + baseOffset
unknown2Add = struct.unpack('<I', f.read(4))[0] + baseOffset
personText = extractText(f, personAdd, textAdd - personAdd - 1, tbl)
#print(personText)
unknown1Text = extractText(f, unknown1Add, unknown2Add - unknown1Add - 1, tbl)
unknown2Text = extractText(f, unknown2Add, structValueOffset + baseOffset - unknown2Add - 1, tbl)
f.seek(personAdd)
print("Offset : {}".format(hex(personAdd)))
personText, person_hex = self.bytes_to_text(f)
print(personText)
#personText = self.extractText(f, personAdd, textAdd - personAdd - 1, tbl)
#unknown1Text = self.extractText(f, unknown1Add, unknown2Add - unknown1Add - 1, tbl)
#unknown2Text = self.extractText(f, unknown2Add, structValueOffset + baseOffset - unknown2Add - 1, tbl)
#Add all the nodes to the base Struct Node
structNode = etree.SubElement(root, "Struct")
etree.SubElement(structNode, "PointerOffset").text = str(structOffset)
etree.SubElement(structNode, "PointerUnknownValue").text = str(pointer)
etree.SubElement(structNode, "Unknown1Text").text = unknown1Text
etree.SubElement(structNode, "Unknown2Text").text = unknown2Text
#etree.SubElement(structNode, "Unknown1Text").text = unknown1Text
#etree.SubElement(structNode, "Unknown2Text").text = unknown2Text
etree.SubElement(structNode, "TextOffset").text = str(personAdd)
etree.SubElement(structNode, "PersonJapaneseText").text = personText
etree.SubElement(structNode, "PersonEnglishText").text = "";
@@ -119,16 +332,16 @@ class ToolsNDX(ToolsTales):
#entryListNode = etree.SubElement(structNode, "EntryList")
japText = extractText(f, textAdd, unknown1Add - textAdd - 1, tbl)
listVoiceId = re.findall("\((\w+)\)", japText)
#japText = extractText(f, textAdd, unknown1Add - textAdd - 1, tbl)
#listVoiceId = re.findall("\((\w+)\)", japText)
if len(listVoiceId) > 0:
etree.SubElement(structNode, "Type").text = "VoiceId"
else:
etree.SubElement(structNode, "Type").text = "NoVoiceId"
#if len(listVoiceId) > 0:
# etree.SubElement(structNode, "Type").text = "VoiceId"
#else:
# etree.SubElement(structNode, "Type").text = "NoVoiceId"
addEntry(japText, structNode, listVoiceId)
#addEntry(japText, structNode, listVoiceId)
return root, personPointers
@@ -196,4 +409,3 @@ class ToolsNDX(ToolsTales):
with open(os.path.join( r"G:\TalesHacking\TOP_Narikiri\GithubProject\TranslationApp\TranslationApp\bin\Debug", fileName+".xml"), "wb") as xmlFile:
xmlFile.write(txt)

11
ToolsTOR.py
View File

@@ -28,21 +28,20 @@ class ToolsTOR(ToolsTales):
datBinNew = '../Data/TOR/Disc/New/DAT.BIN'
elfOriginal = '../Data/TOR/Disc/Original/SLPS_254.50'
elfNew = '../Data/TOR/Disc/New/SLPS_254.50'
storyPathArchives= '../Data/TOR/Story/New/' #Story XML files will be extracted here
storyPathXML = '../Data/TOR/Story/XML/' #SCPK will be repacked here
skitPathArchives = '../Data/TOR/Skits/' #Skits XML files will be extracted here
storyPathArchives= '../Tales-Of-Rebirth/Data/TOR/Story/New/' #Story XML files will be extracted here
storyPathXML = '../Tales-Of-Rebirth/Data/TOR/Story/XML/' #SCPK will be repacked here
skitPathArchives = '../Tales-Of-Rebirth/Data/TOR/Skits/' #Skits XML files will be extracted here
datPathExtract = '../Data/TOR/DAT/'
def __init__(self, tbl):
super().__init__("TOR", tbl)
super().__init__("TOR", tbl, "Tales-of-Rebirth")
#byteCode
self.story_byte_code = b"\xF8"

Some files were not shown because too many files have changed in this diff Show More