pokehacks/pokecrystal-board
0
0
Fork 0
mirror of https://gitlab.com/xCrystal/pokecrystal-board.git synced 2024-09-09 09:51:34 -07:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #587 from Rangi42/master

Browse Source 
Remove dependency on 'extras' submodule
This commit is contained in:
Rangi 2018-12-31 18:26:33 -05:00 committed by GitHub
commit 9f82304186
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 4187 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitmodules vendored
View File

@ -1,3 +0,0 @@
[submodule "extras"]
path = extras
url = git://github.com/pret/pokemon-reverse-engineering-tools.git

29
Makefile
View File

@ -1,15 +1,3 @@
ifeq (,$(shell which sha1sum))
SHA1 := shasum
else
SHA1 := sha1sum
endif
RGBDS ?=
RGBASM ?= $(RGBDS)rgbasm
RGBFIX ?= $(RGBDS)rgbfix
RGBGFX ?= $(RGBDS)rgbgfx
RGBLINK ?= $(RGBDS)rgblink
roms := pokecrystal.gbc pokecrystal11.gbc
crystal_obj := \
@ -31,10 +19,25 @@ lib/mobile/main.o
crystal11_obj := $(crystal_obj:.o=11.o)
### Build tools
ifeq (,$(shell which sha1sum))
SHA1 := shasum
else
SHA1 := sha1sum
endif
RGBDS ?=
RGBASM ?= $(RGBDS)rgbasm
RGBFIX ?= $(RGBDS)rgbfix
RGBGFX ?= $(RGBDS)rgbgfx
RGBLINK ?= $(RGBDS)rgblink
### Build targets
.SUFFIXES:
.PHONY: all crystal crystal11 clean compare tools tidy
.PHONY: all crystal crystal11 clean tidy compare tools
.SECONDEXPANSION:
.PRECIOUS:
.SECONDARY:

1
extras

@ -1 +0,0 @@
Subproject commit 2ae1ae2f19c842da965cd3db1b0da8bf108ac235

4
gfx.py → tools/gfx.py
View File

@ -3,7 +3,7 @@
import os
import argparse
from extras.pokemontools import gfx, lz
from pokemontools import gfx, lz
# Graphics with inverted tilemaps that aren't covered by filepath_rules.
@ -146,7 +146,7 @@ def filepath_rules(filepath):
pokemon_name = ''
if 'gfx/pics/' in filedir:
if 'gfx/pokemon/' in filedir:
pokemon_name = filedir.split('/')[-1]
if pokemon_name.startswith('unown_'):
index = filedir.find(pokemon_name)

1
tools/pokemontools/__init__.py Normal file
View File

@ -0,0 +1 @@
# A subset of https://github.com/pret/pokemon-reverse-engineering-tools

938
tools/pokemontools/gfx.py Normal file
View File

@ -0,0 +1,938 @@
# -*- coding: utf-8 -*-
import os
import sys
import png
from math import sqrt, floor, ceil
import argparse
import operator
from lz import Compressed, Decompressed
def split(list_, interval):
"""
Split a list by length.
"""
for i in xrange(0, len(list_), interval):
j = min(i + interval, len(list_))
yield list_[i:j]
def hex_dump(data, length=0x10):
"""
just use hexdump -C
"""
margin = len('%x' % len(data))
output = []
address = 0
for line in split(data, length):
output += [
hex(address)[2:].zfill(margin) +
' | ' +
' '.join('%.2x' % byte for byte in line)
]
address += length
return '\n'.join(output)
def get_tiles(image):
"""
Split a 2bpp image into 8x8 tiles.
"""
return list(split(image, 0x10))
def connect(tiles):
"""
Combine 8x8 tiles into a 2bpp image.
"""
return [byte for tile in tiles for byte in tile]
def transpose(tiles, width=None):
"""
Transpose a tile arrangement along line y=-x.
00 01 02 03 04 05 00 06 0c 12 18 1e
06 07 08 09 0a 0b 01 07 0d 13 19 1f
0c 0d 0e 0f 10 11 <-> 02 08 0e 14 1a 20
12 13 14 15 16 17 03 09 0f 15 1b 21
18 19 1a 1b 1c 1d 04 0a 10 16 1c 22
1e 1f 20 21 22 23 05 0b 11 17 1d 23
00 01 02 03 00 04 08
04 05 06 07 <-> 01 05 09
08 09 0a 0b 02 06 0a
03 07 0b
"""
if width == None:
width = int(sqrt(len(tiles))) # assume square image
tiles = sorted(enumerate(tiles), key= lambda (i, tile): i % width)
return [tile for i, tile in tiles]
def transpose_tiles(image, width=None):
return connect(transpose(get_tiles(image), width))
def interleave(tiles, width):
"""
00 01 02 03 04 05 00 02 04 06 08 0a
06 07 08 09 0a 0b 01 03 05 07 09 0b
0c 0d 0e 0f 10 11 --> 0c 0e 10 12 14 16
12 13 14 15 16 17 0d 0f 11 13 15 17
18 19 1a 1b 1c 1d 18 1a 1c 1e 20 22
1e 1f 20 21 22 23 19 1b 1d 1f 21 23
"""
interleaved = []
left, right = split(tiles[::2], width), split(tiles[1::2], width)
for l, r in zip(left, right):
interleaved += l + r
return interleaved
def deinterleave(tiles, width):
"""
00 02 04 06 08 0a 00 01 02 03 04 05
01 03 05 07 09 0b 06 07 08 09 0a 0b
0c 0e 10 12 14 16 --> 0c 0d 0e 0f 10 11
0d 0f 11 13 15 17 12 13 14 15 16 17
18 1a 1c 1e 20 22 18 19 1a 1b 1c 1d
19 1b 1d 1f 21 23 1e 1f 20 21 22 23
"""
deinterleaved = []
rows = list(split(tiles, width))
for left, right in zip(rows[::2], rows[1::2]):
for l, r in zip(left, right):
deinterleaved += [l, r]
return deinterleaved
def interleave_tiles(image, width):
return connect(interleave(get_tiles(image), width))
def deinterleave_tiles(image, width):
return connect(deinterleave(get_tiles(image), width))
def condense_image_to_map(image, pic=0):
"""
Reduce an image of adjacent frames to an image containing a base frame and any unrepeated tiles.
Returns the new image and the corresponding tilemap used to reconstruct the input image.
If <pic> is 0, ignore the concept of frames. This behavior might be better off as another function.
"""
tiles = get_tiles(image)
new_tiles, tilemap = condense_tiles_to_map(tiles, pic)
new_image = connect(new_tiles)
return new_image, tilemap
def condense_tiles_to_map(tiles, pic=0):
"""
Reduce a sequence of tiles representing adjacent frames to a base frame and any unrepeated tiles.
Returns the new tiles and the corresponding tilemap used to reconstruct the input tile sequence.
If <pic> is 0, ignore the concept of frames. This behavior might be better off as another function.
"""
# Leave the first frame intact for pics.
new_tiles = tiles[:pic]
tilemap = range(pic)
for i, tile in enumerate(tiles[pic:]):
if tile not in new_tiles:
new_tiles.append(tile)
if pic:
# Match the first frame exactly where possible.
# This reduces the space needed to replace tiles in pic animations.
# For example, if a tile is repeated twice in the first frame,
# but at the same relative index as the second tile, use the second index.
# When creating a bitmask later, the second index would not require a replacement, but the first index would have.
pic_i = i % pic
if tile == new_tiles[pic_i]:
tilemap.append(pic_i)
else:
tilemap.append(new_tiles.index(tile))
else:
tilemap.append(new_tiles.index(tile))
return new_tiles, tilemap
def test_condense_tiles_to_map():
test = condense_tiles_to_map(list('abcadbae'))
if test != (list('abcde'), [0, 1, 2, 0, 3, 1, 0, 4]):
raise Exception(test)
test = condense_tiles_to_map(list('abcadbae'), 2)
if test != (list('abcde'), [0, 1, 2, 0, 3, 1, 0, 4]):
raise Exception(test)
test = condense_tiles_to_map(list('abcadbae'), 4)
if test != (list('abcade'), [0, 1, 2, 3, 4, 1, 0, 5]):
raise Exception(test)
test = condense_tiles_to_map(list('abcadbea'), 4)
if test != (list('abcade'), [0, 1, 2, 3, 4, 1, 5, 3]):
raise Exception(test)
def to_file(filename, data):
"""
Apparently open(filename, 'wb').write(bytearray(data)) won't work.
"""
file = open(filename, 'wb')
for byte in data:
file.write('%c' % byte)
file.close()
def decompress_file(filein, fileout=None):
image = bytearray(open(filein).read())
de = Decompressed(image)
if fileout == None:
fileout = os.path.splitext(filein)[0]
to_file(fileout, de.output)
def compress_file(filein, fileout=None):
image = bytearray(open(filein).read())
lz = Compressed(image)
if fileout == None:
fileout = filein + '.lz'
to_file(fileout, lz.output)
def bin_to_rgb(word):
red = word & 0b11111
word >>= 5
green = word & 0b11111
word >>= 5
blue = word & 0b11111
return (red, green, blue)
def convert_binary_pal_to_text_by_filename(filename):
pal = bytearray(open(filename).read())
return convert_binary_pal_to_text(pal)
def convert_binary_pal_to_text(pal):
output = ''
words = [hi * 0x100 + lo for lo, hi in zip(pal[::2], pal[1::2])]
for word in words:
red, green, blue = ['%.2d' % c for c in bin_to_rgb(word)]
output += '\tRGB ' + ', '.join((red, green, blue))
output += '\n'
return output
def read_rgb_macros(lines):
colors = []
for line in lines:
macro = line.split(" ")[0].strip()
if macro == 'RGB':
params = ' '.join(line.split(" ")[1:]).split(',')
red, green, blue = [int(v) for v in params]
colors += [[red, green, blue]]
return colors
def rewrite_binary_pals_to_text(filenames):
for filename in filenames:
pal_text = convert_binary_pal_to_text_by_filename(filename)
with open(filename, 'w') as out:
out.write(pal_text)
def flatten(planar):
"""
Flatten planar 2bpp image data into a quaternary pixel map.
"""
strips = []
for bottom, top in split(planar, 2):
bottom = bottom
top = top
strip = []
for i in xrange(7,-1,-1):
color = (
(bottom >> i & 1) +
(top *2 >> i & 2)
)
strip += [color]
strips += strip
return strips
def to_lines(image, width):
"""
Convert a tiled quaternary pixel map to lines of quaternary pixels.
"""
tile_width = 8
tile_height = 8
num_columns = width / tile_width
height = len(image) / width
lines = []
for cur_line in xrange(height):
tile_row = cur_line / tile_height
line = []
for column in xrange(num_columns):
anchor = (
num_columns * tile_row * tile_width * tile_height +
column * tile_width * tile_height +
cur_line % tile_height * tile_width
)
line += image[anchor : anchor + tile_width]
lines += [line]
return lines
def dmg2rgb(word):
"""
For PNGs.
"""
def shift(value):
while True:
yield value & (2**5 - 1)
value >>= 5
word = shift(word)
# distribution is less even w/ << 3
red, green, blue = [int(color * 8.25) for color in [word.next() for _ in xrange(3)]]
alpha = 255
return (red, green, blue, alpha)
def rgb_to_dmg(color):
"""
For PNGs.
"""
word = (color['r'] / 8)
word += (color['g'] / 8) << 5
word += (color['b'] / 8) << 10
return word
def pal_to_png(filename):
"""
Interpret a .pal file as a png palette.
"""
with open(filename) as rgbs:
colors = read_rgb_macros(rgbs.readlines())
a = 255
palette = []
for color in colors:
# even distribution over 000-255
r, g, b = [int(hue * 8.25) for hue in color]
palette += [(r, g, b, a)]
white = (255,255,255,255)
black = (000,000,000,255)
if white not in palette and len(palette) < 4:
palette = [white] + palette
if black not in palette and len(palette) < 4:
palette = palette + [black]
return palette
def png_to_rgb(palette):
"""
Convert a png palette to rgb macros.
"""
output = ''
for color in palette:
r, g, b = [color[c] / 8 for c in 'rgb']
output += '\tRGB ' + ', '.join(['%.2d' % hue for hue in (r, g, b)])
output += '\n'
return output
def read_filename_arguments(filename):
"""
Infer graphics conversion arguments given a filename.
Arguments are separated with '.'.
"""
parsed_arguments = {}
int_arguments = {
'w': 'width',
'h': 'height',
't': 'tile_padding',
}
arguments = os.path.splitext(filename)[0].lstrip('.').split('.')[1:]
for argument in arguments:
# Check for integer arguments first (i.e. "w128").
arg = argument[0]
param = argument[1:]
if param.isdigit():
arg = int_arguments.get(arg, False)
if arg:
parsed_arguments[arg] = int(param)
elif argument == 'arrange':
parsed_arguments['norepeat'] = True
parsed_arguments['tilemap'] = True
# Pic dimensions (i.e. "6x6").
elif 'x' in argument and any(map(str.isdigit, argument)):
w, h = argument.split('x')
if w.isdigit() and h.isdigit():
parsed_arguments['pic_dimensions'] = (int(w), int(h))
else:
parsed_arguments[argument] = True
return parsed_arguments
def export_2bpp_to_png(filein, fileout=None, pal_file=None, height=0, width=0, tile_padding=0, pic_dimensions=None, **kwargs):
if fileout == None:
fileout = os.path.splitext(filein)[0] + '.png'
image = open(filein, 'rb').read()
arguments = {
'width': width,
'height': height,
'pal_file': pal_file,
'tile_padding': tile_padding,
'pic_dimensions': pic_dimensions,
}
arguments.update(read_filename_arguments(filein))
if pal_file == None:
if os.path.exists(os.path.splitext(fileout)[0]+'.pal'):
arguments['pal_file'] = os.path.splitext(fileout)[0]+'.pal'
result = convert_2bpp_to_png(image, **arguments)
width, height, palette, greyscale, bitdepth, px_map = result
w = png.Writer(
width,
height,
palette=palette,
compression=9,
greyscale=greyscale,
bitdepth=bitdepth
)
with open(fileout, 'wb') as f:
w.write(f, px_map)
def convert_2bpp_to_png(image, **kwargs):
"""
Convert a planar 2bpp graphic to png.
"""
image = bytearray(image)
pad_color = bytearray([0])
width = kwargs.get('width', 0)
height = kwargs.get('height', 0)
tile_padding = kwargs.get('tile_padding', 0)
pic_dimensions = kwargs.get('pic_dimensions', None)
pal_file = kwargs.get('pal_file', None)
interleave = kwargs.get('interleave', False)
# Width must be specified to interleave.
if interleave and width:
image = interleave_tiles(image, width / 8)
# Pad the image by a given number of tiles if asked.
image += pad_color * 0x10 * tile_padding
# Some images are transposed in blocks.
if pic_dimensions:
w, h = pic_dimensions
if not width: width = w * 8
pic_length = w * h * 0x10
trailing = len(image) % pic_length
pic = []
for i in xrange(0, len(image) - trailing, pic_length):
pic += transpose_tiles(image[i:i+pic_length], h)
image = bytearray(pic) + image[len(image) - trailing:]
# Pad out trailing lines.
image += pad_color * 0x10 * ((w - (len(image) / 0x10) % h) % w)
def px_length(img):
return len(img) * 4
def tile_length(img):
return len(img) * 4 / (8*8)
if width and height:
tile_width = width / 8
more_tile_padding = (tile_width - (tile_length(image) % tile_width or tile_width))
image += pad_color * 0x10 * more_tile_padding
elif width and not height:
tile_width = width / 8
more_tile_padding = (tile_width - (tile_length(image) % tile_width or tile_width))
image += pad_color * 0x10 * more_tile_padding
height = px_length(image) / width
elif height and not width:
tile_height = height / 8
more_tile_padding = (tile_height - (tile_length(image) % tile_height or tile_height))
image += pad_color * 0x10 * more_tile_padding
width = px_length(image) / height
# at least one dimension should be given
if width * height != px_length(image):
# look for possible combos of width/height that would form a rectangle
matches = []
# Height need not be divisible by 8, but width must.
# See pokered gfx/minimize_pic.1bpp.
for w in range(8, px_length(image) / 2 + 1, 8):
h = px_length(image) / w
if w * h == px_length(image):
matches += [(w, h)]
# go for the most square image
if len(matches):
width, height = sorted(matches, key= lambda (w, h): (h % 8 != 0, w + h))[0] # favor height
else:
raise Exception, 'Image can\'t be divided into tiles (%d px)!' % (px_length(image))
# convert tiles to lines
lines = to_lines(flatten(image), width)
if pal_file == None:
palette = None
greyscale = True
bitdepth = 2
px_map = [[3 - pixel for pixel in line] for line in lines]
else: # gbc color
palette = pal_to_png(pal_file)
greyscale = False
bitdepth = 8
px_map = [[pixel for pixel in line] for line in lines]
return width, height, palette, greyscale, bitdepth, px_map
def get_pic_animation(tmap, w, h):
"""
Generate pic animation data from a combined tilemap of each frame.
"""
frame_text = ''
bitmask_text = ''
frames = list(split(tmap, w * h))
base = frames.pop(0)
bitmasks = []
for i in xrange(len(frames)):
frame_text += '\tdw .frame{}\n'.format(i + 1)
for i, frame in enumerate(frames):
bitmask = map(operator.ne, frame, base)
if bitmask not in bitmasks:
bitmasks.append(bitmask)
which_bitmask = bitmasks.index(bitmask)
mask = iter(bitmask)
masked_frame = filter(lambda _: mask.next(), frame)
frame_text += '.frame{}\n'.format(i + 1)
frame_text += '\tdb ${:02x} ; bitmask\n'.format(which_bitmask)
if masked_frame:
frame_text += '\tdb {}\n'.format(', '.join(
map('${:02x}'.format, masked_frame)
))
for i, bitmask in enumerate(bitmasks):
bitmask_text += '; {}\n'.format(i)
for byte in split(bitmask, 8):
byte = int(''.join(map(int.__repr__, reversed(byte))), 2)
bitmask_text += '\tdb %{:08b}\n'.format(byte)
return frame_text, bitmask_text
def export_png_to_2bpp(filein, fileout=None, palout=None, **kwargs):
arguments = {
'tile_padding': 0,
'pic_dimensions': None,
'animate': False,
'stupid_bitmask_hack': [],
}
arguments.update(kwargs)
arguments.update(read_filename_arguments(filein))
image, arguments = png_to_2bpp(filein, **arguments)
if fileout == None:
fileout = os.path.splitext(filein)[0] + '.2bpp'
to_file(fileout, image)
tmap = arguments.get('tmap')
if tmap != None and arguments['animate'] and arguments['pic_dimensions']:
# Generate pic animation data.
frame_text, bitmask_text = get_pic_animation(tmap, *arguments['pic_dimensions'])
frames_path = os.path.join(os.path.split(fileout)[0], 'frames.asm')
with open(frames_path, 'w') as out:
out.write(frame_text)
bitmask_path = os.path.join(os.path.split(fileout)[0], 'bitmask.asm')
# The following Pokemon have a bitmask dummied out.
for exception in arguments['stupid_bitmask_hack']:
if exception in bitmask_path:
bitmasks = bitmask_text.split(';')
bitmasks[-1] = bitmasks[-1].replace('1', '0')
bitmask_text = ';'.join(bitmasks)
with open(bitmask_path, 'w') as out:
out.write(bitmask_text)
elif tmap != None and arguments.get('tilemap', False):
tilemap_path = os.path.splitext(fileout)[0] + '.tilemap'
to_file(tilemap_path, tmap)
palette = arguments.get('palette')
if palout == None:
palout = os.path.splitext(fileout)[0] + '.pal'
export_palette(palette, palout)
def get_image_padding(width, height, wstep=8, hstep=8):
padding = {
'left': 0,
'right': 0,
'top': 0,
'bottom': 0,
}
if width % wstep and width >= wstep:
pad = float(width % wstep) / 2
padding['left'] = int(ceil(pad))
padding['right'] = int(floor(pad))
if height % hstep and height >= hstep:
pad = float(height % hstep) / 2
padding['top'] = int(ceil(pad))
padding['bottom'] = int(floor(pad))
return padding
def png_to_2bpp(filein, **kwargs):
"""
Convert a png image to planar 2bpp.
"""
arguments = {
'tile_padding': 0,
'pic_dimensions': False,
'interleave': False,
'norepeat': False,
'tilemap': False,
}
arguments.update(kwargs)
if type(filein) is str:
filein = open(filein)
assert type(filein) is file
width, height, rgba, info = png.Reader(filein).asRGBA8()
# png.Reader returns flat pixel data. Nested is easier to work with
len_px = len('rgba')
image = []
palette = []
for line in rgba:
newline = []
for px in xrange(0, len(line), len_px):
color = dict(zip('rgba', line[px:px+len_px]))
if color not in palette:
if len(palette) < 4:
palette += [color]
else:
# TODO Find the nearest match
print 'WARNING: %s: Color %s truncated to' % (filein, color),
color = sorted(palette, key=lambda x: sum(x.values()))[0]
print color
newline += [color]
image += [newline]
assert len(palette) <= 4, '%s: palette should be 4 colors, is really %d (%s)' % (filein, len(palette), palette)
# Pad out smaller palettes with greyscale colors
greyscale = {
'black': { 'r': 0x00, 'g': 0x00, 'b': 0x00, 'a': 0xff },
'grey': { 'r': 0x55, 'g': 0x55, 'b': 0x55, 'a': 0xff },
'gray': { 'r': 0xaa, 'g': 0xaa, 'b': 0xaa, 'a': 0xff },
'white': { 'r': 0xff, 'g': 0xff, 'b': 0xff, 'a': 0xff },
}
preference = 'white', 'black', 'grey', 'gray'
for hue in map(greyscale.get, preference):
if len(palette) >= 4:
break
if hue not in palette:
palette += [hue]
palette.sort(key=lambda x: sum(x.values()))
# Game Boy palette order
palette.reverse()
# Map pixels to quaternary color ids
padding = get_image_padding(width, height)
width += padding['left'] + padding['right']
height += padding['top'] + padding['bottom']
pad = bytearray([0])
qmap = []
qmap += pad * width * padding['top']
for line in image:
qmap += pad * padding['left']
for color in line:
qmap += [palette.index(color)]
qmap += pad * padding['right']
qmap += pad * width * padding['bottom']
# Graphics are stored in tiles instead of lines
tile_width = 8
tile_height = 8
num_columns = max(width, tile_width) / tile_width
num_rows = max(height, tile_height) / tile_height
image = []
for row in xrange(num_rows):
for column in xrange(num_columns):
# Split it up into strips to convert to planar data
for strip in xrange(min(tile_height, height)):
anchor = (
row * num_columns * tile_width * tile_height +
column * tile_width +
strip * width
)
line = qmap[anchor : anchor + tile_width]
bottom, top = 0, 0
for bit, quad in enumerate(line):
bottom += (quad & 1) << (7 - bit)
top += (quad /2 & 1) << (7 - bit)
image += [bottom, top]
dim = arguments['pic_dimensions']
if dim:
if type(dim) in (tuple, list):
w, h = dim
else:
# infer dimensions based on width.
w = width / tile_width
h = height / tile_height
if h % w == 0:
h = w
tiles = get_tiles(image)
pic_length = w * h
tile_width = width / 8
trailing = len(tiles) % pic_length
new_image = []
for block in xrange(len(tiles) / pic_length):
offset = (h * tile_width) * ((block * w) / tile_width) + ((block * w) % tile_width)
pic = []
for row in xrange(h):
index = offset + (row * tile_width)
pic += tiles[index:index + w]
new_image += transpose(pic, w)
new_image += tiles[len(tiles) - trailing:]
image = connect(new_image)
# Remove any tile padding used to make the png rectangular.
image = image[:len(image) - arguments['tile_padding'] * 0x10]
tmap = None
if arguments['interleave']:
image = deinterleave_tiles(image, num_columns)
if arguments['pic_dimensions']:
image, tmap = condense_image_to_map(image, w * h)
elif arguments['norepeat']:
image, tmap = condense_image_to_map(image)
if not arguments['tilemap']:
tmap = None
arguments.update({ 'palette': palette, 'tmap': tmap, })
return image, arguments
def export_palette(palette, filename):
"""
Export a palette from png to rgb macros in a .pal file.
"""
if os.path.exists(filename):
# Pic palettes are 2 colors (black/white are added later).
with open(filename) as rgbs:
colors = read_rgb_macros(rgbs.readlines())
if len(colors) == 2:
palette = palette[1:3]
text = png_to_rgb(palette)
with open(filename, 'w') as out:
out.write(text)
def png_to_lz(filein):
name = os.path.splitext(filein)[0]
export_png_to_2bpp(filein)
image = open(name+'.2bpp', 'rb').read()
to_file(name+'.2bpp'+'.lz', Compressed(image).output)
def convert_2bpp_to_1bpp(data):
"""
Convert planar 2bpp image data to 1bpp. Assume images are two colors.
"""
return data[::2]
def convert_1bpp_to_2bpp(data):
"""
Convert 1bpp image data to planar 2bpp (black/white).
"""
output = []
for i in data:
output += [i, i]
return output
def export_2bpp_to_1bpp(filename):
name, extension = os.path.splitext(filename)
image = open(filename, 'rb').read()
image = convert_2bpp_to_1bpp(image)
to_file(name + '.1bpp', image)
def export_1bpp_to_2bpp(filename):
name, extension = os.path.splitext(filename)
image = open(filename, 'rb').read()
image = convert_1bpp_to_2bpp(image)
to_file(name + '.2bpp', image)
def export_1bpp_to_png(filename, fileout=None):
if fileout == None:
fileout = os.path.splitext(filename)[0] + '.png'
arguments = read_filename_arguments(filename)
image = open(filename, 'rb').read()
image = convert_1bpp_to_2bpp(image)
result = convert_2bpp_to_png(image, **arguments)
width, height, palette, greyscale, bitdepth, px_map = result
w = png.Writer(width, height, palette=palette, compression=9, greyscale=greyscale, bitdepth=bitdepth)
with open(fileout, 'wb') as f:
w.write(f, px_map)
def export_png_to_1bpp(filename, fileout=None):
if fileout == None:
fileout = os.path.splitext(filename)[0] + '.1bpp'
arguments = read_filename_arguments(filename)
image = png_to_1bpp(filename, **arguments)
to_file(fileout, image)
def png_to_1bpp(filename, **kwargs):
image, kwargs = png_to_2bpp(filename, **kwargs)
return convert_2bpp_to_1bpp(image)
def convert_to_2bpp(filenames=[]):
for filename in filenames:
filename, name, extension = try_decompress(filename)
if extension == '.1bpp':
export_1bpp_to_2bpp(filename)
elif extension == '.2bpp':
pass
elif extension == '.png':
export_png_to_2bpp(filename)
else:
raise Exception, "Don't know how to convert {} to 2bpp!".format(filename)
def convert_to_1bpp(filenames=[]):
for filename in filenames:
filename, name, extension = try_decompress(filename)
if extension == '.1bpp':
pass
elif extension == '.2bpp':
export_2bpp_to_1bpp(filename)
elif extension == '.png':
export_png_to_1bpp(filename)
else:
raise Exception, "Don't know how to convert {} to 1bpp!".format(filename)
def convert_to_png(filenames=[]):
for filename in filenames:
filename, name, extension = try_decompress(filename)
if extension == '.1bpp':
export_1bpp_to_png(filename)
elif extension == '.2bpp':
export_2bpp_to_png(filename)
elif extension == '.png':
pass
else:
raise Exception, "Don't know how to convert {} to png!".format(filename)
def compress(filenames=[]):
for filename in filenames:
data = open(filename, 'rb').read()
lz_data = Compressed(data).output
to_file(filename + '.lz', lz_data)
def decompress(filenames=[]):
for filename in filenames:
name, extension = os.path.splitext(filename)
lz_data = open(filename, 'rb').read()
data = Decompressed(lz_data).output
to_file(name, data)
def try_decompress(filename):
"""
Try to decompress a graphic when determining the filetype.
This skips the manual unlz step when attempting
to convert lz-compressed graphics to png.
"""
name, extension = os.path.splitext(filename)
if extension == '.lz':
decompress([filename])
filename = name
name, extension = os.path.splitext(filename)
return filename, name, extension
def main():
ap = argparse.ArgumentParser()
ap.add_argument('mode')
ap.add_argument('filenames', nargs='*')
args = ap.parse_args()
method = {
'2bpp': convert_to_2bpp,
'1bpp': convert_to_1bpp,
'png': convert_to_png,
'lz': compress,
'unlz': decompress,
}.get(args.mode, None)
if method == None:
raise Exception, "Unknown conversion method!"
method(args.filenames)
if __name__ == "__main__":
main()

580
tools/pokemontools/lz.py Normal file
View File

@ -0,0 +1,580 @@
# -*- coding: utf-8 -*-
"""
Pokemon Crystal data de/compression.
"""
"""
A rundown of Pokemon Crystal's compression scheme:
Control commands occupy bits 5-7.
Bits 0-4 serve as the first parameter <n> for each command.
"""
lz_commands = {
'literal': 0, # n values for n bytes
'iterate': 1, # one value for n bytes
'alternate': 2, # alternate two values for n bytes
'blank': 3, # zero for n bytes
}
"""
Repeater commands repeat any data that was just decompressed.
They take an additional signed parameter <s> to mark a relative starting point.
These wrap around (positive from the start, negative from the current position).
"""
lz_commands.update({
'repeat': 4, # n bytes starting from s
'flip': 5, # n bytes in reverse bit order starting from s
'reverse': 6, # n bytes backwards starting from s
})
"""
The long command is used when 5 bits aren't enough. Bits 2-4 contain a new control code.
Bits 0-1 are appended to a new byte as 8-9, allowing a 10-bit parameter.
"""
lz_commands.update({
'long': 7, # n is now 10 bits for a new control code
})
max_length = 1 << 10 # can't go higher than 10 bits
lowmax = 1 << 5 # standard 5-bit param
"""
If 0xff is encountered instead of a command, decompression ends.
"""
lz_end = 0xff
bit_flipped = [
sum(((byte >> i) & 1) << (7 - i) for i in xrange(8))
for byte in xrange(0x100)
]
class Compressed:
"""
Usage:
lz = Compressed(data).output
or
lz = Compressed().compress(data)
or
c = Compressed()
c.data = data
lz = c.compress()
There are some issues with reproducing the target compressor.
Some notes are listed here:
- the criteria for detecting a lookback is inconsistent
- sometimes lookbacks that are mostly 0s are pruned, sometimes not
- target appears to skip ahead if it can use a lookback soon, stopping the current command short or in some cases truncating it with literals.
- this has been implemented, but the specifics are unknown
- self.min_scores: It's unknown if blank's minimum score should be 1 or 2. Most likely it's 1, with some other hack to account for edge cases.
- may be related to the above
- target does not appear to compress backwards
"""
def __init__(self, *args, **kwargs):
self.min_scores = {
'blank': 1,
'iterate': 2,
'alternate': 3,
'repeat': 3,
'reverse': 3,
'flip': 3,
}
self.preference = [
'repeat',
'blank',
'flip',
'reverse',
'iterate',
'alternate',
#'literal',
]
self.lookback_methods = 'repeat', 'reverse', 'flip'
self.__dict__.update({
'data': None,
'commands': lz_commands,
'debug': False,
'literal_only': False,
})
self.arg_names = 'data', 'commands', 'debug', 'literal_only'
self.__dict__.update(kwargs)
self.__dict__.update(dict(zip(self.arg_names, args)))
if self.data is not None:
self.compress()
def compress(self, data=None):
if data is not None:
self.data = data
self.data = list(bytearray(self.data))
self.indexes = {}
self.lookbacks = {}
for method in self.lookback_methods:
self.lookbacks[method] = {}
self.address = 0
self.end = len(self.data)
self.output = []
self.literal = None
while self.address < self.end:
if self.score():
self.do_literal()
self.do_winner()
else:
if self.literal == None:
self.literal = self.address
self.address += 1
self.do_literal()
self.output += [lz_end]
return self.output
def reset_scores(self):
self.scores = {}
self.offsets = {}
self.helpers = {}
for method in self.min_scores.iterkeys():
self.scores[method] = 0
def bit_flip(self, byte):
return bit_flipped[byte]
def do_literal(self):
if self.literal != None:
length = abs(self.address - self.literal)
start = min(self.literal, self.address + 1)
self.helpers['literal'] = self.data[start:start+length]
self.do_cmd('literal', length)
self.literal = None
def score(self):
self.reset_scores()
map(self.score_literal, ['iterate', 'alternate', 'blank'])
for method in self.lookback_methods:
self.scores[method], self.offsets[method] = self.find_lookback(method, self.address)
self.stop_short()
return any(
score
> self.min_scores[method] + int(score > lowmax)
for method, score in self.scores.iteritems()
)
def stop_short(self):
"""
If a lookback is close, reduce the scores of other commands.
"""
best_method, best_score = max(
self.scores.items(),
key = lambda x: (
x[1],
-self.preference.index(x[0])
)
)
for method in self.lookback_methods:
min_score = self.min_scores[method]
for address in xrange(self.address+1, self.address+best_score):
length, index = self.find_lookback(method, address)
if length > max(min_score, best_score):
# BUG: lookbacks can reduce themselves. This appears to be a bug in the target also.
for m, score in self.scores.items():
self.scores[m] = min(score, address - self.address)
def read(self, address=None):
if address is None:
address = self.address
if 0 <= address < len(self.data):
return self.data[address]
return None
def find_all_lookbacks(self):
for method in self.lookback_methods:
for address, byte in enumerate(self.data):
self.find_lookback(method, address)
def find_lookback(self, method, address=None):
"""Temporarily stubbed, because the real function doesn't run in polynomial time."""
return 0, None
def broken_find_lookback(self, method, address=None):
if address is None:
address = self.address
existing = self.lookbacks.get(method, {}).get(address)
if existing != None:
return existing
lookback = 0, None
# Better to not carelessly optimize at the moment.
"""
if address < 2:
return lookback
"""
byte = self.read(address)
if byte is None:
return lookback
direction, mutate = {
'repeat': ( 1, int),
'reverse': (-1, int),
'flip': ( 1, self.bit_flip),
}[method]
# Doesn't seem to help
"""
if mutate == self.bit_flip:
if byte == 0:
self.lookbacks[method][address] = lookback
return lookback
"""
data_len = len(self.data)
is_two_byte_index = lambda index: int(index < address - 0x7f)
for index in self.get_indexes(mutate(byte)):
if index >= address:
break
old_length, old_index = lookback
if direction == 1:
if old_length > data_len - index: break
else:
if old_length > index: continue
if self.read(index) in [None]: continue
length = 1 # we know there's at least one match, or we wouldn't be checking this index
while 1:
this_byte = self.read(address + length)
that_byte = self.read(index + length * direction)
if that_byte == None or this_byte != mutate(that_byte):
break
length += 1
score = length - is_two_byte_index(index)
old_score = old_length - is_two_byte_index(old_index)
if score >= old_score or (score == old_score and length > old_length):
# XXX maybe avoid two-byte indexes when possible
if score >= lookback[0] - is_two_byte_index(lookback[1]):
lookback = length, index
self.lookbacks[method][address] = lookback
return lookback
def get_indexes(self, byte):
if not self.indexes.has_key(byte):
self.indexes[byte] = []
index = -1
while 1:
try:
index = self.data.index(byte, index + 1)
except ValueError:
break
self.indexes[byte].append(index)
return self.indexes[byte]
def score_literal(self, method):
address = self.address
compare = {
'blank': [0],
'iterate': [self.read(address)],
'alternate': [self.read(address), self.read(address + 1)],
}[method]
# XXX may or may not be correct
if method == 'alternate' and compare[0] == 0:
return
length = 0
while self.read(address + length) == compare[length % len(compare)]:
length += 1
self.scores[method] = length
self.helpers[method] = compare
def do_winner(self):
winners = filter(
lambda (method, score):
score
> self.min_scores[method] + int(score > lowmax),
self.scores.iteritems()
)
winners.sort(
key = lambda (method, score): (
-(score - self.min_scores[method] - int(score > lowmax)),
self.preference.index(method)
)
)
winner, score = winners[0]
length = min(score, max_length)
self.do_cmd(winner, length)
self.address += length
def do_cmd(self, cmd, length):
start_address = self.address
cmd_length = length - 1
output = []
if length > lowmax:
output.append(
(self.commands['long'] << 5)
+ (self.commands[cmd] << 2)
+ (cmd_length >> 8)
)
output.append(
cmd_length & 0xff
)
else:
output.append(
(self.commands[cmd] << 5)
+ cmd_length
)
self.helpers['blank'] = [] # quick hack
output += self.helpers.get(cmd, [])
if cmd in self.lookback_methods:
offset = self.offsets[cmd]
# Negative offsets are one byte.
# Positive offsets are two.
if 0 < start_address - offset - 1 <= 0x7f:
offset = (start_address - offset - 1) | 0x80
output += [offset]
else:
output += [offset / 0x100, offset % 0x100] # big endian
if self.debug:
print ' '.join(map(str, [
cmd, length, '\t',
' '.join(map('{:02x}'.format, output)),
self.data[start_address:start_address+length] if cmd in self.lookback_methods else '',
]))
self.output += output
class Decompressed:
"""
Interpret and decompress lz-compressed data, usually 2bpp.
"""
"""
Usage:
data = Decompressed(lz).output
or
data = Decompressed().decompress(lz)
or
d = Decompressed()
d.lz = lz
data = d.decompress()
To decompress from offset 0x80000 in a rom:
data = Decompressed(rom, start=0x80000).output
"""
lz = None
start = 0
commands = lz_commands
debug = False
arg_names = 'lz', 'start', 'commands', 'debug'
def __init__(self, *args, **kwargs):
self.__dict__.update(dict(zip(self.arg_names, args)))
self.__dict__.update(kwargs)
self.command_names = dict(map(reversed, self.commands.items()))
self.address = self.start
if self.lz is not None:
self.decompress()
if self.debug: print self.command_list()
def command_list(self):
"""
Print a list of commands that were used. Useful for debugging.
"""
text = ''
output_address = 0
for name, attrs in self.used_commands:
length = attrs['length']
address = attrs['address']
offset = attrs['offset']
direction = attrs['direction']
text += '{2:03x} {0}: {1}'.format(name, length, output_address)
text += '\t' + ' '.join(
'{:02x}'.format(int(byte))
for byte in self.lz[ address : address + attrs['cmd_length'] ]
)
if offset is not None:
repeated_data = self.output[ offset : offset + length * direction : direction ]
if name == 'flip':
repeated_data = map(bit_flipped.__getitem__, repeated_data)
text += ' [' + ' '.join(map('{:02x}'.format, repeated_data)) + ']'
text += '\n'
output_address += length
return text
def decompress(self, lz=None):
if lz is not None:
self.lz = lz
self.lz = bytearray(self.lz)
self.used_commands = []
self.output = []
while 1:
cmd_address = self.address
self.offset = None
self.direction = None
if (self.byte == lz_end):
self.next()
break
self.cmd = (self.byte & 0b11100000) >> 5
if self.cmd_name == 'long':
# 10-bit length
self.cmd = (self.byte & 0b00011100) >> 2
self.length = (self.next() & 0b00000011) * 0x100
self.length += self.next() + 1
else:
# 5-bit length
self.length = (self.next() & 0b00011111) + 1
self.__class__.__dict__[self.cmd_name](self)
self.used_commands += [(
self.cmd_name,
{
'length': self.length,
'address': cmd_address,
'offset': self.offset,
'cmd_length': self.address - cmd_address,
'direction': self.direction,
}
)]
# Keep track of the data we just decompressed.
self.compressed_data = self.lz[self.start : self.address]
@property
def byte(self):
return self.lz[ self.address ]
def next(self):
byte = self.byte
self.address += 1
return byte
@property
def cmd_name(self):
return self.command_names.get(self.cmd)
def get_offset(self):
if self.byte >= 0x80: # negative
# negative
offset = self.next() & 0x7f
offset = len(self.output) - offset - 1
else:
# positive
offset = self.next() * 0x100
offset += self.next()
self.offset = offset
def literal(self):
"""
Copy data directly.
"""
self.output += self.lz[ self.address : self.address + self.length ]
self.address += self.length
def iterate(self):
"""
Write one byte repeatedly.
"""
self.output += [self.next()] * self.length
def alternate(self):
"""
Write alternating bytes.
"""
alts = [self.next(), self.next()]
self.output += [ alts[x & 1] for x in xrange(self.length) ]
def blank(self):
"""
Write zeros.
"""
self.output += [0] * self.length
def flip(self):
"""
Repeat flipped bytes from output.
Example: 11100100 -> 00100111
"""
self._repeat(table=bit_flipped)
def reverse(self):
"""
Repeat reversed bytes from output.
"""
self._repeat(direction=-1)
def repeat(self):
"""
Repeat bytes from output.
"""
self._repeat()
def _repeat(self, direction=1, table=None):
self.get_offset()
self.direction = direction
# Note: appends must be one at a time (this way, repeats can draw from themselves if required)
for i in xrange(self.length):
byte = self.output[ self.offset + i * direction ]
self.output.append( table[byte] if table else byte )

2650
tools/pokemontools/png.py Normal file
View File

File diff suppressed because it is too large Load Diff