Merge branch 'master' into yenatch-master

preprocessor.py

@@ -16,6 +16,12 @@ from extras.pokemontools.crystal import (
     effect_classes,
 )
 
+def load_pokecrystal_macros():
+    """
+    Construct a list of macros that are needed for pokecrystal preprocessing.
+    """
+    ourmacros = []
+
     even_more_macros = [
         Warp,
         XYTrigger,
@@ -24,18 +30,14 @@ even_more_macros = [
         DataByteWordMacro,
     ]
 
-macros = command_classes
+    ourmacros += command_classes
-macros += even_more_macros
+    ourmacros += even_more_macros
-macros += [each[1] for each in text_command_classes]
+    ourmacros += [each[1] for each in text_command_classes]
-macros += movement_command_classes
+    ourmacros += movement_command_classes
-macros += music_classes
+    ourmacros += music_classes
-macros += effect_classes
+    ourmacros += effect_classes
 
-# show lines before preprocessing in stdout
+    return ourmacros
-show_original_lines = False
-
-# helpful for debugging macros
-do_macro_sanity_check = False
 
 chars = {
 "ガ": 0x05,
@@ -302,6 +304,16 @@ chars = {
 "9": 0xFF
 }
 
+class PreprocessorException(Exception):
+    """
+    There was a problem in the preprocessor.
+    """
+
+class MacroException(PreprocessorException):
+    """
+    There was a problem with a macro.
+    """
+
 def separate_comment(l):
     """
     Separates asm and comments on a single line.
@@ -415,11 +427,9 @@ def macro_test(asm, macro_table):
     token = extract_token(asm)
 
     # skip db and dw since rgbasm handles those and they aren't macros
-    if token not in ["db", "dw"]:
+    if token is not None and token not in ["db", "dw"] and token in macro_table:
-        # check against all names
-        if token in macro_table:
         return (macro_table[token], token)
-
+    else:
         return (None, None)
 
 def is_based_on(something, base):
@@ -434,12 +444,69 @@ def is_based_on(something, base):
     options += [something.__name__]
     return (base in options)
 
-def macro_translator(macro, token, line):
+def check_macro_sanity(params, macro, original_line):
+    """
+    Checks whether or not the correct number of arguments are being passed to a
+    certain macro. There are a number of possibilities based on the types of
+    parameters that define the macro.
+
+    @param params: a list of parameters given to the macro
+    @param macro: macro klass
+    @param original_line: the line being preprocessed
+    """
+    allowed_length = 0
+
+    for (index, param_type) in macro.param_types.items():
+        param_klass = param_type["class"]
+
+        if param_klass.byte_type == "db":
+            allowed_length += 1 # just one value
+        elif param_klass.byte_type == "dw":
+            if param_klass.size == 2:
+                allowed_length += 1 # just label
+            elif param_klass.size == 3:
+                allowed_length += 2 # bank and label
+            else:
+                raise MacroException(
+                    "dunno what to do with a macro param with a size > 3 (size={size})"
+                    .format(size=param_klass.size)
+                )
+        else:
+            raise MacroException(
+                "dunno what to do with this non db/dw macro param: {klass} in line {line}"
+                .format(klass=param_klass, line=original_line)
+            )
+
+    # sometimes the allowed length can vary
+    if hasattr(macro, "allowed_lengths"):
+        allowed_lengths = macro.allowed_lengths + [allowed_length]
+    else:
+        allowed_lengths = [allowed_length]
+
+    # used twice, so precompute once
+    params_len = len(params)
+
+    if params_len not in allowed_lengths:
+        raise PreprocessorException(
+            "mismatched number of parameters ({count}, instead of any of {allowed}) on this line: {line}"
+            .format(
+                count=params_len,
+                allowed=allowed_lengths,
+                line=original_line,
+            )
+        )
+
+    return True
+
+def macro_translator(macro, token, line, show_original_lines=False, do_macro_sanity_check=False):
     """
     Converts a line with a macro into a rgbasm-compatible line.
-    """
 
-    assert macro.macro_name == token, "macro/token mismatch"
+    @param show_original_lines: show lines before preprocessing in stdout
+    @param do_macro_sanity_check: helpful for debugging macros
+    """
+    if macro.macro_name != token:
+        raise MacroException("macro/token mismatch")
 
     original_line = line
 
@@ -467,7 +534,7 @@ def macro_translator(macro, token, line):
 
         # check if there are no params (redundant)
         if len(params) == 1 and params[0] == "":
-            raise Exception, "macro has no params?"
+            raise MacroException("macro has no params?")
 
     # write out a comment showing the original line
     if show_original_lines:
@@ -485,45 +552,10 @@ def macro_translator(macro, token, line):
     if not macro.override_byte_check:
         sys.stdout.write("db ${0:02X}\n".format(macro.id))
 
-    # --- long-winded sanity check goes here ---
+    # Does the number of parameters on this line match any allowed number of
-
+    # parameters that the macro expects?
     if do_macro_sanity_check:
-
+        check_macro_sanity(params, macro, original_line)
-        # sanity check... this won't work because PointerLabelBeforeBank shows
-        # up as two params, so these two lengths will always be different.
-        #assert len(params) == len(macro.param_types), \
-        #       "mismatched number of parameters on this line: " + \
-        #       original_line
-
-        # v2 sanity check :) although it sorta sucks that this loop happens twice?
-        allowed_length = 0
-        for (index, param_type) in macro.param_types.items():
-            param_klass = param_type["class"]
-
-            if param_klass.byte_type == "db":
-                allowed_length += 1 # just one value
-            elif param_klass.byte_type == "dw":
-                if param_klass.size == 2:
-                    allowed_length += 1 # just label
-                elif param_klass.size == 3:
-                    allowed_length += 2 # bank and label
-                else:
-                    raise Exception, "dunno what to do with a macro param with a size > 3"
-            else:
-                raise Exception, "dunno what to do with this non db/dw macro param: " + \
-                                 str(param_klass) + " in line: " + original_line
-
-        # sometimes the allowed length can vary
-        if hasattr(macro, "allowed_lengths"):
-            allowed_lengths = macro.allowed_lengths + [allowed_length]
-        else:
-            allowed_lengths = [allowed_length]
-
-        assert len(params) in allowed_lengths, \
-               "mismatched number of parameters on this line: " + \
-               original_line
-
-    # --- end of ridiculously long sanity check ---
 
     # used for storetext
     correction = 0
@@ -532,10 +564,7 @@ def macro_translator(macro, token, line):
 
     index = 0
     while index < len(params):
-        try:
         param_type  = macro.param_types[index - correction]
-        except KeyError as exception:
-            raise Exception("line is: " + str(line) + " and macro is: " + str(macro))
         description = param_type["name"]
         param_klass = param_type["class"]
         byte_type   = param_klass.byte_type # db or dw
@@ -569,9 +598,13 @@ def macro_translator(macro, token, line):
                 output += ("db " + param_klass.from_asm(param) + "\n")
                 index += 1
             else:
-                raise Exception, "dunno what to do with this macro " + \
+                raise MacroException(
-                "param (" + str(param_klass) + ") " + "on this line: " + \
+                    "dunno what to do with this macro param ({klass}) in line: {line}"
-                original_line
+                    .format(
+                        klass=param_klass,
+                        line=original_line,
+                    )
+                )
 
         # or just print out the byte
         else:
@@ -584,6 +617,10 @@ def macro_translator(macro, token, line):
 def read_line(l, macro_table):
     """Preprocesses a given line of asm."""
 
+    if l in ["\n", ""] or l[0] == ";":
+        sys.stdout.write(l)
+        return # jump out early
+
     # strip comments from asm
     asm, comment = separate_comment(l)
 
@@ -597,7 +634,7 @@ def read_line(l, macro_table):
         sys.stdout.write(asm)
 
     # ascii string macro preserves the bytes as ascii (skip the translator)
-    elif len(asm) > 6 and "ascii " == asm[:6] or "\tascii " == asm[:7]:
+    elif len(asm) > 6 and ("ascii " == asm[:6] or "\tascii " == asm[:7]):
         asm = asm.replace("ascii", "db", 1)
         sys.stdout.write(asm)
 
@@ -613,11 +650,11 @@ def read_line(l, macro_table):
         else:
             sys.stdout.write(asm)
 
-    if comment: sys.stdout.write(comment)
+    if comment:
+        sys.stdout.write(comment)
 
-def preprocess(macros, lines=None):
+def preprocess(macro_table, lines=None):
     """Main entry point for the preprocessor."""
-    macro_table = make_macro_table(macros)
 
     if not lines:
         # read each line from stdin
@@ -629,6 +666,11 @@ def preprocess(macros, lines=None):
     for l in lines:
         read_line(l, macro_table)
 
+def main():
+    macros = load_pokecrystal_macros()
+    macro_table = make_macro_table(macros)
+    preprocess(macro_table)
+
 # only run against stdin when not included as a module
 if __name__ == "__main__":
-    preprocess(macros)
+    main()

prequeue.py

@@ -1,16 +1,28 @@
 # coding: utf-8
-
+"""
-# Starting a new python process to preprocess each source file
+Starting a new python process to preprocess each source file creates too much
-# creates too much overhead. Instead, a list of files to preprocess
+overhead. Instead, a list of files to preprocess is fed into a script run from
-# is fed into a script run from a single process.
+a single process.
+"""
 
 import os
 import sys
 import preprocessor
 
-if __name__ == '__main__':
+def main():
+    macros = preprocessor.load_pokecrystal_macros()
+    macro_table = preprocessor.make_macro_table(macros)
+
+    stdout = sys.stdout
+
     for source in sys.argv[1:]:
         dest = os.path.splitext(source)[0] + '.tx'
         sys.stdin  = open(source, 'r')
         sys.stdout = open(dest, 'w')
-        preprocessor.preprocess(preprocessor.macros)
+        preprocessor.preprocess(macro_table)
+
+    # reset stdout
+    sys.stdout = stdout
+
+if __name__ == '__main__':
+    main()