* Fix multiline directives

* Add macros
* Add operator parsing

CLexer.cpp

@@ -6,25 +6,57 @@ const std::string numbers = "0123456789";
 const std::string identifierStart = "_abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
 const std::string identifierBody = "_abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
 const std::string hexnumbers = "0123456789abcdefABCDEF";
+const std::string binaryNumbers = "01";
+const std::vector<std::string> keywords =
+{
+    "and", "abstract", "auto", "bool", "break",
+    "case", "cast", "class", "const", "continue",
+    "default", "do", "double", "else", "enum",
+    "false", "final", "float", "for", "from",
+    "funcdef", "get", "if", "import", "in",
+    "inout", "int", "interface", "int8",
+    "int16", "int32", "int64", "is",
+    "mixin", "namespace", "not", "null",
+    "or", "out", "override", "private",
+    "protected", "return", "set", "shared",
+    "super", "switch", "this", "true", "typedef",
+    "uint", "uint8", "uint16", "uint32", "uint64",
+    "void", "while", "xor"
+};
+
 const std::string trivials = ",;\n\r\t [{(]})";
+const std::string opStart = "*/%+-<=>!?:^&>@|~.";
+
+const std::vector<std::string> operators=
+{
+    "*","**","/","%","+","-","<=",">"">=","==",
+    "!=","?","+=","-=","*=","/=","%=","**=","++",
+    "--","&","|","~","^","<<",">>",">>>","<<=",
+    ">>=",">>>=",".","||","!","^^","::","="
+};
 
 const CLexer::TokenType TrivialTypes[] =
 {
-	CLexer::COMMA,
-	CLexer::SEMICOLON,
-	CLexer::NEWLINE,
-	CLexer::WHITESPACE,
-	CLexer::WHITESPACE,
-	CLexer::WHITESPACE,
-	CLexer::OPEN,
-	CLexer::OPEN,
-	CLexer::OPEN,
-	CLexer::CLOSE,
-	CLexer::CLOSE,
-	CLexer::CLOSE
+    CLexer::COMMA,
+    CLexer::SEMICOLON,
+    CLexer::NEWLINE,
+    CLexer::WHITESPACE,
+    CLexer::WHITESPACE,
+    CLexer::WHITESPACE,
+    CLexer::OPEN,
+    CLexer::OPEN,
+    CLexer::OPEN,
+    CLexer::CLOSE,
+    CLexer::CLOSE,
+    CLexer::CLOSE
 };
 
-int CLexer::lex(char* start, char* end, TokenList& tokens)
+CLexer::CLexer()
+    : m_lastIdentifier(nullptr)
+{
+}
+
+void CLexer::lex(char* start, char* end, TokenList& tokens)
 {
     assert(start != 0 && end != 0 && "start and end cannot be null");
     assert(start <= end && "degenerate lex detected: end < start");
@@ -33,10 +65,19 @@ int CLexer::lex(char* start, char* end, TokenList& tokens)
     {
         CLexer::Token currentToken;
         start = _parseToken(start, end, currentToken);
-        //if (currentToken.type != CLexer::COMMENT)
-        tokens.push_back(currentToken);
+        if (currentToken.type != CLexer::INVALID)
+            tokens.push_back(currentToken);
+
+        if (currentToken.value != "#include")
+        {
+            if ((currentToken.type == CLexer::IDENTIFIER || currentToken.type == CLexer::PREPROCESSOR) && !m_lastIdentifier)
+                m_lastIdentifier = &tokens.back();
+        }
+        else
+            m_lastIdentifier = nullptr;
+
         if (start == end)
-            return 0;
+            break;
     }
 }
 
@@ -65,25 +106,76 @@ bool CLexer::_isNumber(char in) const
     return _searchString(numbers, in);
 }
 
+bool CLexer::_isBinary(char in) const
+{
+    return _searchString(binaryNumbers, in);
+}
+
 bool CLexer::_isHex(char in) const
 {
     return _searchString(hexnumbers, in);
 }
 
+bool CLexer::_isOperatorStart(char in) const
+{
+    return _searchString(opStart, in);
+}
+
+bool CLexer::_isOperator(const std::string& val) const
+{
+    auto it = std::find(operators.begin(), operators.end(), val);
+    return (it != operators.end());
+}
+
+bool CLexer::_isKeyword(const std::string& val) const
+{
+    auto it = std::find(keywords.begin(), keywords.end(), val);
+    return (it != keywords.end());
+}
+
 char* CLexer::_parseToken(char* start, char* end, CLexer::Token& out)
 {
     if (start == end)
         return start;
     char curChar = *start;
+
     if (_isTrivial(curChar))
     {
         out.value += curChar;
         out.type = TrivialTypes[trivials.find_first_of(curChar)];
+        if (out.value == "(" && m_lastIdentifier)
+        {
+            if (m_lastIdentifier->type == CLexer::IDENTIFIER)
+                m_lastIdentifier->type = CLexer::FUNCTION;
+            else if (m_lastIdentifier->type == CLexer::PREPROCESSOR && m_lastIdentifier->value == "#define")
+                m_lastIdentifier->type = CLexer::MACRO;
+        }
+        else if (out.value == "\n")
+            m_lastIdentifier = nullptr;
+
         return ++start;
     }
 
+    if (m_lastIdentifier && curChar == '\\')
+    {
+        if ((m_lastIdentifier->type == CLexer::PREPROCESSOR || m_lastIdentifier->type == CLexer::MACRO))
+        {
+            // only handle this if we're working on a preprocessor or a macro
+            while(*start != '\n')
+                ++start;
+            *start = ' ';
+            return start;
+        }
+    }
+
     if (_isIdentifierStart(curChar))
-        return _parseIdentifier(start, end, out);
+    {
+        start = _parseIdentifier(start, end, out);
+        if (_isKeyword(out.value))
+            out.type = CLexer::KEYWORD;
+
+        return start;
+    }
 
     if (curChar == '#')
     {
@@ -92,7 +184,6 @@ char* CLexer::_parseToken(char* start, char* end, CLexer::Token& out)
         return start;
     }
 
-
     if (_isNumber(curChar))
         return _parseNumber(start, end, out);
     if (curChar == '\"')
@@ -111,6 +202,8 @@ char* CLexer::_parseToken(char* start, char* end, CLexer::Token& out)
             return _parseLineComment(start, end, out);
         --start;
     }
+    if (_isOperatorStart(curChar))
+        return _parseOperator(start, end, out);
 
     // Nothing intradasting
     out.value = curChar;
@@ -138,6 +231,7 @@ char* CLexer::_parseBlockComment(char* start, char* end, CLexer::Token& out)
 {
     out.type = CLexer::COMMENT;
     out.value += "/*";
+    bool opened = true;
 
     while (true)
     {
@@ -155,6 +249,7 @@ char* CLexer::_parseBlockComment(char* start, char* end, CLexer::Token& out)
             {
                 out.value += *start;
                 ++start;
+                opened = false;
                 break;
             }
             else
@@ -162,6 +257,7 @@ char* CLexer::_parseBlockComment(char* start, char* end, CLexer::Token& out)
         }
     }
 
+    out.degenerate = opened;
     return start;
 }
 
@@ -179,7 +275,26 @@ char* CLexer::_parseNumber(char* start, char* end, CLexer::Token& out)
         else if (*start == '.')
             return _parseFloatingPoint(start, end, out);
         else if (*start == 'x')
-            return _parseFloatingPoint(start, end, out);
+            return _parseHexConstant(start, end, out);
+        else if (*start == 'd')
+            out.value += *start;
+        else if (*start == 'b')
+            return _parseBinaryConstant(start, end, out);
+        else
+            return start;
+    }
+}
+
+char* CLexer::_parseBinaryConstant(char* start, char* end, CLexer::Token& out)
+{
+    out.value += *start;
+    while (true)
+    {
+        ++start;
+        if (start == end)
+            return start;
+        if (_isBinary(*start))
+            out.value += *start;
         else
             return start;
     }
@@ -203,6 +318,7 @@ char* CLexer::_parseHexConstant(char* start, char* end, CLexer::Token& out)
 char* CLexer::_parseFloatingPoint(char* start, char* end, CLexer::Token& out)
 {
     out.value += *start;
+    bool hasExponent = false;
     while (true)
     {
         ++start;
@@ -210,7 +326,14 @@ char* CLexer::_parseFloatingPoint(char* start, char* end, CLexer::Token& out)
             return start;
         if (!_isNumber(*start))
         {
-            if (*start == 'f' || *start == 'F')
+            if (*start == 'e')
+            {
+                out.degenerate = hasExponent;
+                out.value += *start;
+                hasExponent = true;
+                continue;
+            }
+            else if (*start == 'f')
             {
                 out.value += *start;
                 ++start;
@@ -298,4 +421,29 @@ char* CLexer::_parseIdentifier(char* start, char* end, CLexer::Token& out)
     }
 }
 
+char* CLexer::_parseOperator(char* start, char* end, CLexer::Token& out)
+{
+    out.type = CLexer::OPERATOR;
+    std::string operatorValue;
+
+    while (true)
+    {
+        if (start == end)
+        {
+            out.type = CLexer::INVALID;
+            return start;
+        }
+        if (_isOperatorStart(*start))
+            operatorValue += *start;
+        else
+            break;
+
+        ++start;
+    }
+
+    if (_isOperator(operatorValue))
+        out.value = operatorValue;
+    return start;
+}
+
 

CLexer.hpp

@@ -10,51 +10,76 @@ class CLexer
 public:
     enum TokenType
     {
+        INVALID,
         IDENTIFIER,		//Names which can be expanded.
         COMMA,			//,
         SEMICOLON,
         OPEN,			//{[(
         CLOSE,			//}])
-        PREPROCESSOR,	//Begins with #
+        PREPROCESSOR,	        //Begins with #
         NEWLINE,
         WHITESPACE,
         IGNORE,
         COMMENT,
         STRING,
-        NUMBER
+        NUMBER,
+        KEYWORD,
+        FUNCTION,
+        MACRO,
+        OPERATOR
+    };
+
+    enum OperatorType
+    {
     };
 
     struct Token
     {
+        Token()
+            : type(INVALID),
+              degenerate(false)
+        {
+        }
+
         std::string value;
         TokenType type;
+        union
+        {
+            OperatorType opType;
+        };
+        bool degenerate;
     };
 
     typedef std::list<CLexer::Token> TokenList;
     typedef TokenList::iterator TokenIterator;
 
-    CLexer()
-    {
-    }
+    CLexer();
 
-    int lex(char* start, char* end, TokenList& tokens);
+    void lex(char* start, char* end, TokenList& tokens);
 private:
     bool _searchString(const std::string& str, char in) const;
     bool _isTrivial(char in) const;
     bool _isIdentifierStart(char in) const;
     bool _isIdentifierBody(char in) const;
     bool _isNumber(char in) const;
+    bool _isBinary(char in) const;
     bool _isHex(char in) const;
+    bool _isOperatorStart(char in) const;
+    bool _isOperator(const std::string& val) const;
+    bool _isKeyword(const std::string& val) const;
     char* _parseToken(char* start, char* end, Token& out);
     char* _parseLiteral(char* start, char* end, Token& out);
     char* _parseLineComment(char* start, char* end, Token& out);
     char* _parseBlockComment(char* start, char* end, Token& out);
     char* _parseNumber(char* start, char* end, Token& out);
+    char* _parseBinaryConstant(char* start, char* end, Token& out);
     char* _parseHexConstant(char* start, char* end, Token& out);
     char* _parseFloatingPoint(char* start, char* end, Token& out);
     char* _parseCharacterLiteral(char* start, char* end, Token& out);
     char* _parseStringLiteral(char* start, char* end, Token& out);
     char* _parseIdentifier(char* start, char* end, Token& out);
+    char* _parseOperator(char* start, char* end, Token& out);
+    Token* m_lastIdentifier;
 };
 
 #endif // CLEXER_HPP

CPreprocessor.cpp

@@ -2,6 +2,7 @@
 #include <stdio.h>
 #include <sstream>
 #include <iostream>
+#include <algorithm>
 
 static std::string removeQuotes(const std::string& in)
 {
@@ -41,6 +42,7 @@ static void setFileMacro(CPreprocessor::DefineTable& defineTable, const std::str
 }
 
 CPreprocessor::CPreprocessor()
+    : m_errorCount(0)
 {
 }
 
@@ -72,7 +74,9 @@ void CPreprocessor::registerPragma(const std::string& name, std::function<void (
     m_registeredPragmas[name] = cb;
 }
 
-void CPreprocessor::registerHook(const std::string& name, std::function<void (CLexer::TokenList&, CPreprocessor::DefineTable&)> cb)
+void CPreprocessor::registerHook(const std::string& name, std::function<void (CLexer::TokenList&,
+                                                                              CPreprocessor::DefineTable&,
+                                                                              PreprocessorState)> cb)
 {
     std::string pre = "#" + name;
     HookIterator iter = m_registeredHooks.find(pre);
@@ -91,17 +95,19 @@ std::string CPreprocessor::finalizedSource()
     return ret;
 }
 
-int CPreprocessor::preprocessFile(const std::string& filename)
+bool CPreprocessor::preprocessFile(const std::string& filename)
 {
     std::string code = _loadSource(filename);
     if (code == std::string())
+    {
         printErrorMessage(std::string("Empty source file specified: ") + filename);
-    else
-        preprocessCode(filename, code);
-    return m_errorCount;
+        return false;
+    }
+
+    return preprocessCode(filename, code);;
 }
 
-int CPreprocessor::preprocessCode(const std::string& filename, const std::string& code)
+bool CPreprocessor::preprocessCode(const std::string& filename, const std::string& code)
 {
     m_currentLine = 0;
     m_errorCount = 0;
@@ -109,9 +115,8 @@ int CPreprocessor::preprocessCode(const std::string& filename, const std::string
     DefineTable defineTable = m_applicationDefined;
     m_lineTranslator.reset();
     m_tokens.clear();
-    preprocessRecursive(filename, code, m_tokens, defineTable);
 
-    return m_errorCount;
+    return preprocessRecursive(filename, code, m_tokens, defineTable);;
 }
 
 std::string CPreprocessor::_loadSource(const std::string& filename)
@@ -158,7 +163,22 @@ void CPreprocessor::printWarningMessage(const std::string& warnMesg)
     std::cout << warnMesg << std::endl;
 }
 
-void CPreprocessor::preprocessRecursive(const std::string& filename, const std::string& code, CLexer::TokenList& tokens, DefineTable& defineTable)
+CLexer::TokenIterator CPreprocessor::_parseIdentifier(CLexer::TokenIterator begin, CLexer::TokenIterator end, CLexer::TokenList& tokens, DefineTable& defineTable)
+{
+    std::string macroName = begin->value;
+    MacroIterator iter = std::find_if(m_macros.begin(), m_macros.end(), [&macroName](const Macro& m) -> bool { return m.name == macroName; });
+    if (iter != m_macros.end())
+    {
+        _expandMacro(begin, end, tokens, *iter);
+        --begin;
+    }
+    else
+        begin = _expandDefine(begin, end, tokens, defineTable);
+
+    return begin;
+}
+
+bool CPreprocessor::preprocessRecursive(const std::string& filename, const std::string& code, CLexer::TokenList& tokens, DefineTable& defineTable)
 {
     unsigned int startLine = m_currentLine;
     m_currentFile = filename;
@@ -184,10 +204,42 @@ void CPreprocessor::preprocessRecursive(const std::string& filename, const std::
             ++begin;
             setLineMacro(defineTable, m_currentFileLines);
         }
+        else if (begin->type == CLexer::MACRO)
+        {
+            CLexer::TokenIterator lineStart = begin;
+            CLexer::TokenIterator lineEnd = _findToken(begin, end, CLexer::NEWLINE);
+            CLexer::TokenList directive(lineStart, lineEnd);
+            begin = tokens.erase(lineStart, lineEnd);
+            advanceList(directive);
+            Macro macro;
+            macro.name = directive.begin()->value;
+            while (directive.begin()->type != CLexer::CLOSE && directive.begin()->value != ")")
+            {
+                advanceList(directive);
+                if (directive.empty())
+                    break;
+
+                if (directive.begin()->type == CLexer::IDENTIFIER || directive.begin()->type == CLexer::PREPROCESSOR)
+                   macro.args.push_back(*directive.begin());
+            }
+
+            advanceList(directive);
+            while(directive.begin()->value != "\n")
+            {
+                if (directive.empty())
+                    break;
+                if(directive.begin()->value != "\\")
+                    macro.code.push_back(*directive.begin());
+
+                directive.pop_front();
+            }
+            m_macros.push_back(macro);
+        }
         else if (begin->type == CLexer::PREPROCESSOR)
         {
             CLexer::TokenIterator lineStart = begin;
             CLexer::TokenIterator lineEnd = _findToken(begin, end, CLexer::NEWLINE);
+
             CLexer::TokenList directive(lineStart, lineEnd);
             begin = tokens.erase(lineStart, lineEnd);
 
@@ -257,16 +309,40 @@ void CPreprocessor::preprocessRecursive(const std::string& filename, const std::
             {
                 HookIterator iter = m_registeredHooks.find(value);
                 if (iter != m_registeredHooks.end() && m_registeredHooks[value])
-                    m_registeredHooks[value](directive, defineTable);
+                {
+                    PreprocessorState state;
+                    state.currentFile = m_currentFile;
+                    state.rootFile = m_rootFile;
+                    state.currentLine = m_currentFileLines;
+                    state.globalLine = m_currentLine;
+                    m_registeredHooks[value](directive, defineTable, state);
+                }
             }
         }
         else if (begin->type == CLexer::IDENTIFIER)
+            begin = _parseIdentifier(begin, end, tokens, defineTable);
+        else if (begin->degenerate)
         {
-            begin = _expandDefine(begin, end, tokens, defineTable);
+            switch(begin->type)
+            {
+                case CLexer::COMMENT:
+                {
+                    std::stringstream ss;
+                    ss << m_currentFile << ": Degenerate comment on line " << m_currentFileLines << std::endl;
+                    printErrorMessage(ss.str());
+                    break;
+                }
+                default:
+                    printErrorMessage(m_currentFile + ": Degenerate token: " + begin->value);
+            }
+
+            ++begin;
         }
         else
             ++begin;
     }
+
+    return !(m_errorCount > 0);
 }
 
 void CPreprocessor::callPragma(const std::string& name, const PragmaInstance& parms)
@@ -404,6 +480,80 @@ CLexer::TokenIterator CPreprocessor::_expandDefine(CLexer::TokenIterator begin,
     return begin;
 }
 
+void CPreprocessor::_expandMacro(CLexer::TokenIterator begin, CLexer::TokenIterator end, CLexer::TokenList& tokens, const CPreprocessor::Macro& macro)
+{
+    std::vector<CLexer::Token>  macroArgs = macro.args;
+    std::vector<CLexer::Token>  args;
+    std::vector<CLexer::Token>  code = macro.code;
+    std::vector<CLexer::Token>::iterator codeIter = code.begin();
+
+    int depth = 0;
+    while (true)
+    {
+        begin = tokens.erase(begin);
+        if (begin == end)
+            break;
+        if (begin->type == CLexer::OPEN && begin->value == "(")
+        {
+            depth++;
+            continue;
+        }
+
+        if (begin->type == CLexer::CLOSE && begin->value == ")")
+        {
+            depth--;
+            if (depth == 0)
+            {
+                begin = tokens.erase(begin);
+                break;
+            }
+        }
+
+        if (begin->type != CLexer::COMMA && begin->type != CLexer::WHITESPACE)
+        {
+            args.push_back(*begin);
+        }
+    }
+
+    if (args.empty())
+    {
+        printErrorMessage("Expected args");
+        return;
+    }
+
+    if (args.size() != macroArgs.size())
+    {
+        printErrorMessage("Argument count mismatch");
+        return;
+    }
+
+    while (codeIter != code.end())
+    {
+        std::string value = codeIter->value;
+        bool stringify = (value[0] == '#');
+        if (stringify)
+            value.erase(value.begin());
+
+        auto it = std::find_if(macroArgs.begin(), macroArgs.end(), [&value](const CLexer::Token& t) -> bool { return t.value == value; });
+        if (it != macroArgs.end())
+        {
+            int index = it - macroArgs.begin();
+            code.erase(codeIter);
+            code.insert(codeIter, *(args.begin() + index));
+            if (stringify)
+            {
+                codeIter->type = CLexer::STRING;
+                codeIter->value = "\"" + codeIter->value + "\"";
+            }
+        }
+        ++codeIter;
+    }
+
+    CLexer::TokenList newCode;
+    std::copy(code.begin(), code.end(), std::back_inserter(newCode));
+    tokens.splice(begin, newCode);
+}
+
 void CPreprocessor::_parseDefine(CPreprocessor::DefineTable& defineTable, CLexer::TokenList& tokens)
 {
     advanceList(tokens);
@@ -430,7 +580,7 @@ void CPreprocessor::_parseDefine(CPreprocessor::DefineTable& defineTable, CLexer
 
     if (!tokens.empty())
     {
-        if (tokens.begin()->type == CLexer::PREPROCESSOR && tokens.begin()->value == "#")
+        if (tokens.begin()->type == CLexer::PREPROCESSOR || tokens.begin()->type == CLexer::MACRO)
         {
             // macro has arguments
             advanceList(tokens);
@@ -526,7 +676,7 @@ void CPreprocessor::_parseIf(CLexer::TokenList& directive, std::string& nameOut)
     nameOut = directive.begin()->value;
     advanceList(directive);
     if (!directive.empty())
-        printErrorMessage("Too many arguments.");
+        printErrorMessage((m_lineTranslator.resolveOriginalFile(m_currentLine) + ": Too many arguments."));
 }
 
 void CPreprocessor::_parsePragma(CLexer::TokenList& args)
@@ -550,15 +700,15 @@ void CPreprocessor::_parsePragma(CLexer::TokenList& args)
         advanceList(args);
     }
     if (!args.empty())
-        printErrorMessage("Too many paremeters for pragma.");
+        printErrorMessage((m_lineTranslator.resolveOriginalFile(m_currentLine) +  ": Too many paremeters for pragma."));
 
     PragmaInstance pi;
     pi.name = pragmaName;
     pi.text = pragmaArgs;
-    pi.currentFile = m_currentFile;
-    pi.currentLine = m_currentFileLines;
-    pi.rootFile    = m_rootFile;
-    pi.globalLine  = m_currentLine;
+    pi.state.currentFile = m_currentFile;
+    pi.state.currentLine = m_currentFileLines;
+    pi.state.rootFile    = m_rootFile;
+    pi.state.globalLine  = m_currentLine;
     callPragma(pragmaName, pi);
 }
 
@@ -597,7 +747,7 @@ void CPreprocessor::_parseWarning(CLexer::TokenList& args, DefineTable& defineTa
     }
 
     std::string msg = _expandMessage(defineTable, args);
-    printWarningMessage(msg);
+    printWarningMessage((m_lineTranslator.resolveOriginalFile(m_currentLine) + ": Warning ") + msg);
 }
 
 void CPreprocessor::_parseError(CLexer::TokenList& args, CPreprocessor::DefineTable& defineTable)

CPreprocessor.hpp

@@ -10,14 +10,26 @@
 class CPreprocessor
 {
 public:
+    struct Macro
+    {
+        std::string name;
+        std::vector<CLexer::Token> args;
+        std::vector<CLexer::Token> code;
+    };
+
+    struct PreprocessorState
+    {
+        std::string currentFile;
+        std::string rootFile;
+        unsigned int currentLine;
+        unsigned int globalLine;
+    };
+
     struct PragmaInstance
     {
         std::string name;
         std::string text;
-        std::string currentFile;
-        unsigned int currentLine;
-        std::string rootFile;
-        std::string globalLine;
+        PreprocessorState state;
     };
 
     CPreprocessor();
@@ -32,17 +44,19 @@ public:
     typedef DefineTable::iterator DefineIterator;
     typedef std::map<std::string, std::function<void(PragmaInstance)> > PragmaMap;
     typedef PragmaMap::iterator PragmaIterator;
-    typedef std::map<std::string, std::function<void(CLexer::TokenList&, DefineTable&)> > HookMap;
+    typedef std::map<std::string, std::function<void(CLexer::TokenList&, DefineTable&, PreprocessorState)> > HookMap;
     typedef HookMap::iterator HookIterator;
+    typedef std::vector<Macro> MacroList;
+    typedef MacroList::iterator MacroIterator;
 
     void define(const std::string& def);
     void undefine(const std::string& def);
     void registerPragma(const std::string& name, std::function<void(PragmaInstance)>  cb);
-    void registerHook(const std::string& name, std::function<void(CLexer::TokenList&, DefineTable&)> cb);
+    void registerHook(const std::string& name, std::function<void(CLexer::TokenList&, DefineTable&, PreprocessorState)> cb);
 
     std::string finalizedSource();
-    int preprocessFile(const std::string& filename);
-    int preprocessCode(const std::string& filename, const std::string& code);
+    bool preprocessFile(const std::string& filename);
+    bool preprocessCode(const std::string& filename, const std::string& code);
 
     static void advanceList(CLexer::TokenList& tokens);
 private:
@@ -50,13 +64,14 @@ private:
     void printErrorMessage(const std::string& errMsg);
     void printWarningMessage(const std::string& warnMesg);
 
-    void preprocessRecursive(const std::string& filename, const std::string& code, CLexer::TokenList& tokens, DefineTable& defineTable);
+    bool preprocessRecursive(const std::string& filename, const std::string& code, CLexer::TokenList& tokens, DefineTable& defineTable);
 
     void callPragma(const std::string& name, const PragmaInstance& parms);
     CLexer::TokenIterator _findToken(CLexer::TokenIterator begin, CLexer::TokenIterator end, CLexer::TokenType type);
     CLexer::TokenIterator _parseStatement(CLexer::TokenIterator begin, CLexer::TokenIterator end, CLexer::TokenList& dest);
     CLexer::TokenIterator _parseDefineArguments(CLexer::TokenIterator begin, CLexer::TokenIterator end, CLexer::TokenList& tokens, std::vector<CLexer::TokenList>& args);
     CLexer::TokenIterator _expandDefine(CLexer::TokenIterator begin, CLexer::TokenIterator end, CLexer::TokenList& tokens, DefineTable& defineTable);
+    void _expandMacro(CLexer::TokenIterator begin, CLexer::TokenIterator end, CLexer::TokenList& tokens, const Macro& macro);
     void _parseDefine(DefineTable& defineTable, CLexer::TokenList& tokens);
     CLexer::TokenIterator _parseIfDef(CLexer::TokenIterator begin, CLexer::TokenIterator end);
     void _parseIf(CLexer::TokenList& directive, std::string& nameOut);
@@ -64,6 +79,7 @@ private:
     std::string _expandMessage(DefineTable& defineTable, CLexer::TokenList& args);
     void _parseWarning(CLexer::TokenList& args, DefineTable& defineTable);
     void _parseError(CLexer::TokenList& args, DefineTable& defineTable);
+    CLexer::TokenIterator _parseIdentifier(CLexer::TokenIterator begin, CLexer::TokenIterator end, CLexer::TokenList& tokens, DefineTable& defineTable);
 
     DefineTable      m_applicationDefined;
     PragmaMap        m_registeredPragmas;
@@ -76,6 +92,7 @@ private:
     unsigned int m_currentLine;
     unsigned int m_currentFileLines;
     unsigned int m_errorCount;
+    MacroList    m_macros;
 };
 
 #endif // CPREPROCESSOR_HPP

main.cpp

@@ -5,30 +5,73 @@
 #include <stdlib.h>
 #include <list>
 
-void printTokenList(CLexer::TokenList& out)
-{
-    for (const CLexer::Token& token : out)
-        std::cout << token.value;
-    std::cout << std::endl;
-}
-
-
-void linkInstancePreprocessor(CLexer::TokenList& tokens, CPreprocessor::DefineTable& defineTable)
+void linkInstancePreprocessor(CLexer::TokenList& tokens, CPreprocessor::DefineTable& defineTable, CPreprocessor::PreprocessorState state)
 {
     (void)(defineTable);
     CPreprocessor::advanceList(tokens);
-    if (tokens.size() == 0)
+    if (tokens.empty())
         return;
 
-    std::cout << tokens.begin()->value << std::endl;
+    CLexer::Token source = tokens.front();
+    CPreprocessor::advanceList(tokens);
+    if (source.type != CLexer::NUMBER)
+    {
+        std::cout << state.currentFile + ": Degenerate link request, invalid source" << std::endl;
+        return;
+    }
+    if (tokens.empty())
+    {
+        std::cout << state.currentFile + ": Degenerate link request, missing arguments" << std::endl;
+        return;
+    }
+
+    CLexer::Token target = tokens.front();
+    CPreprocessor::advanceList(tokens);
+    if (target.type != CLexer::NUMBER)
+    {
+        std::cout << state.currentFile + ": Degenerate link request, invalid target" << std::endl;
+        return;
+    }
+    if (tokens.empty())
+    {
+        std::cout << state.currentFile + ": Degenerate link request, missing arguments" << std::endl;
+        return;
+    }
+
+    CLexer::Token token = tokens.front();
+    CPreprocessor::advanceList(tokens);
+
+    std::vector<CLexer::Token> args;
+    if (token.type == CLexer::OPEN && token.value == "(")
+    {
+        while (!tokens.empty())
+        {
+            CLexer::Token currentToken = tokens.front();
+            CPreprocessor::advanceList(tokens);
+            if (currentToken.type == CLexer::CLOSE)
+                break;
+            if (currentToken.type == CLexer::COMMA)
+                continue;
+            if (currentToken.type == CLexer::NUMBER)
+                args.push_back(currentToken);
+        }
+    }
+
+    if (args.size() != 2)
+        std::cout << "Unable to link object instances, missing arguments" << std::endl;
+    else
+        std::cout << "Connecting object " << source.value << " to " << target.value << " with " << args.at(0).value << " " << args.at(1).value << std::endl;
 }
 
 int main()
 {
     CPreprocessor preprocessor;
+    preprocessor.define("_DEBUG_");
     preprocessor.registerHook("link_instance", linkInstancePreprocessor);
-    preprocessor.preprocessFile("main.as");
-    std::cout << preprocessor.finalizedSource() << std::endl;
+    if (preprocessor.preprocessFile("main.as"))
+        std::cout << preprocessor.finalizedSource() << std::endl;
+    else
+        std::cout << "Failed to preprocess script source :(" << std::endl;
     return 0;
 }