Initial Commit

AngelScriptPreprocessor.pro

@@ -0,0 +1,15 @@
+TEMPLATE = app
+CONFIG += console c++11
+CONFIG -= app_bundle
+CONFIG -= qt
+
+SOURCES += main.cpp \
+    CLexer.cpp \
+    CPreprocessor.cpp \
+    CLineTranslator.cpp
+
+HEADERS += \
+    CLexer.hpp \
+    CPreprocessor.hpp \
+    CLineTranslator.hpp
+

CLexer.cpp

@@ -0,0 +1,300 @@
+#include "CLexer.hpp"
+#include <algorithm>
+#include <assert.h>
+
+const std::string numbers = "0123456789";
+const std::string identifierStart = "_abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
+const std::string identifierBody = "_abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
+const std::string hexnumbers = "0123456789abcdefABCDEF";
+const std::string trivials = ",;\n\r\t [{(]})";
+
+const CLexer::TokenType TrivialTypes[] =
+{
+	CLexer::COMMA,
+	CLexer::SEMICOLON,
+	CLexer::NEWLINE,
+	CLexer::WHITESPACE,
+	CLexer::WHITESPACE,
+	CLexer::WHITESPACE,
+	CLexer::OPEN,
+	CLexer::OPEN,
+	CLexer::OPEN,
+	CLexer::CLOSE,
+	CLexer::CLOSE,
+	CLexer::CLOSE
+};
+
+int CLexer::lex(char* start, char* end, TokenList& tokens)
+{
+    assert(start != 0 && end != 0 && "start and end cannot be null");
+    assert(start <= end && "degenerate lex detected: end < start");
+
+    while (true)
+    {
+        CLexer::Token currentToken;
+        start = _parseToken(start, end, currentToken);
+        if (currentToken.type != CLexer::COMMENT)
+            tokens.push_back(currentToken);
+        if (start == end)
+            return 0;
+    }
+}
+
+bool CLexer::_searchString(const std::string& str, char in) const
+{
+    return (str.find_first_of(in) != std::string::npos);
+}
+
+bool CLexer::_isTrivial(char in) const
+{
+    return _searchString(trivials, in);
+}
+
+bool CLexer::_isIdentifierStart(char in) const
+{
+    return _searchString(identifierStart, in);
+}
+
+bool CLexer::_isIdentifierBody(char in) const
+{
+    return _searchString(identifierBody, in);
+}
+
+bool CLexer::_isNumber(char in) const
+{
+    return _searchString(numbers, in);
+}
+
+bool CLexer::_isHex(char in) const
+{
+    return _searchString(hexnumbers, in);
+}
+
+char* CLexer::_parseToken(char* start, char* end, CLexer::Token& out)
+{
+    if (start == end)
+        return start;
+    char curChar = *start;
+    if (_isTrivial(curChar))
+    {
+        out.value += curChar;
+        out.type = TrivialTypes[trivials.find_first_of(curChar)];
+        return ++start;
+    }
+
+    if (_isIdentifierStart(curChar))
+        return _parseIdentifier(start, end, out);
+
+    if (curChar == '#')
+    {
+        start = _parseIdentifier(start, end, out);
+        out.type = CLexer::PREPROCESSOR;
+        return start;
+    }
+
+
+    if (_isNumber(curChar))
+        return _parseNumber(start, end, out);
+    if (curChar == '\"')
+        return _parseStringLiteral(start, end, out);
+    if (curChar == '\'')
+        return _parseCharacterLiteral(start, end, out);
+    if (curChar == '/')
+    {
+        ++start;
+        // Is it a comment?
+        if (start == end)
+            return start;
+        if (*start == '*')
+            return _parseBlockComment(start, end, out);
+        if (*start == '/')
+            return _parseLineComment(start, end, out);
+        --start;
+    }
+
+    // Nothing intradasting
+    out.value = curChar;
+    out.type = CLexer::IGNORE;
+    return ++start;
+}
+
+char* CLexer::_parseLineComment(char* start, char* end, CLexer::Token& out)
+{
+    out.type = CLexer::COMMENT;
+    out.value += "//";
+
+    while (true)
+    {
+        ++start;
+        if (start == end)
+            return start;
+        out.value += start;
+        if  (*start == '\n')
+            return start;
+    }
+}
+
+char* CLexer::_parseBlockComment(char* start, char* end, CLexer::Token& out)
+{
+    out.type = CLexer::COMMENT;
+    out.value += "/*";
+
+    while (true)
+    {
+        ++start;
+        if (start == end)
+            break;
+
+        out.value += *start;
+        if (*start == '*')
+        {
+            ++start;
+            if (start == end)
+                break;
+            if (*start == '/')
+            {
+                out.value += *start;
+                break;
+            }
+            else
+                --start;
+        }
+    }
+
+    return start;
+}
+
+char* CLexer::_parseNumber(char* start, char* end, CLexer::Token& out)
+{
+    out.type = CLexer::NUMBER;
+    out.value += *start;
+    while (true)
+    {
+        ++start;
+        if (start == end)
+            return start;
+        if (_isNumber(*start))
+            out.value += *start;
+        else if (*start == '.')
+            return _parseFloatingPoint(start, end, out);
+        else if (*start == 'x')
+            return _parseFloatingPoint(start, end, out);
+        else
+            return start;
+    }
+}
+
+char* CLexer::_parseHexConstant(char* start, char* end, CLexer::Token& out)
+{
+    out.value += *start;
+    while (true)
+    {
+        ++start;
+        if (start == end)
+            return start;
+        if (_isHex(*start))
+            out.value += *start;
+        else
+            return start;
+    }
+}
+
+char* CLexer::_parseFloatingPoint(char* start, char* end, CLexer::Token& out)
+{
+    out.value += *start;
+    while (true)
+    {
+        ++start;
+        if (start == end)
+            return start;
+        if (!_isNumber(*start))
+        {
+            if (*start == 'f' || *start == 'F')
+            {
+                out.value += *start;
+                ++start;
+                return start;
+            }
+            return start;
+        }
+        else
+            out.value += *start;
+    }
+}
+
+char* CLexer::_parseCharacterLiteral(char* start, char* end, CLexer::Token& out)
+{
+    ++start;
+    if (start == end)
+        return start;
+    out.type = CLexer::NUMBER;
+    if (*start == '\\')
+    {
+        ++start;
+        if (start == end)
+            return start;
+        if (*start == 'n')
+            out.value = '\n';
+        if (*start == 't')
+            out.value = '\t';
+        if (*start == 'r')
+            out.value = '\r';
+        ++start;
+        if (start == end)
+            return start;
+        ++start;
+    }
+    else
+    {
+        out.value = *start;
+        ++start;
+        if (start == end)
+            return start;
+        ++start;
+    }
+    return start;
+}
+
+char* CLexer::_parseStringLiteral(char* start, char* end, CLexer::Token& out)
+{
+    out.type = CLexer::STRING;
+    out.value += *start;
+
+    while (true)
+    {
+        ++start;
+        if (start == end)
+            return start;
+        out.value += *start;
+
+        // Are we at the end of the string?
+        if (*start == '\"')
+            return ++start;
+        // Is it an escape sequence?
+        if (*start == '\\')
+        {
+            ++start;
+            if (start == end)
+                return start;
+            out.value += *start;
+        }
+    }
+}
+
+char* CLexer::_parseIdentifier(char* start, char* end, CLexer::Token& out)
+{
+    out.type = CLexer::IDENTIFIER;
+    out.value += *start;
+    while (true)
+    {
+        ++start;
+        if (start == end)
+            return start;
+        if (_isIdentifierBody(*start))
+            out.value += *start;
+        else
+            return start;
+    }
+}
+
+

CLexer.hpp

@@ -0,0 +1,60 @@
+#ifndef CLEXER_HPP
+#define CLEXER_HPP
+
+
+#include <list>
+#include <string>
+
+class CLexer
+{
+public:
+    enum TokenType
+    {
+        IDENTIFIER,		//Names which can be expanded.
+        COMMA,			//,
+        SEMICOLON,
+        OPEN,			//{[(
+        CLOSE,			//}])
+        PREPROCESSOR,	//Begins with #
+        NEWLINE,
+        WHITESPACE,
+        IGNORE,
+        COMMENT,
+        STRING,
+        NUMBER
+    };
+
+    struct Token
+    {
+        std::string value;
+        TokenType type;
+    };
+
+    typedef std::list<CLexer::Token> TokenList;
+    typedef TokenList::iterator TokenIterator;
+
+    CLexer()
+    {
+    }
+
+    int lex(char* start, char* end, TokenList& tokens);
+private:
+    bool _searchString(const std::string& str, char in) const;
+    bool _isTrivial(char in) const;
+    bool _isIdentifierStart(char in) const;
+    bool _isIdentifierBody(char in) const;
+    bool _isNumber(char in) const;
+    bool _isHex(char in) const;
+    char* _parseToken(char* start, char* end, Token& out);
+    char* _parseLiteral(char* start, char* end, Token& out);
+    char* _parseLineComment(char* start, char* end, Token& out);
+    char* _parseBlockComment(char* start, char* end, Token& out);
+    char* _parseNumber(char* start, char* end, Token& out);
+    char* _parseHexConstant(char* start, char* end, Token& out);
+    char* _parseFloatingPoint(char* start, char* end, Token& out);
+    char* _parseCharacterLiteral(char* start, char* end, Token& out);
+    char* _parseStringLiteral(char* start, char* end, Token& out);
+    char* _parseIdentifier(char* start, char* end, Token& out);
+};
+
+#endif // CLEXER_HPP

CLineTranslator.cpp

@@ -0,0 +1,42 @@
+#include "CLineTranslator.hpp"
+
+
+CLineTranslator::Table::Entry& CLineTranslator::Table::search(unsigned int line)
+{
+    for (size_t i = 1; i < lines.size(); ++i)
+    {
+        if (line < lines[i].startLine)
+            return lines[i-1];
+    }
+
+    return lines[lines.size() - 1];
+}
+
+void CLineTranslator::Table::addLineRange(const std::string& file, unsigned int startLine, unsigned int offset)
+{
+    Entry e;
+    e.file = file;
+    e.startLine  = startLine;
+    e.offset = offset;
+    lines.push_back(e);
+}
+
+
+CLineTranslator::CLineTranslator()
+{
+}
+
+std::string CLineTranslator::resolveOriginalFile(unsigned int lineNumber)
+{
+    return m_table.search(lineNumber).file;
+}
+
+unsigned int CLineTranslator::resolveOriginalLine(unsigned int lineNumber)
+{
+    return lineNumber - m_table.search(lineNumber).offset;
+}
+
+void CLineTranslator::reset()
+{
+    m_table.lines.clear();
+}

CLineTranslator.hpp

@@ -0,0 +1,36 @@
+#ifndef CLINENUMBERTRANSLATOR_HPP
+#define CLINENUMBERTRANSLATOR_HPP
+
+#include <string>
+#include <vector>
+
+class CLineTranslator
+{
+public:
+    struct Table
+    {
+        struct Entry
+        {
+            std::string file;
+            unsigned int startLine;
+            unsigned int offset;
+        };
+
+        std::vector<Entry> lines;
+
+        Entry& search(unsigned int line);
+
+        void addLineRange(const std::string& file, unsigned int startLine, unsigned int offset);
+    };
+
+    CLineTranslator();
+    std::string resolveOriginalFile(unsigned int lineNumber);
+    unsigned int resolveOriginalLine(unsigned int lineNumber);
+    inline void setTable(Table table) { m_table = table; }
+    inline Table& table() { return m_table; }
+    void reset();
+private:
+    Table m_table;
+};
+
+#endif // CLINENUMBERTRANSLATOR_HPP

CPreprocessor.hpp

@@ -0,0 +1,76 @@
+#ifndef CPREPROCESSOR_HPP
+#define CPREPROCESSOR_HPP
+
+#include <map>
+#include <string>
+#include <functional>
+#include "CLexer.hpp"
+#include "CLineTranslator.hpp"
+
+class CPreprocessor
+{
+public:
+    struct PragmaInstance
+    {
+        std::string name;
+        std::string text;
+        std::string currentFile;
+        unsigned int currentLine;
+        std::string rootFile;
+        std::string globalLine;
+    };
+
+    CPreprocessor();
+    typedef std::map<std::string, int> ArgSet;
+    struct DefineEntry
+    {
+        CLexer::TokenList tokens;
+        ArgSet arguments;
+    };
+
+    typedef std::map<std::string, DefineEntry> DefineTable;
+    typedef DefineTable::iterator DefineIterator;
+    typedef std::map<std::string, std::function<void(PragmaInstance)> > PragmaMap;
+    typedef PragmaMap::iterator PragmaIterator;
+
+    void define(const std::string& def);
+    void undefine(const std::string& def);
+    void registerPragma(const std::string& name, std::function<void(PragmaInstance)>  cb);
+
+    std::string finalizedSource();
+    int preprocessFile(const std::string& filename);
+    int preprocessCode(const std::string& filename, const std::string& code);
+private:
+    std::string _loadSource(const std::string& filename);
+    void printErrorMessage(const std::string& errMsg);
+    void printWarningMessage(const std::string& warnMesg);
+
+    void preprocessRecursive(const std::string& filename, const std::string& code, CLexer::TokenList& tokens, DefineTable& defineTable);
+
+    void callPragma(const std::string& name, const PragmaInstance& parms);
+    void _advanceList(CLexer::TokenList& tokens);
+    CLexer::TokenIterator _findToken(CLexer::TokenIterator begin, CLexer::TokenIterator end, CLexer::TokenType type);
+    CLexer::TokenIterator _parseStatement(CLexer::TokenIterator begin, CLexer::TokenIterator end, CLexer::TokenList& dest);
+    CLexer::TokenIterator _parseDefineArguments(CLexer::TokenIterator begin, CLexer::TokenIterator end, CLexer::TokenList& tokens, std::vector<CLexer::TokenList>& args);
+    CLexer::TokenIterator _expandDefine(CLexer::TokenIterator begin, CLexer::TokenIterator end, CLexer::TokenList& tokens, DefineTable& defineTable);
+    void _parseDefine(DefineTable& defineTable, CLexer::TokenList& tokens);
+    CLexer::TokenIterator _parseIfDef(CLexer::TokenIterator begin, CLexer::TokenIterator end);
+    void _parseIf(CLexer::TokenList& directive, std::string& nameOut);
+    void _parsePragma(CLexer::TokenList& args);
+    std::string _expandMessage(DefineTable& defineTable, CLexer::TokenList& args);
+    void _parseWarning(CLexer::TokenList& args, DefineTable& defineTable);
+    void _parseError(CLexer::TokenList& args, DefineTable& defineTable);
+
+    DefineTable      m_applicationDefined;
+    PragmaMap        m_registeredPragmas;
+    CLineTranslator  m_lineTranslator;
+    CLexer::TokenList m_tokens;
+
+    std::string  m_rootFile;
+    std::string  m_currentFile;
+    unsigned int m_currentLine;
+    unsigned int m_currentFileLines;
+    unsigned int m_errorCount;
+};
+
+#endif // CPREPROCESSOR_HPP

main.cpp

@@ -0,0 +1,22 @@
+#include <iostream>
+#include "CLexer.hpp"
+#include "CPreprocessor.hpp"
+#include <string.h>
+#include <stdlib.h>
+#include <list>
+
+void printTokenList(CLexer::TokenList& out)
+{
+    for (const CLexer::Token& token : out)
+        std::cout << token.value;
+    std::cout << std::endl;
+}
+
+int main()
+{
+    CPreprocessor preprocessor;
+    preprocessor.preprocessFile("main.as");
+    std::cout << preprocessor.finalizedSource() << std::endl;
+    return 0;
+}
+