2007-03-22 10:30:00 -07:00
|
|
|
/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
|
|
|
|
*
|
|
|
|
* ***** BEGIN LICENSE BLOCK *****
|
|
|
|
* Version: MPL 1.1/GPL 2.0/LGPL 2.1
|
|
|
|
*
|
|
|
|
* The contents of this file are subject to the Mozilla Public License Version
|
|
|
|
* 1.1 (the "License"); you may not use this file except in compliance with
|
|
|
|
* the License. You may obtain a copy of the License at
|
|
|
|
* http://www.mozilla.org/MPL/
|
|
|
|
*
|
|
|
|
* Software distributed under the License is distributed on an "AS IS" basis,
|
|
|
|
* WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
|
|
|
|
* for the specific language governing rights and limitations under the
|
|
|
|
* License.
|
|
|
|
*
|
|
|
|
* The Original Code is Mozilla Communicator client code, released
|
|
|
|
* March 31, 1998.
|
|
|
|
*
|
|
|
|
* The Initial Developer of the Original Code is
|
|
|
|
* Netscape Communications Corporation.
|
|
|
|
* Portions created by the Initial Developer are Copyright (C) 1998
|
|
|
|
* the Initial Developer. All Rights Reserved.
|
|
|
|
*
|
|
|
|
* Contributor(s):
|
|
|
|
*
|
|
|
|
* Alternatively, the contents of this file may be used under the terms of
|
|
|
|
* either of the GNU General Public License Version 2 or later (the "GPL"),
|
|
|
|
* or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
|
|
|
|
* in which case the provisions of the GPL or the LGPL are applicable instead
|
|
|
|
* of those above. If you wish to allow use of your version of this file only
|
|
|
|
* under the terms of either the GPL or the LGPL, and not to allow others to
|
|
|
|
* use your version of this file under the terms of the MPL, indicate your
|
|
|
|
* decision by deleting the provisions above and replace them with the notice
|
|
|
|
* and other provisions required by the GPL or the LGPL. If you do not delete
|
|
|
|
* the provisions above, a recipient may use your version of this file under
|
|
|
|
* the terms of any one of the MPL, the GPL or the LGPL.
|
|
|
|
*
|
|
|
|
* ***** END LICENSE BLOCK ***** */
|
|
|
|
|
|
|
|
#ifndef jsscan_h___
|
|
|
|
#define jsscan_h___
|
|
|
|
/*
|
|
|
|
* JS lexical scanner interface.
|
|
|
|
*/
|
|
|
|
#include <stddef.h>
|
|
|
|
#include <stdio.h>
|
2010-04-08 12:06:54 -07:00
|
|
|
#include <stdarg.h>
|
2008-09-05 10:19:17 -07:00
|
|
|
#include "jsversion.h"
|
2007-03-22 10:30:00 -07:00
|
|
|
#include "jsopcode.h"
|
|
|
|
#include "jsprvtd.h"
|
|
|
|
#include "jspubtd.h"
|
2009-08-14 16:10:59 -07:00
|
|
|
#include "jsvector.h"
|
2007-03-22 10:30:00 -07:00
|
|
|
|
|
|
|
#define JS_KEYWORD(keyword, type, op, version) \
|
|
|
|
extern const char js_##keyword##_str[];
|
|
|
|
#include "jskeyword.tbl"
|
|
|
|
#undef JS_KEYWORD
|
|
|
|
|
2010-03-21 16:07:48 -07:00
|
|
|
namespace js {
|
|
|
|
|
|
|
|
enum TokenKind {
|
2007-03-22 10:30:00 -07:00
|
|
|
TOK_ERROR = -1, /* well-known as the only code < EOF */
|
|
|
|
TOK_EOF = 0, /* end of file */
|
|
|
|
TOK_EOL = 1, /* end of line */
|
|
|
|
TOK_SEMI = 2, /* semicolon */
|
|
|
|
TOK_COMMA = 3, /* comma operator */
|
|
|
|
TOK_ASSIGN = 4, /* assignment ops (= += -= etc.) */
|
|
|
|
TOK_HOOK = 5, TOK_COLON = 6, /* conditional (?:) */
|
|
|
|
TOK_OR = 7, /* logical or (||) */
|
|
|
|
TOK_AND = 8, /* logical and (&&) */
|
|
|
|
TOK_BITOR = 9, /* bitwise-or (|) */
|
|
|
|
TOK_BITXOR = 10, /* bitwise-xor (^) */
|
|
|
|
TOK_BITAND = 11, /* bitwise-and (&) */
|
|
|
|
TOK_EQOP = 12, /* equality ops (== !=) */
|
|
|
|
TOK_RELOP = 13, /* relational ops (< <= > >=) */
|
|
|
|
TOK_SHOP = 14, /* shift ops (<< >> >>>) */
|
|
|
|
TOK_PLUS = 15, /* plus */
|
|
|
|
TOK_MINUS = 16, /* minus */
|
|
|
|
TOK_STAR = 17, TOK_DIVOP = 18, /* multiply/divide ops (* / %) */
|
|
|
|
TOK_UNARYOP = 19, /* unary prefix operator */
|
|
|
|
TOK_INC = 20, TOK_DEC = 21, /* increment/decrement (++ --) */
|
|
|
|
TOK_DOT = 22, /* member operator (.) */
|
|
|
|
TOK_LB = 23, TOK_RB = 24, /* left and right brackets */
|
|
|
|
TOK_LC = 25, TOK_RC = 26, /* left and right curlies (braces) */
|
|
|
|
TOK_LP = 27, TOK_RP = 28, /* left and right parentheses */
|
|
|
|
TOK_NAME = 29, /* identifier */
|
|
|
|
TOK_NUMBER = 30, /* numeric constant */
|
|
|
|
TOK_STRING = 31, /* string constant */
|
2007-10-02 16:10:09 -07:00
|
|
|
TOK_REGEXP = 32, /* RegExp constant */
|
2007-03-22 10:30:00 -07:00
|
|
|
TOK_PRIMARY = 33, /* true, false, null, this, super */
|
|
|
|
TOK_FUNCTION = 34, /* function keyword */
|
2008-08-06 17:34:58 -07:00
|
|
|
TOK_IF = 35, /* if keyword */
|
|
|
|
TOK_ELSE = 36, /* else keyword */
|
|
|
|
TOK_SWITCH = 37, /* switch keyword */
|
|
|
|
TOK_CASE = 38, /* case keyword */
|
|
|
|
TOK_DEFAULT = 39, /* default keyword */
|
|
|
|
TOK_WHILE = 40, /* while keyword */
|
|
|
|
TOK_DO = 41, /* do keyword */
|
|
|
|
TOK_FOR = 42, /* for keyword */
|
|
|
|
TOK_BREAK = 43, /* break keyword */
|
|
|
|
TOK_CONTINUE = 44, /* continue keyword */
|
|
|
|
TOK_IN = 45, /* in keyword */
|
|
|
|
TOK_VAR = 46, /* var keyword */
|
|
|
|
TOK_WITH = 47, /* with keyword */
|
|
|
|
TOK_RETURN = 48, /* return keyword */
|
|
|
|
TOK_NEW = 49, /* new keyword */
|
|
|
|
TOK_DELETE = 50, /* delete keyword */
|
|
|
|
TOK_DEFSHARP = 51, /* #n= for object/array initializers */
|
|
|
|
TOK_USESHARP = 52, /* #n# for object/array initializers */
|
|
|
|
TOK_TRY = 53, /* try keyword */
|
|
|
|
TOK_CATCH = 54, /* catch keyword */
|
|
|
|
TOK_FINALLY = 55, /* finally keyword */
|
|
|
|
TOK_THROW = 56, /* throw keyword */
|
|
|
|
TOK_INSTANCEOF = 57, /* instanceof keyword */
|
|
|
|
TOK_DEBUGGER = 58, /* debugger keyword */
|
|
|
|
TOK_XMLSTAGO = 59, /* XML start tag open (<) */
|
|
|
|
TOK_XMLETAGO = 60, /* XML end tag open (</) */
|
|
|
|
TOK_XMLPTAGC = 61, /* XML point tag close (/>) */
|
|
|
|
TOK_XMLTAGC = 62, /* XML start or end tag close (>) */
|
|
|
|
TOK_XMLNAME = 63, /* XML start-tag non-final fragment */
|
|
|
|
TOK_XMLATTR = 64, /* XML quoted attribute value */
|
|
|
|
TOK_XMLSPACE = 65, /* XML whitespace */
|
|
|
|
TOK_XMLTEXT = 66, /* XML text */
|
|
|
|
TOK_XMLCOMMENT = 67, /* XML comment */
|
|
|
|
TOK_XMLCDATA = 68, /* XML CDATA section */
|
|
|
|
TOK_XMLPI = 69, /* XML processing instruction */
|
|
|
|
TOK_AT = 70, /* XML attribute op (@) */
|
|
|
|
TOK_DBLCOLON = 71, /* namespace qualified name op (::) */
|
|
|
|
TOK_ANYNAME = 72, /* XML AnyName singleton (*) */
|
|
|
|
TOK_DBLDOT = 73, /* XML descendant op (..) */
|
|
|
|
TOK_FILTER = 74, /* XML filtering predicate op (.()) */
|
|
|
|
TOK_XMLELEM = 75, /* XML element node type (no token) */
|
|
|
|
TOK_XMLLIST = 76, /* XML list node type (no token) */
|
|
|
|
TOK_YIELD = 77, /* yield from generator function */
|
|
|
|
TOK_ARRAYCOMP = 78, /* array comprehension initialiser */
|
|
|
|
TOK_ARRAYPUSH = 79, /* array push within comprehension */
|
|
|
|
TOK_LEXICALSCOPE = 80, /* block scope AST node label */
|
|
|
|
TOK_LET = 81, /* let keyword */
|
2008-10-08 18:43:28 -07:00
|
|
|
TOK_SEQ = 82, /* synthetic sequence of statements,
|
|
|
|
not a block */
|
2008-10-14 16:13:56 -07:00
|
|
|
TOK_FORHEAD = 83, /* head of for(;;)-style loop */
|
2009-04-05 21:17:22 -07:00
|
|
|
TOK_ARGSBODY = 84, /* formal args in list + body at end */
|
|
|
|
TOK_UPVARS = 85, /* lexical dependencies as JSAtomList
|
|
|
|
of definitions paired with a parse
|
|
|
|
tree full of uses of those names */
|
2007-03-22 10:30:00 -07:00
|
|
|
TOK_RESERVED, /* reserved keywords */
|
|
|
|
TOK_LIMIT /* domain size */
|
2010-03-21 16:07:48 -07:00
|
|
|
};
|
2009-04-29 16:46:43 -07:00
|
|
|
|
2010-03-22 10:26:08 -07:00
|
|
|
static inline bool
|
|
|
|
TokenKindIsXML(TokenKind tt)
|
2010-03-21 16:07:48 -07:00
|
|
|
{
|
|
|
|
return tt == TOK_AT || tt == TOK_DBLCOLON || tt == TOK_ANYNAME;
|
|
|
|
}
|
2007-03-22 10:30:00 -07:00
|
|
|
|
2010-03-22 10:26:08 -07:00
|
|
|
static inline bool
|
|
|
|
TreeTypeIsXML(TokenKind tt)
|
2010-03-21 16:07:48 -07:00
|
|
|
{
|
|
|
|
return tt == TOK_XMLCOMMENT || tt == TOK_XMLCDATA || tt == TOK_XMLPI ||
|
|
|
|
tt == TOK_XMLELEM || tt == TOK_XMLLIST;
|
|
|
|
}
|
2010-03-18 22:26:47 -07:00
|
|
|
|
2010-03-22 10:26:08 -07:00
|
|
|
static inline bool
|
|
|
|
TokenKindIsDecl(TokenKind tt)
|
2010-03-21 16:07:48 -07:00
|
|
|
{
|
2010-03-18 22:26:47 -07:00
|
|
|
#if JS_HAS_BLOCK_SCOPE
|
2010-03-21 16:07:48 -07:00
|
|
|
return tt == TOK_VAR || tt == TOK_LET;
|
2010-03-18 22:26:47 -07:00
|
|
|
#else
|
2010-03-21 16:07:48 -07:00
|
|
|
return tt == TOK_VAR;
|
2010-03-18 22:26:47 -07:00
|
|
|
#endif
|
2010-03-21 16:07:48 -07:00
|
|
|
}
|
2007-03-22 10:30:00 -07:00
|
|
|
|
2010-03-21 16:07:48 -07:00
|
|
|
struct TokenPtr {
|
2009-05-08 14:00:56 -07:00
|
|
|
uint32 index; /* index of char in physical line */
|
|
|
|
uint32 lineno; /* physical line number */
|
2009-04-05 21:17:22 -07:00
|
|
|
|
2010-03-21 16:07:48 -07:00
|
|
|
bool operator==(const TokenPtr& bptr) {
|
2009-10-22 10:07:56 -07:00
|
|
|
return index == bptr.index && lineno == bptr.lineno;
|
|
|
|
}
|
|
|
|
|
2010-03-21 16:07:48 -07:00
|
|
|
bool operator!=(const TokenPtr& bptr) {
|
2009-10-22 10:07:56 -07:00
|
|
|
return index != bptr.index || lineno != bptr.lineno;
|
|
|
|
}
|
|
|
|
|
2010-03-21 16:07:48 -07:00
|
|
|
bool operator <(const TokenPtr& bptr) {
|
2009-04-05 21:17:22 -07:00
|
|
|
return lineno < bptr.lineno ||
|
|
|
|
(lineno == bptr.lineno && index < bptr.index);
|
|
|
|
}
|
|
|
|
|
2010-03-21 16:07:48 -07:00
|
|
|
bool operator <=(const TokenPtr& bptr) {
|
2009-04-05 21:17:22 -07:00
|
|
|
return lineno < bptr.lineno ||
|
|
|
|
(lineno == bptr.lineno && index <= bptr.index);
|
|
|
|
}
|
|
|
|
|
2010-03-21 16:07:48 -07:00
|
|
|
bool operator >(const TokenPtr& bptr) {
|
2009-04-05 21:17:22 -07:00
|
|
|
return !(*this <= bptr);
|
|
|
|
}
|
|
|
|
|
2010-03-21 16:07:48 -07:00
|
|
|
bool operator >=(const TokenPtr& bptr) {
|
2009-04-05 21:17:22 -07:00
|
|
|
return !(*this < bptr);
|
|
|
|
}
|
2007-03-22 10:30:00 -07:00
|
|
|
};
|
|
|
|
|
2010-03-21 16:07:48 -07:00
|
|
|
struct TokenPos {
|
|
|
|
TokenPtr begin; /* first character and line of token */
|
|
|
|
TokenPtr end; /* index 1 past last char, last line */
|
2009-04-05 21:17:22 -07:00
|
|
|
|
2010-03-21 16:07:48 -07:00
|
|
|
bool operator==(const TokenPos& bpos) {
|
2009-10-22 10:07:56 -07:00
|
|
|
return begin == bpos.begin && end == bpos.end;
|
|
|
|
}
|
|
|
|
|
2010-03-21 16:07:48 -07:00
|
|
|
bool operator!=(const TokenPos& bpos) {
|
2009-10-22 10:07:56 -07:00
|
|
|
return begin != bpos.begin || end != bpos.end;
|
|
|
|
}
|
|
|
|
|
2010-03-21 16:07:48 -07:00
|
|
|
bool operator <(const TokenPos& bpos) {
|
2009-04-05 21:17:22 -07:00
|
|
|
return begin < bpos.begin;
|
|
|
|
}
|
|
|
|
|
2010-03-21 16:07:48 -07:00
|
|
|
bool operator <=(const TokenPos& bpos) {
|
2009-04-05 21:17:22 -07:00
|
|
|
return begin <= bpos.begin;
|
|
|
|
}
|
|
|
|
|
2010-03-21 16:07:48 -07:00
|
|
|
bool operator >(const TokenPos& bpos) {
|
2009-04-05 21:17:22 -07:00
|
|
|
return !(*this <= bpos);
|
|
|
|
}
|
|
|
|
|
2010-03-21 16:07:48 -07:00
|
|
|
bool operator >=(const TokenPos& bpos) {
|
2009-04-05 21:17:22 -07:00
|
|
|
return !(*this < bpos);
|
|
|
|
}
|
2007-03-22 10:30:00 -07:00
|
|
|
};
|
|
|
|
|
2010-03-21 16:07:48 -07:00
|
|
|
struct Token {
|
|
|
|
TokenKind type; /* char value or above enumerator */
|
|
|
|
TokenPos pos; /* token position in file */
|
2007-03-22 10:30:00 -07:00
|
|
|
jschar *ptr; /* beginning of token in line buffer */
|
|
|
|
union {
|
2007-07-08 02:03:34 -07:00
|
|
|
struct { /* name or string literal */
|
2007-03-22 10:30:00 -07:00
|
|
|
JSOp op; /* operator, for minimal parser */
|
|
|
|
JSAtom *atom; /* atom table entry */
|
|
|
|
} s;
|
2007-10-02 16:10:09 -07:00
|
|
|
uintN reflags; /* regexp flags, use tokenbuf to access
|
|
|
|
regexp chars */
|
2007-03-22 10:30:00 -07:00
|
|
|
struct { /* atom pair, for XML PIs */
|
|
|
|
JSAtom *atom2; /* auxiliary atom table entry */
|
|
|
|
JSAtom *atom; /* main atom table entry */
|
|
|
|
} p;
|
|
|
|
jsdouble dval; /* floating point number */
|
|
|
|
} u;
|
|
|
|
};
|
|
|
|
|
2010-03-21 16:07:48 -07:00
|
|
|
enum TokenStreamFlags
|
2010-03-15 16:42:06 -07:00
|
|
|
{
|
|
|
|
TSF_ERROR = 0x01, /* fatal error while compiling */
|
|
|
|
TSF_EOF = 0x02, /* hit end of file */
|
|
|
|
TSF_NEWLINES = 0x04, /* tokenize newlines */
|
|
|
|
TSF_OPERAND = 0x08, /* looking for operand, not operator */
|
2010-05-12 21:43:01 -07:00
|
|
|
TSF_UNEXPECTED_EOF = 0x10, /* unexpected end of input, i.e. TOK_EOF not at top-level. */
|
|
|
|
TSF_KEYWORD_IS_NAME = 0x20, /* Ignore keywords and return TOK_NAME instead to the parser. */
|
2010-05-12 21:40:28 -07:00
|
|
|
TSF_STRICT_MODE_CODE = 0x40,/* Tokenize as appropriate for strict mode code. */
|
2010-03-15 16:42:06 -07:00
|
|
|
TSF_DIRTYLINE = 0x80, /* non-whitespace since start of line */
|
|
|
|
TSF_OWNFILENAME = 0x100, /* ts->filename is malloc'd */
|
|
|
|
TSF_XMLTAGMODE = 0x200, /* scanning within an XML tag in E4X */
|
|
|
|
TSF_XMLTEXTMODE = 0x400, /* scanning XMLText terminal from E4X */
|
|
|
|
TSF_XMLONLYMODE = 0x800, /* don't scan {expr} within text/tag */
|
|
|
|
|
|
|
|
/*
|
|
|
|
* To handle the hard case of contiguous HTML comments, we want to clear the
|
|
|
|
* TSF_DIRTYINPUT flag at the end of each such comment. But we'd rather not
|
|
|
|
* scan for --> within every //-style comment unless we have to. So we set
|
|
|
|
* TSF_IN_HTML_COMMENT when a <!-- is scanned as an HTML begin-comment, and
|
|
|
|
* clear it (and TSF_DIRTYINPUT) when we scan --> either on a clean line, or
|
|
|
|
* only if (ts->flags & TSF_IN_HTML_COMMENT), in a //-style comment.
|
|
|
|
*
|
|
|
|
* This still works as before given a malformed comment hiding hack such as:
|
|
|
|
*
|
|
|
|
* <script>
|
|
|
|
* <!-- comment hiding hack #1
|
|
|
|
* code goes here
|
|
|
|
* // --> oops, markup for script-unaware browsers goes here!
|
|
|
|
* </script>
|
|
|
|
*
|
|
|
|
* It does not cope with malformed comment hiding hacks where --> is hidden
|
|
|
|
* by C-style comments, or on a dirty line. Such cases are already broken.
|
|
|
|
*/
|
2010-05-12 21:43:01 -07:00
|
|
|
TSF_IN_HTML_COMMENT = 0x2000
|
2010-03-15 16:42:06 -07:00
|
|
|
};
|
|
|
|
|
2010-03-21 16:07:48 -07:00
|
|
|
#define t_op u.s.op
|
|
|
|
#define t_reflags u.reflags
|
|
|
|
#define t_atom u.s.atom
|
|
|
|
#define t_atom2 u.p.atom2
|
|
|
|
#define t_dval u.dval
|
|
|
|
|
|
|
|
class TokenStream
|
2010-03-15 16:42:06 -07:00
|
|
|
{
|
2010-03-21 16:07:48 -07:00
|
|
|
static const size_t ntokens = 4; /* 1 current + 2 lookahead, rounded
|
|
|
|
to power of 2 to avoid divmod by 3 */
|
|
|
|
static const uintN ntokensMask = ntokens - 1;
|
2010-04-09 18:54:51 -07:00
|
|
|
|
2010-03-15 16:42:06 -07:00
|
|
|
public:
|
2009-08-14 16:10:59 -07:00
|
|
|
/*
|
2010-03-21 16:07:48 -07:00
|
|
|
* To construct a TokenStream, first call the constructor, which is
|
|
|
|
* infallible, then call |init|, which can fail. To destroy a TokenStream,
|
2009-08-14 16:10:59 -07:00
|
|
|
* first call |close| then call the destructor. If |init| fails, do not call
|
|
|
|
* |close|.
|
|
|
|
*
|
|
|
|
* This class uses JSContext.tempPool to allocate internal buffers. The
|
|
|
|
* caller should JS_ARENA_MARK before calling |init| and JS_ARENA_RELEASE
|
|
|
|
* after calling |close|.
|
|
|
|
*/
|
2010-03-21 16:07:48 -07:00
|
|
|
TokenStream(JSContext *);
|
2009-08-14 16:10:59 -07:00
|
|
|
|
|
|
|
/*
|
2010-12-07 15:22:52 -08:00
|
|
|
* Create a new token stream from an input buffer.
|
|
|
|
* Return false on memory-allocation failure.
|
2009-08-14 16:10:59 -07:00
|
|
|
*/
|
2010-12-07 15:22:52 -08:00
|
|
|
bool init(JSVersion version, const jschar *base, size_t length,
|
2010-09-13 09:38:22 -07:00
|
|
|
const char *filename, uintN lineno);
|
2010-03-15 16:42:06 -07:00
|
|
|
void close();
|
2010-03-21 16:07:48 -07:00
|
|
|
~TokenStream() {}
|
2007-03-22 10:30:00 -07:00
|
|
|
|
2010-03-15 16:42:06 -07:00
|
|
|
/* Accessors. */
|
|
|
|
JSContext *getContext() const { return cx; }
|
2010-03-21 16:07:48 -07:00
|
|
|
bool onCurrentLine(const TokenPos &pos) const { return lineno == pos.end.lineno; }
|
|
|
|
const Token ¤tToken() const { return tokens[cursor]; }
|
2010-03-15 16:42:06 -07:00
|
|
|
const JSCharBuffer &getTokenbuf() const { return tokenbuf; }
|
|
|
|
const char *getFilename() const { return filename; }
|
|
|
|
uintN getLineno() const { return lineno; }
|
|
|
|
|
2010-04-09 18:54:51 -07:00
|
|
|
/* Flag methods. */
|
|
|
|
void setStrictMode(bool enabled = true) { setFlag(enabled, TSF_STRICT_MODE_CODE); }
|
|
|
|
void setXMLTagMode(bool enabled = true) { setFlag(enabled, TSF_XMLTAGMODE); }
|
|
|
|
void setXMLOnlyMode(bool enabled = true) { setFlag(enabled, TSF_XMLONLYMODE); }
|
|
|
|
void setUnexpectedEOF(bool enabled = true) { setFlag(enabled, TSF_UNEXPECTED_EOF); }
|
2010-04-16 14:00:51 -07:00
|
|
|
bool isStrictMode() { return !!(flags & TSF_STRICT_MODE_CODE); }
|
|
|
|
bool isXMLTagMode() { return !!(flags & TSF_XMLTAGMODE); }
|
|
|
|
bool isXMLOnlyMode() { return !!(flags & TSF_XMLONLYMODE); }
|
|
|
|
bool isUnexpectedEOF() { return !!(flags & TSF_UNEXPECTED_EOF); }
|
|
|
|
bool isEOF() const { return !!(flags & TSF_EOF); }
|
|
|
|
bool isError() const { return !!(flags & TSF_ERROR); }
|
2010-04-09 18:54:51 -07:00
|
|
|
|
2010-03-15 16:42:06 -07:00
|
|
|
/* Mutators. */
|
|
|
|
bool reportCompileErrorNumberVA(JSParseNode *pn, uintN flags, uintN errorNumber, va_list ap);
|
2010-04-09 18:54:51 -07:00
|
|
|
void mungeCurrentToken(TokenKind newKind) { tokens[cursor].type = newKind; }
|
|
|
|
void mungeCurrentToken(JSOp newOp) { tokens[cursor].t_op = newOp; }
|
|
|
|
void mungeCurrentToken(TokenKind newKind, JSOp newOp) {
|
|
|
|
mungeCurrentToken(newKind);
|
|
|
|
mungeCurrentToken(newOp);
|
|
|
|
}
|
|
|
|
|
|
|
|
private:
|
|
|
|
/*
|
|
|
|
* Enables flags in the associated tokenstream for the object lifetime.
|
|
|
|
* Useful for lexically-scoped flag toggles.
|
|
|
|
*/
|
|
|
|
class Flagger {
|
|
|
|
TokenStream * const parent;
|
|
|
|
uintN flags;
|
|
|
|
public:
|
|
|
|
Flagger(TokenStream *parent, uintN withFlags) : parent(parent), flags(withFlags) {
|
|
|
|
parent->flags |= flags;
|
|
|
|
}
|
|
|
|
|
|
|
|
~Flagger() { parent->flags &= ~flags; }
|
|
|
|
};
|
|
|
|
friend class Flagger;
|
2010-03-15 16:42:06 -07:00
|
|
|
|
2010-04-09 18:54:51 -07:00
|
|
|
void setFlag(bool enabled, TokenStreamFlags flag) {
|
|
|
|
if (enabled)
|
|
|
|
flags |= flag;
|
|
|
|
else
|
|
|
|
flags &= ~flag;
|
|
|
|
}
|
|
|
|
|
|
|
|
public:
|
2010-03-23 17:30:36 -07:00
|
|
|
/*
|
|
|
|
* Get the next token from the stream, make it the current token, and
|
|
|
|
* return its kind.
|
|
|
|
*/
|
2010-12-07 15:22:52 -08:00
|
|
|
TokenKind getToken() {
|
2010-03-15 16:42:06 -07:00
|
|
|
/* Check for a pushed-back token resulting from mismatching lookahead. */
|
|
|
|
while (lookahead != 0) {
|
|
|
|
JS_ASSERT(!(flags & TSF_XMLTEXTMODE));
|
|
|
|
lookahead--;
|
2010-03-21 16:07:48 -07:00
|
|
|
cursor = (cursor + 1) & ntokensMask;
|
|
|
|
TokenKind tt = currentToken().type;
|
2010-05-11 21:33:29 -07:00
|
|
|
JS_ASSERT(!(flags & TSF_NEWLINES));
|
|
|
|
if (tt != TOK_EOL)
|
2010-03-15 16:42:06 -07:00
|
|
|
return tt;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* If there was a fatal error, keep returning TOK_ERROR. */
|
|
|
|
if (flags & TSF_ERROR)
|
|
|
|
return TOK_ERROR;
|
2007-03-22 10:30:00 -07:00
|
|
|
|
2010-04-09 18:54:51 -07:00
|
|
|
return getTokenInternal();
|
2010-03-15 16:42:06 -07:00
|
|
|
}
|
2007-03-22 10:30:00 -07:00
|
|
|
|
2010-12-07 15:22:52 -08:00
|
|
|
/* Similar, but also sets flags. */
|
|
|
|
TokenKind getToken(uintN withFlags) {
|
|
|
|
Flagger flagger(this, withFlags);
|
|
|
|
return getToken();
|
|
|
|
}
|
|
|
|
|
2010-03-23 17:30:36 -07:00
|
|
|
/*
|
|
|
|
* Push the last scanned token back into the stream.
|
|
|
|
*/
|
2010-03-15 16:42:06 -07:00
|
|
|
void ungetToken() {
|
2010-03-21 16:07:48 -07:00
|
|
|
JS_ASSERT(lookahead < ntokensMask);
|
2010-03-15 16:42:06 -07:00
|
|
|
lookahead++;
|
2010-03-21 16:07:48 -07:00
|
|
|
cursor = (cursor - 1) & ntokensMask;
|
2010-03-15 16:42:06 -07:00
|
|
|
}
|
|
|
|
|
2010-04-09 18:54:51 -07:00
|
|
|
TokenKind peekToken(uintN withFlags = 0) {
|
|
|
|
Flagger flagger(this, withFlags);
|
2010-03-15 16:42:06 -07:00
|
|
|
if (lookahead != 0) {
|
2010-05-11 21:33:29 -07:00
|
|
|
JS_ASSERT(lookahead == 1);
|
2010-03-21 16:07:48 -07:00
|
|
|
return tokens[(cursor + lookahead) & ntokensMask].type;
|
2010-03-15 16:42:06 -07:00
|
|
|
}
|
2010-03-21 16:07:48 -07:00
|
|
|
TokenKind tt = getToken();
|
2010-03-15 16:42:06 -07:00
|
|
|
ungetToken();
|
|
|
|
return tt;
|
|
|
|
}
|
|
|
|
|
2010-04-09 18:54:51 -07:00
|
|
|
TokenKind peekTokenSameLine(uintN withFlags = 0) {
|
|
|
|
Flagger flagger(this, withFlags);
|
2010-03-15 16:42:06 -07:00
|
|
|
if (!onCurrentLine(currentToken().pos))
|
|
|
|
return TOK_EOL;
|
2010-04-09 18:54:51 -07:00
|
|
|
TokenKind tt = peekToken(TSF_NEWLINES);
|
2010-03-15 16:42:06 -07:00
|
|
|
return tt;
|
|
|
|
}
|
|
|
|
|
2010-03-23 17:30:36 -07:00
|
|
|
/*
|
|
|
|
* Get the next token from the stream if its kind is |tt|.
|
|
|
|
*/
|
2010-04-09 18:54:51 -07:00
|
|
|
JSBool matchToken(TokenKind tt, uintN withFlags = 0) {
|
|
|
|
Flagger flagger(this, withFlags);
|
2010-03-15 16:42:06 -07:00
|
|
|
if (getToken() == tt)
|
|
|
|
return JS_TRUE;
|
|
|
|
ungetToken();
|
|
|
|
return JS_FALSE;
|
|
|
|
}
|
|
|
|
|
2010-09-13 09:38:22 -07:00
|
|
|
void setVersion(JSVersion newVersion) { version = newVersion; }
|
|
|
|
|
2010-03-15 16:42:06 -07:00
|
|
|
private:
|
2010-03-21 16:07:48 -07:00
|
|
|
typedef struct TokenBuf {
|
2010-03-15 16:42:06 -07:00
|
|
|
jschar *base; /* base of line or stream buffer */
|
|
|
|
jschar *limit; /* limit for quick bounds check */
|
|
|
|
jschar *ptr; /* next char to get, or slot to use */
|
2010-03-21 16:07:48 -07:00
|
|
|
} TokenBuf;
|
2010-03-15 16:42:06 -07:00
|
|
|
|
2010-03-21 16:07:48 -07:00
|
|
|
TokenKind getTokenInternal(); /* doesn't check for pushback or error flag. */
|
2010-08-17 18:01:23 -07:00
|
|
|
|
2010-12-07 15:22:52 -08:00
|
|
|
int32 getChar();
|
|
|
|
int32 getCharIgnoreEOL();
|
2010-03-15 16:42:06 -07:00
|
|
|
void ungetChar(int32 c);
|
2010-12-07 15:22:52 -08:00
|
|
|
void ungetCharIgnoreEOL(int32 c);
|
2010-03-21 16:07:48 -07:00
|
|
|
Token *newToken(ptrdiff_t adjust);
|
2010-03-15 16:42:06 -07:00
|
|
|
int32 getUnicodeEscape();
|
|
|
|
JSBool peekChars(intN n, jschar *cp);
|
|
|
|
JSBool getXMLEntity();
|
2010-12-07 15:22:52 -08:00
|
|
|
jschar *findEOL();
|
2010-03-15 16:42:06 -07:00
|
|
|
|
|
|
|
JSBool matchChar(int32 expect) {
|
|
|
|
int32 c = getChar();
|
|
|
|
if (c == expect)
|
|
|
|
return JS_TRUE;
|
|
|
|
ungetChar(c);
|
|
|
|
return JS_FALSE;
|
|
|
|
}
|
|
|
|
|
|
|
|
int32 peekChar() {
|
|
|
|
int32 c = getChar();
|
|
|
|
ungetChar(c);
|
|
|
|
return c;
|
|
|
|
}
|
|
|
|
|
|
|
|
void skipChars(intN n) {
|
|
|
|
while (--n >= 0)
|
|
|
|
getChar();
|
|
|
|
}
|
|
|
|
|
|
|
|
JSContext * const cx;
|
2010-03-21 16:07:48 -07:00
|
|
|
Token tokens[ntokens];/* circular token buffer */
|
2010-03-15 16:42:06 -07:00
|
|
|
uintN cursor; /* index of last parsed token */
|
|
|
|
uintN lookahead; /* count of lookahead tokens */
|
|
|
|
uintN lineno; /* current line number */
|
|
|
|
uintN flags; /* flags -- see above */
|
2010-12-07 15:22:52 -08:00
|
|
|
jschar *linebase; /* start of current line; points into userbuf */
|
|
|
|
jschar *prevLinebase; /* start of previous line; NULL if on the first line */
|
|
|
|
TokenBuf userbuf; /* user input buffer */
|
2010-03-15 16:42:06 -07:00
|
|
|
const char *filename; /* input filename or null */
|
|
|
|
JSSourceHandler listener; /* callback for source; eg debugger */
|
|
|
|
void *listenerData; /* listener 'this' data */
|
|
|
|
void *listenerTSData;/* listener data for this TokenStream */
|
|
|
|
JSCharBuffer tokenbuf; /* current token string buffer */
|
2010-08-17 18:01:23 -07:00
|
|
|
bool maybeEOL[256]; /* probabilistic EOL lookup table */
|
|
|
|
bool maybeStrSpecial[256];/* speeds up string scanning */
|
2010-09-13 09:38:22 -07:00
|
|
|
JSVersion version; /* cached version number for scan */
|
2010-03-15 16:42:06 -07:00
|
|
|
};
|
2009-11-18 13:33:53 -08:00
|
|
|
|
2010-03-21 16:07:48 -07:00
|
|
|
} /* namespace js */
|
|
|
|
|
2007-03-22 10:30:00 -07:00
|
|
|
/* Unicode separators that are treated as line terminators, in addition to \n, \r */
|
|
|
|
#define LINE_SEPARATOR 0x2028
|
|
|
|
#define PARA_SEPARATOR 0x2029
|
|
|
|
|
2007-10-02 16:10:09 -07:00
|
|
|
extern void
|
2010-03-21 16:07:48 -07:00
|
|
|
js_CloseTokenStream(JSContext *cx, js::TokenStream *ts);
|
2007-03-22 10:30:00 -07:00
|
|
|
|
|
|
|
extern JS_FRIEND_API(int)
|
|
|
|
js_fgets(char *buf, int size, FILE *file);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If the given char array forms JavaScript keyword, return corresponding
|
|
|
|
* token. Otherwise return TOK_EOF.
|
|
|
|
*/
|
2010-03-21 16:07:48 -07:00
|
|
|
extern js::TokenKind
|
2007-03-22 10:30:00 -07:00
|
|
|
js_CheckKeyword(const jschar *chars, size_t length);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Friend-exported API entry point to call a mapping function on each reserved
|
|
|
|
* identifier in the scanner's keyword table.
|
|
|
|
*/
|
2009-12-13 23:08:55 -08:00
|
|
|
typedef void (*JSMapKeywordFun)(const char *);
|
|
|
|
|
2007-03-24 00:34:47 -07:00
|
|
|
/*
|
|
|
|
* Check that str forms a valid JS identifier name. The function does not
|
|
|
|
* check if str is a JS keyword.
|
|
|
|
*/
|
|
|
|
extern JSBool
|
|
|
|
js_IsIdentifier(JSString *str);
|
|
|
|
|
2010-03-21 16:07:48 -07:00
|
|
|
/*
|
|
|
|
* Steal one JSREPORT_* bit (see jsapi.h) to tell that arguments to the error
|
|
|
|
* message have const jschar* type, not const char*.
|
|
|
|
*/
|
|
|
|
#define JSREPORT_UC 0x100
|
|
|
|
|
|
|
|
namespace js {
|
|
|
|
|
2007-03-22 10:30:00 -07:00
|
|
|
/*
|
2007-11-13 14:18:17 -08:00
|
|
|
* Report a compile-time error by its number. Return true for a warning, false
|
|
|
|
* for an error. When pn is not null, use it to report error's location.
|
|
|
|
* Otherwise use ts, which must not be null.
|
2007-03-22 10:30:00 -07:00
|
|
|
*/
|
2009-11-19 09:23:20 -08:00
|
|
|
bool
|
2010-03-21 16:07:48 -07:00
|
|
|
ReportCompileErrorNumber(JSContext *cx, TokenStream *ts, JSParseNode *pn, uintN flags,
|
|
|
|
uintN errorNumber, ...);
|
2007-03-22 10:30:00 -07:00
|
|
|
|
2009-11-19 09:23:20 -08:00
|
|
|
/*
|
|
|
|
* Report a condition that should elicit a warning with JSOPTION_STRICT,
|
|
|
|
* or an error if ts or tc is handling strict mode code. This function
|
2010-03-21 16:07:48 -07:00
|
|
|
* defers to ReportCompileErrorNumber to do the real work. Either tc
|
2009-11-19 09:23:20 -08:00
|
|
|
* or ts may be NULL, if there is no tree context or token stream state
|
|
|
|
* whose strictness should affect the report.
|
|
|
|
*
|
2010-03-21 16:07:48 -07:00
|
|
|
* One could have ReportCompileErrorNumber recognize the
|
2009-11-19 09:23:20 -08:00
|
|
|
* JSREPORT_STRICT_MODE_ERROR flag instead of having a separate function
|
|
|
|
* like this one. However, the strict mode code flag we need to test is
|
|
|
|
* in the JSTreeContext structure for that code; we would have to change
|
2010-03-21 16:07:48 -07:00
|
|
|
* the ~120 ReportCompileErrorNumber calls to pass the additional
|
2009-11-19 09:23:20 -08:00
|
|
|
* argument, even though many of those sites would never use it. Using
|
|
|
|
* ts's TSF_STRICT_MODE_CODE flag instead of tc's would be brittle: at some
|
|
|
|
* points ts's flags don't correspond to those of the tc relevant to the
|
|
|
|
* error.
|
|
|
|
*/
|
|
|
|
bool
|
2010-03-21 16:07:48 -07:00
|
|
|
ReportStrictModeError(JSContext *cx, TokenStream *ts, JSTreeContext *tc, JSParseNode *pn,
|
|
|
|
uintN errorNumber, ...);
|
2007-03-22 10:30:00 -07:00
|
|
|
|
2010-03-21 16:07:48 -07:00
|
|
|
} /* namespace js */
|
2007-03-22 10:30:00 -07:00
|
|
|
|
|
|
|
#endif /* jsscan_h___ */
|