Imported Upstream version 5.4.0.167

Former-commit-id: 5624ac747d633e885131e8349322922b6a59baaa
This commit is contained in:
Xamarin Public Jenkins (auto-signing)
2017-08-21 15:34:15 +00:00
parent e49d6f06c0
commit 536cd135cc
12856 changed files with 563812 additions and 223249 deletions

View File

@@ -0,0 +1 @@
/EcmaUrlParser.cs

View File

@@ -0,0 +1,419 @@
using System;
using System.Linq;
using System.Text;
using System.Collections.Generic;
namespace Monodoc.Ecma
{
/* Some properties might not be filled/meaningful depending on kind
* like a namespace EcmaUrl won't have a valid TypeName
*/
public class EcmaDesc : IEquatable<EcmaDesc>
{
public enum Kind
{
Type,
Constructor,
Method,
Namespace,
Field,
Property,
Event,
Operator
}
public enum Mod
{
Normal,
Pointer,
Ref,
Out
}
public enum Format
{
WithArgs,
WithoutArgs
}
public Kind DescKind {
get;
set;
}
public Mod DescModifier {
get;
set;
}
public string Namespace {
get;
set;
}
public string TypeName {
get;
set;
}
public string MemberName {
get;
set;
}
public EcmaDesc NestedType {
get;
set;
}
/* A list of the array dimensions attached to this type.
* The list count corresponds to the number of recursive
* array definition (jagged arrays) the value of the
* corresponding list item is the number of dimension
* attached to that array definition instance
*/
public IList<int> ArrayDimensions {
get;
set;
}
/* Depending on the form of the url, we might not have the type
* of the argument but only how many the type/member has i.e.
* when such number is specified with a backtick
*/
public IList<EcmaDesc> GenericTypeArguments {
get;
set;
}
/* The GenericTypeArguments list may be null, in which case, this
* is an easier/safer way to check the count.
*/
public int GenericTypeArgumentsCount {
get { return GenericTypeArguments != null ? GenericTypeArguments.Count : 0; }
}
/* This property tells if the above collections only correct value
* is the number of item in it to represent generic arguments
*/
public bool GenericTypeArgumentsIsNumeric {
get {
return GenericTypeArguments != null && GenericTypeArguments.FirstOrDefault () == null;
}
}
public IList<EcmaDesc> GenericMemberArguments {
get;
set;
}
/* The GenericMemberArguments list may be null, in which case, this
* is an easier/safer way to check the count.
*/
public int GenericMemberArgumentsCount {
get { return GenericMemberArguments != null ? GenericMemberArguments.Count : 0; }
}
public bool GenericMemberArgumentsIsNumeric {
get {
return GenericMemberArguments != null && GenericMemberArguments.FirstOrDefault () == null;
}
}
public IList<EcmaDesc> MemberArguments {
get;
set;
}
/* The GenericTypeArguments list may be null, in which case, this
* is an easier/safer way to check the count.
*/
public int MemberArgumentsCount {
get { return MemberArguments != null ? MemberArguments.Count : 0; }
}
/* This indicates that we actually want an inner part of the ecmadesc
* i.e. in case of T: we could want the members (*), ctor (C), methods (M), ...
*/
public char Etc {
get;
set;
}
public bool IsEtc {
get {
return Etc != (char)0;
}
}
/* EtcFilter is only valid in some case of IsEtc when the inner part needs
* to be further filtered e.g. in case we want a listing of the type overloads
* Equals
*/
public string EtcFilter {
get;
set;
}
/* When a member is an explicit implementation of an interface member, we register
* the member EcmaDesc with its interface parent here
*/
public EcmaDesc ExplicitImplMember {
get;
set;
}
// Returns the TypeName and the generic/inner type information if existing
public string ToCompleteTypeName (char innerTypeSeparator = '.')
{
var result = TypeName;
if (GenericTypeArguments != null)
result += FormatGenericArgs (GenericTypeArguments);
if (NestedType != null)
result += innerTypeSeparator + NestedType.ToCompleteTypeName ();
if (ArrayDimensions != null && ArrayDimensions.Count > 0)
result += ArrayDimensions.Select (dim => "[" + new string (',', dim - 1) + "]").Aggregate (string.Concat);
return result;
}
// Returns the member name with its generic types if existing
public string ToCompleteMemberName (Format format)
{
/* We special process two cases:
* - Explicit member implementation which append a full type specification
* - Conversion operator which are exposed as normal method but have specific captioning in the end
*/
if (ExplicitImplMember != null) {
var impl = ExplicitImplMember;
return impl.FormattedNamespace + impl.ToCompleteTypeName () + "." + impl.ToCompleteMemberName (format);
} else if (format == Format.WithArgs && DescKind == Kind.Operator && MemberName.EndsWith ("Conversion")) {
var type1 = MemberArguments[0].FormattedNamespace + MemberArguments[0].ToCompleteTypeName () + ModToString (MemberArguments[0]);
var type2 = MemberArguments[1].FormattedNamespace + MemberArguments[1].ToCompleteTypeName () + ModToString (MemberArguments[1]);
return type1 + " to " + type2;
}
var result = IsEtc && !string.IsNullOrEmpty (EtcFilter) ? EtcFilter : MemberName;
// Temporary hack for monodoc produced inner type ctor
//if (DescKind == Kind.Constructor && NestedType != null)
//result = ToCompleteTypeName ();
if (GenericMemberArguments != null)
result += FormatGenericArgs (GenericMemberArguments);
if (format == Format.WithArgs) {
result += '(';
if (MemberArguments != null && MemberArguments.Count > 0) {
var args = MemberArguments.Select (a => FormatNamespace (a) + a.ToCompleteTypeName ('+') + ModToString (a));
result += string.Join (",", args);
}
result += ')';
}
return result;
}
public string ToEcmaCref ()
{
var sb = new StringBuilder ();
// Cref type
sb.Append (DescKind.ToString ()[0]);
sb.Append (":");
// Create the rest
ConstructCRef (sb);
return sb.ToString ();
}
void ConstructCRef (StringBuilder sb, bool skipLeadingDot = false)
{
if (string.IsNullOrEmpty (Namespace))
skipLeadingDot = true;
sb.Append (Namespace);
if (DescKind == Kind.Namespace)
return;
if (!skipLeadingDot)
sb.Append ('.');
sb.Append (TypeName);
AppendGenericArguments (sb, GenericTypeArguments, GenericTypeArgumentsIsNumeric, GenericTypeArgumentsCount);
if (NestedType != null) {
sb.Append ('+');
NestedType.ConstructCRef (sb, skipLeadingDot: true);
}
if (ArrayDimensions != null && ArrayDimensions.Count > 0) {
for (int i = 0; i < ArrayDimensions.Count; i++) {
sb.Append ('[');
sb.Append (new string (',', ArrayDimensions[i] - 1));
sb.Append (']');
}
}
if (DescKind == Kind.Type)
return;
if (ExplicitImplMember != null) {
sb.Append ('$');
ExplicitImplMember.DescKind = this.DescKind;
ExplicitImplMember.ConstructCRef (sb, skipLeadingDot: false);
return;
}
sb.Append (".");
sb.Append (MemberName);
AppendGenericArguments (sb, GenericMemberArguments, GenericMemberArgumentsIsNumeric, GenericMemberArgumentsCount);
if (MemberArguments != null && MemberArgumentsCount > 0) {
sb.Append ("(");
int i=0;
foreach (var a in MemberArguments) {
if (i > 0) {
sb.Append(",");
}
a.ConstructCRef (sb);
i++;
}
sb.Append (")");
}
}
void AppendGenericArguments (StringBuilder sb, IEnumerable<EcmaDesc> arguments, bool isNumeric, int argumentsCount)
{
if (arguments != null && isNumeric) {
sb.AppendFormat ("`{0}", argumentsCount);
} else if (arguments != null) {
sb.Append ('<');
int i=0;
foreach (var t in arguments) {
if (i > 0) {
sb.Append (",");
}
t.ConstructCRef (sb);
i++;
}
sb.Append ('>');
}
}
public override string ToString ()
{
return string.Format ("({8}) {0}::{1}{2}{3}{7} {4}{5}{6} {9} {10}",
Namespace,
TypeName,
FormatGenericArgsFull (GenericTypeArguments),
NestedType != null ? "+" + NestedType.ToString () : string.Empty,
MemberName ?? string.Empty,
FormatGenericArgsFull (GenericMemberArguments),
MemberArguments != null ? "(" + string.Join (",", MemberArguments.Select (m => m.ToString ())) + ")" : string.Empty,
ArrayDimensions != null && ArrayDimensions.Count > 0 ? ArrayDimensions.Select (dim => "[" + new string (',', dim - 1) + "]").Aggregate (string.Concat) : string.Empty,
DescKind.ToString ()[0],
Etc != 0 ? '(' + Etc.ToString () + ')' : string.Empty,
ExplicitImplMember != null ? "$" + ExplicitImplMember.ToString () : string.Empty);
}
public override bool Equals (object other)
{
var otherDesc = other as EcmaDesc;
return otherDesc != null && Equals (otherDesc);
}
public bool Equals (EcmaDesc other)
{
if (other == null)
return false;
if (NestedType == null ^ other.NestedType == null
|| ArrayDimensions == null ^ other.ArrayDimensions == null
|| GenericTypeArguments == null ^ other.GenericTypeArguments == null
|| GenericMemberArguments == null ^ other.GenericMemberArguments == null
|| MemberArguments == null ^ other.MemberArguments == null
|| ExplicitImplMember == null ^ other.ExplicitImplMember == null)
return false;
return other != null
&& DescKind == other.DescKind
&& TypeName == other.TypeName
&& Namespace == other.Namespace
&& MemberName == other.MemberName
&& (NestedType == null || NestedType.Equals (other.NestedType))
&& (ArrayDimensions == null || ArrayDimensions.SequenceEqual (other.ArrayDimensions))
&& (GenericTypeArguments == null || GenericTypeArguments.SequenceEqual (other.GenericTypeArguments))
&& (GenericMemberArguments == null || GenericMemberArguments.SequenceEqual (other.GenericMemberArguments))
&& (MemberArguments == null || MemberArguments.SequenceEqual (other.MemberArguments))
&& Etc == other.Etc
&& EtcFilter == other.EtcFilter
&& (ExplicitImplMember == null || ExplicitImplMember.Equals (other.ExplicitImplMember));
}
public override int GetHashCode ()
{
return DescKind.GetHashCode ()
^ TypeName.GetHashCode ()
^ Namespace.GetHashCode ()
^ MemberName.GetHashCode ();
}
bool What (bool input)
{
if (!input)
throw new Exception ("Not equal");
return input;
}
bool WhatT (bool input)
{
if (input)
throw new Exception ("Not equal");
return input;
}
string FormatNamespace (EcmaDesc desc)
{
return string.IsNullOrEmpty (desc.Namespace) ? string.Empty : desc.Namespace + ".";
}
string FormatGenericArgs (IEnumerable<EcmaDesc> args)
{
if (args == null || !args.Any ())
return string.Empty;
// If we only have the number of generic arguments, use ` notation
if (args.First () == null)
return "`" + args.Count ();
IEnumerable<string> argsList = args.Select (t => FormatNamespace (t) + t.ToCompleteTypeName ());
return "<" + string.Join (",", argsList) + ">";
}
string FormatGenericArgsFull (IEnumerable<EcmaDesc> genericArgs)
{
return genericArgs != null ? "<" + string.Join (",", genericArgs.Select (t => t.ToString ())) + ">" : string.Empty;
}
string ModToString (EcmaDesc desc)
{
switch (desc.DescModifier) {
case Mod.Pointer:
return "*";
case Mod.Ref:
return "&";
case Mod.Out:
return "@";
default:
return string.Empty;
}
}
string FormattedNamespace {
get {
return !string.IsNullOrEmpty (Namespace) ? Namespace + "." : string.Empty;
}
}
}
}

View File

@@ -0,0 +1,263 @@
%{
using System.Text;
using System.IO;
using System;
using System.Linq;
using System.Collections.Generic;
namespace Monodoc.Ecma
{
public class EcmaUrlParser
{
int yacc_verbose_flag = 0;
public void IsValid (string input)
{
var lexer = new EcmaUrlTokenizer (input);
this.yyparse (lexer);
}
public EcmaDesc Parse (string input)
{
var lexer = new EcmaUrlTokenizer (input);
return (EcmaDesc)this.yyparse (lexer);
}
public bool TryParse (string input, out EcmaDesc desc)
{
desc = null;
try {
desc = Parse (input);
} catch {
return false;
}
return true;
}
EcmaDesc SetEcmaDescType (object result, EcmaDesc.Kind kind)
{
var desc = result as EcmaDesc;
desc.DescKind = kind;
return desc;
}
List<T> SafeReverse<T> (List<T> input)
{
if (input == null)
return null;
input.Reverse ();
return input;
}
%}
%token ERROR
%token IDENTIFIER
%token DIGIT
%token DOT
%token COMMA
%token COLON
%token INNER_TYPE_SEPARATOR
%token OP_GENERICS_LT
%token OP_GENERICS_GT
%token OP_GENERICS_BACKTICK
%token OP_OPEN_PAREN
%token OP_CLOSE_PAREN
%token OP_ARRAY_OPEN
%token OP_ARRAY_CLOSE
%token SLASH_SEPARATOR
%token STAR
%token REF_ARG
%token OUT_ARG
%token EXPLICIT_IMPL_SEP
%start expression
%%
expression
: 'T' COLON type_expression { $$ = SetEcmaDescType ($3, EcmaDesc.Kind.Type); }
| 'N' COLON namespace_expression { $$ = SetEcmaDescType ($3, EcmaDesc.Kind.Namespace); }
| 'M' COLON method_expression { $$ = SetEcmaDescType ($3, EcmaDesc.Kind.Method); }
| 'F' COLON simple_member_expression { $$ = SetEcmaDescType ($3, EcmaDesc.Kind.Field); }
| 'C' COLON constructor_expression { $$ = SetEcmaDescType ($3, EcmaDesc.Kind.Constructor); }
| 'P' COLON property_expression { $$ = SetEcmaDescType ($3, EcmaDesc.Kind.Property); }
| 'E' COLON simple_member_expression { $$ = SetEcmaDescType ($3, EcmaDesc.Kind.Event); }
| 'O' COLON operator_expression { $$ = SetEcmaDescType ($3, EcmaDesc.Kind.Operator); }
/* i.e. id.id.id or id */
dot_expression
: IDENTIFIER { $$ = new List<string> { (string)$1 }; }
| IDENTIFIER DOT dot_expression { ((ICollection<string>)$3).Add ((string)$1); $$ = $3; }
namespace_expression
: dot_expression { $$ = new EcmaDesc { Namespace = string.Join (".", ((IEnumerable<string>)$1).Reverse ()) }; }
type_expression
: dot_expression type_expression_suffix {
var dotExpr = ((List<string>)$1);
dotExpr.Reverse ();
var desc = $2 as EcmaDesc;
desc.DescKind = EcmaDesc.Kind.Type;
desc.Namespace = string.Join (".", dotExpr.Take (dotExpr.Count - 1));
desc.TypeName = dotExpr.Last ();
$$ = desc;
}
/* To be used in types with no namespaces attached to them like an inner type*/
reduced_type_expression
: IDENTIFIER type_expression_suffix {
var desc = $2 as EcmaDesc;
desc.DescKind = EcmaDesc.Kind.Type;
desc.TypeName = $1 as string;
$$ = desc;
}
type_expression_suffix
: opt_generic_type_suffix opt_inner_type_description opt_array_definition opt_etc {
bool nestedDescHasEtc = $2 != null && ((EcmaDesc)$2).IsEtc;
EcmaDesc nestedType = (EcmaDesc)$2;
$$ = new EcmaDesc {
GenericTypeArguments = $1 as List<EcmaDesc>,
NestedType = nestedType,
ArrayDimensions = SafeReverse ($3 as List<int>),
Etc = $4 != null ? ((Tuple<char, string>)$4).Item1 : nestedDescHasEtc ? nestedType.Etc : (char)0,
EtcFilter = $4 != null ? ((Tuple<char, string>)$4).Item2 : nestedDescHasEtc ? nestedType.EtcFilter : null
};
if (nestedDescHasEtc) {
nestedType.Etc = (char)0;
nestedType.EtcFilter = null;
}
}
opt_inner_type_description
: /* empty */ { $$ = null; }
| INNER_TYPE_SEPARATOR reduced_type_expression { $$ = $2; }
opt_generic_type_suffix
: /* empty */ { $$ = null; }
| OP_GENERICS_BACKTICK DIGIT { $$ = Enumerable.Repeat<EcmaDesc> (null, (int)$2).ToList (); }
| OP_GENERICS_LT generic_type_arg_list OP_GENERICS_GT { $$ = $2; }
generic_type_arg_list
: type_expression { $$ = new List<EcmaDesc> () { (EcmaDesc)$1 }; }
| generic_type_arg_list COMMA type_expression { ((List<EcmaDesc>)$1).Add ((EcmaDesc)$3); $$ = $1; }
opt_array_definition
: /* empty */ { $$ = null; }
| OP_ARRAY_OPEN opt_array_definition_list OP_ARRAY_CLOSE opt_array_definition {
var dims = ((IList<int>)$4) ?? new List<int> (2);
dims.Add ((int)$2);
$$ = dims;
}
opt_array_definition_list
: /* empty */ { $$ = 1; }
| COMMA opt_array_definition_list { $$ = ((int)$2) + 1; }
opt_etc
: /* empty */ { $$ = null; }
| SLASH_SEPARATOR etc_identifier { $$ = Tuple.Create<char, string> (((string)$2)[0], null); }
| SLASH_SEPARATOR etc_identifier SLASH_SEPARATOR reduced_member_expression { $$ = Tuple.Create<char, string> (((string)$2)[0], (string)$4); }
/* | SLASH_SEPARATOR etc_identifier SLASH_SEPARATOR IDENTIFIER opt_generic_type_suffix { $$ = Tuple.Create<char, string> (((string)$2)[0], (string)$4 + ($5 == null ? string.Empty : "<" + string.Join (",", ((IEnumerable<EcmaDesc>)$5).Select (t => t.ToCompleteTypeName ())) + ">")); } */
etc_identifier
: STAR { $$ = "*"; }
| IDENTIFIER { $$ = $1; }
method_expression
: type_expression DOT IDENTIFIER opt_generic_type_suffix opt_arg_list_suffix {
var desc = $1 as EcmaDesc;
desc.MemberName = $3 as string;
desc.GenericMemberArguments = $4 as List<EcmaDesc>;
desc.MemberArguments = SafeReverse ($5 as List<EcmaDesc>);
$$ = desc;
}
| dot_expression opt_generic_type_suffix opt_arg_list_suffix {
var dotExpr = ((List<string>)$1);
$$ = new EcmaDesc {
Namespace = string.Join (".", dotExpr.Skip (2).DefaultIfEmpty (string.Empty).Reverse ()),
TypeName = dotExpr.Skip (1).First (),
MemberName = dotExpr.First (),
GenericMemberArguments = $2 as List<EcmaDesc>,
MemberArguments = SafeReverse ($3 as List<EcmaDesc>)
};
}
| type_expression EXPLICIT_IMPL_SEP method_expression {
var desc = $1 as EcmaDesc;
desc.ExplicitImplMember = $3 as EcmaDesc;
$$ = desc;
}
/* To be used with members that may have no type/namespace attached */
reduced_member_expression
: IDENTIFIER opt_generic_type_suffix { $$ = (string)$1 + ($2 == null ? string.Empty : "<" + string.Join (",", ((IEnumerable<EcmaDesc>)$2).Select (t => t.ToCompleteTypeName ())) + ">"); }
| IDENTIFIER opt_generic_type_suffix DOT reduced_member_expression {
var existing = $4 as string;
var expr = (string)$1 + ($2 == null ? string.Empty : "<" + string.Join (",", ((IEnumerable<EcmaDesc>)$2).Select (t => t.ToCompleteTypeName ())) + ">");
$$ = expr + "." + existing;
}
arg_type_expression
: type_expression opt_arg_type_suffix { var desc = (EcmaDesc)$1; desc.DescModifier = (EcmaDesc.Mod)$2; $$ = desc; }
opt_arg_type_suffix
: /* empty */ { $$ = EcmaDesc.Mod.Normal; }
| STAR { $$ = EcmaDesc.Mod.Pointer; }
| REF_ARG { $$ = EcmaDesc.Mod.Ref; }
| OUT_ARG { $$ = EcmaDesc.Mod.Out; }
type_expression_list
: /* empty */ { $$ = null; }
| arg_type_expression { $$ = new List<EcmaDesc> () { (EcmaDesc)$1 }; }
| arg_type_expression COMMA type_expression_list { ((List<EcmaDesc>)$3).Add ((EcmaDesc)$1); $$ = $3; }
simple_member_expression
: dot_expression {
var dotExpr = ((List<string>)$1);
dotExpr.Reverse ();
$$ = new EcmaDesc {
Namespace = dotExpr.Count > 2 ? string.Join (".", dotExpr.Take (dotExpr.Count - 2)) : string.Empty,
TypeName = dotExpr.Count > 1 ? dotExpr[dotExpr.Count - 2] : string.Empty,
MemberName = dotExpr[dotExpr.Count - 1]
};
}
| type_expression DOT IDENTIFIER {
var desc = $1 as EcmaDesc;
desc.MemberName = $3 as string;
$$ = desc;
}
| type_expression EXPLICIT_IMPL_SEP simple_member_expression {
var desc = $1 as EcmaDesc;
desc.ExplicitImplMember = $3 as EcmaDesc;
$$ = desc;
}
constructor_expression
: method_expression { $$ = $1; }
operator_expression
: method_expression { $$ = $1; }
property_expression
: simple_member_expression opt_property_indexer {
var desc = $1 as EcmaDesc;
(desc.ExplicitImplMember ?? desc).MemberArguments = SafeReverse ($2 as List<EcmaDesc>);
$$ = desc;
}
opt_property_indexer
: opt_arg_list_suffix { $$ = $1; }
/*simple_member_expression opt_arg_list_suffix { $$ = CopyFromEcmaDesc (new EcmaDesc {
MemberArguments = SafeReverse ($2 as List<EcmaDesc>)
}, (EcmaDesc)$1);
}*/
opt_arg_list_suffix
: /* empty */ { $$ = null; }
| OP_OPEN_PAREN type_expression_list OP_CLOSE_PAREN { $$ = $2; }
%%
}

View File

@@ -0,0 +1,17 @@
using System;
using System.IO;
namespace Monodoc.Ecma
{
public class EcmaUrlParserDriver
{
public static void Main (string[] args)
{
var input = new StringReader (args[0]);
var lexer = new EcmaUrlTokenizer (input);
var parser = new EcmaUrlParser ();
Console.WriteLine (parser.yyparse (lexer));
}
}
}

View File

@@ -0,0 +1,171 @@
using System;
using System.Text;
using System.Globalization;
namespace Monodoc.Ecma
{
public class EcmaUrlTokenizer : yyParser.yyInput
{
const char EndOfStream = (char)0;
string input;
object val;
int current_token;
int current_pos;
int real_current_pos;
int identCount = 0;
public EcmaUrlTokenizer (string input)
{
this.input = input;
}
static bool is_identifier_start_character (char c)
{
return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || Char.IsLetter (c);
}
static bool is_identifier_part_character (char c)
{
if (c >= 'a' && c <= 'z')
return true;
if (c >= 'A' && c <= 'Z')
return true;
if (c == '_' || (c >= '0' && c <= '9'))
return true;
if (c < 0x80)
return false;
return Char.IsLetter (c) || Char.GetUnicodeCategory (c) == UnicodeCategory.ConnectorPunctuation;
}
public bool advance ()
{
return Peek () != EndOfStream;
}
public Object Value {
get {
return val;
}
}
public Object value ()
{
return val;
}
public int token ()
{
int token = xtoken ();
//Console.WriteLine ("Current token {0} with value {1}", token, val == null ? "(none)" : val.ToString ());
if (token == Token.ERROR) {
throw new Exception (string.Format ("Error at position {0} parsing url '{0}'", current_pos, input));
}
current_token = token;
return token;
}
int xtoken ()
{
char next = Read ();
while (char.IsWhiteSpace (next))
next = Read ();
current_pos++;
val = null;
switch (next) {
case ',':
return Token.COMMA;
case '.':
return Token.DOT;
case '{':
case '<':
return Token.OP_GENERICS_LT;
case '}':
case '>':
return Token.OP_GENERICS_GT;
case '`':
return Token.OP_GENERICS_BACKTICK;
case '(':
return Token.OP_OPEN_PAREN;
case ')':
return Token.OP_CLOSE_PAREN;
case '+':
return Token.INNER_TYPE_SEPARATOR;
case ':':
return Token.COLON;
case '/':
return Token.SLASH_SEPARATOR;
case '[':
return Token.OP_ARRAY_OPEN;
case ']':
return Token.OP_ARRAY_CLOSE;
case '*':
return Token.STAR;
case '&':
return Token.REF_ARG;
case '@':
return Token.OUT_ARG;
case '$':
return Token.EXPLICIT_IMPL_SEP;
default:
return TokenizeIdentifierOrNumber (next);
}
}
int TokenizeIdentifierOrNumber (char current)
{
// We must first return the expression type which is a uppercase letter and a colon
if (current_pos < 2) {
val = null;
return (int)current;
}
if (is_identifier_start_character (current) || current == '*') {
unsafe {
// identifier length is artificially limited to 1024 bytes by implementations
char* pIdent = stackalloc char[512];
*pIdent = current;
identCount = 1;
char peek;
while ((peek = Peek ()) != EndOfStream && is_identifier_part_character (peek)) {
*(pIdent + identCount) = Read ();
++current_pos;
++identCount;
}
val = new string ((char*)pIdent, 0, identCount);
return Token.IDENTIFIER;
}
} else if (char.IsDigit (current)) {
val = current - '0';
return Token.DIGIT;
} else {
val = null;
return Token.ERROR;
}
}
char Read ()
{
try {
return input[real_current_pos++];
} catch {
return EndOfStream;
}
}
char Peek ()
{
try {
return input[real_current_pos];
} catch {
return EndOfStream;
}
}
}
}

File diff suppressed because it is too large Load Diff