9 types derived from Token
Microsoft.ML.Transforms (9)
Expression\Tokens.cs (9)
65internal sealed class KeyToken : Token 85internal sealed class IdentToken : Token 109internal abstract class NumLitToken : Token 196internal sealed class CharLitToken : Token 210internal sealed class StrLitToken : Token 243internal sealed class CommentToken : Token 256internal sealed class NewLineToken : Token 267internal sealed class ErrorToken : Token 285internal sealed class EofToken : Token
83 references to Token
Microsoft.ML.Tests (1)
ExpressionLanguageTests\ExpressionLanguageTests.cs (1)
411var tok = err.Token;
Microsoft.ML.Transforms (82)
Expression\Error.cs (3)
12public readonly Token Token; 17public Error(Token tok, string msg) 26public Error(Token tok, string msg, params object[] args)
Expression\LambdaParser.cs (25)
207private void PostError(Token tok, string msg) 213private void PostError(Token tok, string msg, params object[] args) 219private void PostTidError(Token tok, TokKind tidWanted) 226private string Stringize(Token tok) 270private Token TokCur 275private Token TokPeek(int cv = 1) 281var tok = _curs.TokPeek(++ctok); 334private Token TokMove() 336Token tok = TokCur; 356private Token TokEat(TokKind tid) 365private LambdaNode ParseLambda(Token tokFirst) 389Token tok = TidCur == TokKind.Colon ? TokMove() : TokEat(TokKind.EquGrt); 398Token tok = TokCur; 630Token tok = TokCur; 632List<Token> ops = new List<Token>(); 682Token tok = TokMove(); 695Token tokDot = TokMove(); 698Token tokParen = TokMove(); 713private ListNode ParseList(Token tok, TokKind tidEmpty) 718List<Token> commas = null; 732private WithNode ParseWith(Token tokWith = null) 734Token tok; 735Token tokOpen; 773Token tok = TokCur;
Expression\Lexer.cs (19)
34public IEnumerable<Token> LexSource(CharCursor cursor) 39Token tok; 52private readonly Queue<Token> _queue; // For multiple returns. 63_queue = new Queue<Token>(4); 136public Token GetNextToken() 146Token tokNew = FetchToken(); 152Token tok = _queue.Dequeue(); 166private Token FetchToken() 211private Token LexPunc() 248private Token LexNumLit() 314private Token LexHexInt() 340private Token LexDecInt(IntLitKind ilk) 362private Token LexRealNum(char chSuf) 457private Token LexStrLit() 651private Token LexIdent() 719private Token LexComment() 782private Token LexSpace() 823private Token LexPreProc() 833private Token LexError()
Expression\Node.cs (25)
194public readonly Token Token; 196protected Node(Token tok) 266protected ExprNode(Token tok) 607public LambdaNode(Token tok, ParamNode[] vars, ExprNode expr) 650public ParamNode(Token tok, string name, int index, DataViewType type) 727public IdentNode(Token tok, string value, bool missing = false) 834public BoolLitNode(Token tok) 860public UnaryOpNode(Token tok, UnaryOp op, ExprNode arg) 892public BinaryOpNode(Token tok, BinaryOp op, ExprNode left, ExprNode right) 925public readonly Token TokColon; 928public ConditionalNode(Token tok, ExprNode cond, ExprNode left, Token tokColon, ExprNode right) 961public readonly Token[] Delimiters; 964public ListNode(Token tok, Node[] items, Token[] delimiters) 997public readonly Token Dot; 1002public readonly Token CloseToken; 1006public CallNode(Token tok, NameNode head, ListNode args, Token tokClose) 1017public CallNode(Token tok, NameNode ns, Token dot, NameNode head, ListNode args, Token tokClose) 1076public CompareNode(Token tok, CompareOp op, ListNode operands) 1140public WithNode(Token tok, WithLocalNode local, ExprNode body) 1183public WithLocalNode(Token tok, string name, ExprNode value)
Expression\TokenCursor.cs (8)
18private IEnumerator<Token> _tokens; 21private Token[] _buffer; 31private Token _tokCur; 38public TokenCursor(IEnumerable<Token> tokens) 42_buffer = new Token[0x0400]; 70public Token TokCur 144Token tok = _buffer[_itokLim++] = _tokens.Current; 184public Token TokPeek(int ditok)
Expression\Tokens.cs (2)
52public T As<T>() where T : Token 72public static Token CreateKeyWord(TextSpan span, string str, TokKind tid, bool isContextKeyWord)