9 types derived from Token
Microsoft.ML.Transforms (9)
Expression\Tokens.cs (9)
65
internal sealed class KeyToken :
Token
85
internal sealed class IdentToken :
Token
109
internal abstract class NumLitToken :
Token
196
internal sealed class CharLitToken :
Token
210
internal sealed class StrLitToken :
Token
243
internal sealed class CommentToken :
Token
256
internal sealed class NewLineToken :
Token
267
internal sealed class ErrorToken :
Token
285
internal sealed class EofToken :
Token
83 references to Token
Microsoft.ML.Tests (1)
ExpressionLanguageTests\ExpressionLanguageTests.cs (1)
411
var
tok = err.Token;
Microsoft.ML.Transforms (82)
Expression\Error.cs (3)
12
public readonly
Token
Token;
17
public Error(
Token
tok, string msg)
26
public Error(
Token
tok, string msg, params object[] args)
Expression\LambdaParser.cs (25)
207
private void PostError(
Token
tok, string msg)
213
private void PostError(
Token
tok, string msg, params object[] args)
219
private void PostTidError(
Token
tok, TokKind tidWanted)
226
private string Stringize(
Token
tok)
270
private
Token
TokCur
275
private
Token
TokPeek(int cv = 1)
281
var
tok = _curs.TokPeek(++ctok);
334
private
Token
TokMove()
336
Token
tok = TokCur;
356
private
Token
TokEat(TokKind tid)
365
private LambdaNode ParseLambda(
Token
tokFirst)
389
Token
tok = TidCur == TokKind.Colon ? TokMove() : TokEat(TokKind.EquGrt);
398
Token
tok = TokCur;
630
Token
tok = TokCur;
632
List<
Token
> ops = new List<
Token
>();
682
Token
tok = TokMove();
695
Token
tokDot = TokMove();
698
Token
tokParen = TokMove();
713
private ListNode ParseList(
Token
tok, TokKind tidEmpty)
718
List<
Token
> commas = null;
732
private WithNode ParseWith(
Token
tokWith = null)
734
Token
tok;
735
Token
tokOpen;
773
Token
tok = TokCur;
Expression\Lexer.cs (19)
34
public IEnumerable<
Token
> LexSource(CharCursor cursor)
39
Token
tok;
52
private readonly Queue<
Token
> _queue; // For multiple returns.
63
_queue = new Queue<
Token
>(4);
136
public
Token
GetNextToken()
146
Token
tokNew = FetchToken();
152
Token
tok = _queue.Dequeue();
166
private
Token
FetchToken()
211
private
Token
LexPunc()
248
private
Token
LexNumLit()
314
private
Token
LexHexInt()
340
private
Token
LexDecInt(IntLitKind ilk)
362
private
Token
LexRealNum(char chSuf)
457
private
Token
LexStrLit()
651
private
Token
LexIdent()
719
private
Token
LexComment()
782
private
Token
LexSpace()
823
private
Token
LexPreProc()
833
private
Token
LexError()
Expression\Node.cs (25)
194
public readonly
Token
Token;
196
protected Node(
Token
tok)
266
protected ExprNode(
Token
tok)
607
public LambdaNode(
Token
tok, ParamNode[] vars, ExprNode expr)
650
public ParamNode(
Token
tok, string name, int index, DataViewType type)
727
public IdentNode(
Token
tok, string value, bool missing = false)
834
public BoolLitNode(
Token
tok)
860
public UnaryOpNode(
Token
tok, UnaryOp op, ExprNode arg)
892
public BinaryOpNode(
Token
tok, BinaryOp op, ExprNode left, ExprNode right)
925
public readonly
Token
TokColon;
928
public ConditionalNode(
Token
tok, ExprNode cond, ExprNode left,
Token
tokColon, ExprNode right)
961
public readonly
Token
[] Delimiters;
964
public ListNode(
Token
tok, Node[] items,
Token
[] delimiters)
997
public readonly
Token
Dot;
1002
public readonly
Token
CloseToken;
1006
public CallNode(
Token
tok, NameNode head, ListNode args,
Token
tokClose)
1017
public CallNode(
Token
tok, NameNode ns,
Token
dot, NameNode head, ListNode args,
Token
tokClose)
1076
public CompareNode(
Token
tok, CompareOp op, ListNode operands)
1140
public WithNode(
Token
tok, WithLocalNode local, ExprNode body)
1183
public WithLocalNode(
Token
tok, string name, ExprNode value)
Expression\TokenCursor.cs (8)
18
private IEnumerator<
Token
> _tokens;
21
private
Token
[] _buffer;
31
private
Token
_tokCur;
38
public TokenCursor(IEnumerable<
Token
> tokens)
42
_buffer = new
Token
[0x0400];
70
public
Token
TokCur
144
Token
tok = _buffer[_itokLim++] = _tokens.Current;
184
public
Token
TokPeek(int ditok)
Expression\Tokens.cs (2)
52
public T As<T>() where T :
Token
72
public static
Token
CreateKeyWord(TextSpan span, string str, TokKind tid, bool isContextKeyWord)