24 writes to Tokens
Microsoft.ML.Tokenizers (23)
Model\BPETokenizer.cs (4)
317
return new EncodeResults<EncodedToken> {
Tokens
= [], NormalizedText = null, CharsConsumed = 0 };
346
return new EncodeResults<EncodedToken> {
Tokens
= tokens, NormalizedText = normalizedText, CharsConsumed = charsConsumed };
366
return new EncodeResults<int> {
Tokens
= [], NormalizedText = null, CharsConsumed = 0 };
402
return new EncodeResults<int> {
Tokens
= ids, NormalizedText = normalizedText, CharsConsumed = charsConsumed };
Model\CodeGenTokenizer.cs (3)
328
return new EncodeResults<EncodedToken> {
Tokens
= [], NormalizedText = null, CharsConsumed = 0 };
402
return new EncodeResults<EncodedToken> {
Tokens
= tokens, NormalizedText = normalizedText, CharsConsumed = textSpanToEncode.Length };
492
Tokens
= EncodeToIds(text, textSpan, AddPrefixSpace, AddBeginningOfSentence, AddEndOfSentence, settings.ConsiderPreTokenization, settings.ConsiderNormalization,
Model\EnglishRobertaTokenizer.cs (5)
318
return new EncodeResults<EncodedToken> {
Tokens
= [], NormalizedText = null, CharsConsumed = 0 };
343
return new EncodeResults<EncodedToken> {
Tokens
= tokens, NormalizedText = normalizedText, CharsConsumed = charsConsumed };
347
return new EncodeResults<EncodedToken> {
Tokens
= EncodeInternal(textSpanToEncode), NormalizedText = normalizedText, CharsConsumed = charsConsumed };
419
return new EncodeResults<int> {
Tokens
= [], NormalizedText = null, CharsConsumed = 0 };
455
return new EncodeResults<int> {
Tokens
= ids, NormalizedText = normalizedText, CharsConsumed = textLength };
Model\SentencePieceTokenizer.cs (2)
200
Tokens
= EncodeToTokens(text, textSpan, out string? normalizedText, AddBeginningOfSentence, AddEndOfSentence, settings.ConsiderPreTokenization, settings.ConsiderNormalization),
457
Tokens
= EncodeToIds(text, textSpan, AddBeginningOfSentence, AddEndOfSentence, settings.ConsiderNormalization, out string? normalizedText, out int charsConsumed, settings.MaxTokenCount),
Model\TiktokenTokenizer.cs (4)
262
return new EncodeResults<EncodedToken> { NormalizedText = null,
Tokens
= [], CharsConsumed = 0 };
290
return new EncodeResults<EncodedToken> { NormalizedText = normalizedText,
Tokens
= tokens, CharsConsumed = charsConsumed };
372
return new EncodeResults<int> { NormalizedText = null,
Tokens
= [], CharsConsumed = 0 };
407
return new EncodeResults<int> { NormalizedText = normalizedText,
Tokens
= ids, CharsConsumed = charsConsumed };
Model\WordPieceTokenizer.cs (4)
276
return new EncodeResults<EncodedToken> { NormalizedText = null,
Tokens
= [], CharsConsumed = 0 };
304
return new EncodeResults<EncodedToken> { NormalizedText = normalizedText,
Tokens
= tokens, CharsConsumed = charsConsumed };
399
return new EncodeResults<int> { NormalizedText = null,
Tokens
= [], CharsConsumed = 0 };
435
return new EncodeResults<int> { NormalizedText = normalizedText,
Tokens
= ids, CharsConsumed = charsConsumed };
Tokenizer.cs (1)
54
Tokens
= ids,
Microsoft.ML.Tokenizers.Tests (1)
TokenizerTests.cs (1)
119
return new EncodeResults<EncodedToken> {
Tokens
= tokens, CharsConsumed = count };
15 references to Tokens
Microsoft.ML.Tokenizers (15)
Model\CodeGenTokenizer.cs (2)
303
return result.
Tokens
;
321
return result.
Tokens
;
Tokenizer.cs (13)
46
var ids = new int[results.
Tokens
.Count];
49
ids[i] = results.
Tokens
[i].Id;
68
=> EncodeToIds(text, text.AsSpan(), new EncodeSettings { ConsiderPreTokenization = considerPreTokenization, ConsiderNormalization = considerNormalization }).
Tokens
;
78
=> EncodeToIds(null, text, new EncodeSettings { ConsiderPreTokenization = considerPreTokenization, ConsiderNormalization = considerNormalization }).
Tokens
;
103
return result.
Tokens
;
129
return result.
Tokens
;
153
return result.
Tokens
;
169
return result.
Tokens
;
184
=> EncodeToTokens(text, textSpan, settings).
Tokens
.Count;
237
tokenCount = Math.Min(maxTokenCount, tokens.
Tokens
.Count);
243
var token = tokens.
Tokens
[tokenCount - 1];
253
var token = tokens.
Tokens
[tokens.
Tokens
.Count - tokenCount];