24 writes to CharsConsumed
Microsoft.ML.Tokenizers (23)
Model\BPETokenizer.cs (4)
317
return new EncodeResults<EncodedToken> { Tokens = [], NormalizedText = null,
CharsConsumed
= 0 };
346
return new EncodeResults<EncodedToken> { Tokens = tokens, NormalizedText = normalizedText,
CharsConsumed
= charsConsumed };
366
return new EncodeResults<int> { Tokens = [], NormalizedText = null,
CharsConsumed
= 0 };
402
return new EncodeResults<int> { Tokens = ids, NormalizedText = normalizedText,
CharsConsumed
= charsConsumed };
Model\CodeGenTokenizer.cs (3)
328
return new EncodeResults<EncodedToken> { Tokens = [], NormalizedText = null,
CharsConsumed
= 0 };
402
return new EncodeResults<EncodedToken> { Tokens = tokens, NormalizedText = normalizedText,
CharsConsumed
= textSpanToEncode.Length };
495
CharsConsumed
= charsConsumed
Model\EnglishRobertaTokenizer.cs (5)
318
return new EncodeResults<EncodedToken> { Tokens = [], NormalizedText = null,
CharsConsumed
= 0 };
343
return new EncodeResults<EncodedToken> { Tokens = tokens, NormalizedText = normalizedText,
CharsConsumed
= charsConsumed };
347
return new EncodeResults<EncodedToken> { Tokens = EncodeInternal(textSpanToEncode), NormalizedText = normalizedText,
CharsConsumed
= charsConsumed };
419
return new EncodeResults<int> { Tokens = [], NormalizedText = null,
CharsConsumed
= 0 };
455
return new EncodeResults<int> { Tokens = ids, NormalizedText = normalizedText,
CharsConsumed
= textLength };
Model\SentencePieceTokenizer.cs (2)
202
CharsConsumed
= normalizedText?.Length ?? text?.Length ?? textSpan.Length
459
CharsConsumed
= charsConsumed
Model\TiktokenTokenizer.cs (4)
262
return new EncodeResults<EncodedToken> { NormalizedText = null, Tokens = [],
CharsConsumed
= 0 };
290
return new EncodeResults<EncodedToken> { NormalizedText = normalizedText, Tokens = tokens,
CharsConsumed
= charsConsumed };
372
return new EncodeResults<int> { NormalizedText = null, Tokens = [],
CharsConsumed
= 0 };
407
return new EncodeResults<int> { NormalizedText = normalizedText, Tokens = ids,
CharsConsumed
= charsConsumed };
Model\WordPieceTokenizer.cs (4)
276
return new EncodeResults<EncodedToken> { NormalizedText = null, Tokens = [],
CharsConsumed
= 0 };
304
return new EncodeResults<EncodedToken> { NormalizedText = normalizedText, Tokens = tokens,
CharsConsumed
= charsConsumed };
399
return new EncodeResults<int> { NormalizedText = null, Tokens = [],
CharsConsumed
= 0 };
435
return new EncodeResults<int> { NormalizedText = normalizedText, Tokens = ids,
CharsConsumed
= charsConsumed };
Tokenizer.cs (1)
55
CharsConsumed
= results.CharsConsumed,
Microsoft.ML.Tokenizers.Tests (1)
TokenizerTests.cs (1)
119
return new EncodeResults<EncodedToken> { Tokens = tokens,
CharsConsumed
= count };
3 references to CharsConsumed
Microsoft.ML.Tokenizers (3)
Tokenizer.cs (3)
55
CharsConsumed = results.
CharsConsumed
,
101
charsConsumed = result.
CharsConsumed
;
127
charsConsumed = result.
CharsConsumed
;