24 writes to CharsConsumed
Microsoft.ML.Tokenizers (23)
Model\BPETokenizer.cs (4)
317return new EncodeResults<EncodedToken> { Tokens = [], NormalizedText = null, CharsConsumed = 0 }; 346return new EncodeResults<EncodedToken> { Tokens = tokens, NormalizedText = normalizedText, CharsConsumed = charsConsumed }; 366return new EncodeResults<int> { Tokens = [], NormalizedText = null, CharsConsumed = 0 }; 402return new EncodeResults<int> { Tokens = ids, NormalizedText = normalizedText, CharsConsumed = charsConsumed };
Model\CodeGenTokenizer.cs (3)
328return new EncodeResults<EncodedToken> { Tokens = [], NormalizedText = null, CharsConsumed = 0 }; 402return new EncodeResults<EncodedToken> { Tokens = tokens, NormalizedText = normalizedText, CharsConsumed = textSpanToEncode.Length }; 495CharsConsumed = charsConsumed
Model\EnglishRobertaTokenizer.cs (5)
318return new EncodeResults<EncodedToken> { Tokens = [], NormalizedText = null, CharsConsumed = 0 }; 343return new EncodeResults<EncodedToken> { Tokens = tokens, NormalizedText = normalizedText, CharsConsumed = charsConsumed }; 347return new EncodeResults<EncodedToken> { Tokens = EncodeInternal(textSpanToEncode), NormalizedText = normalizedText, CharsConsumed = charsConsumed }; 419return new EncodeResults<int> { Tokens = [], NormalizedText = null, CharsConsumed = 0 }; 455return new EncodeResults<int> { Tokens = ids, NormalizedText = normalizedText, CharsConsumed = textLength };
Model\SentencePieceTokenizer.cs (2)
202CharsConsumed = normalizedText?.Length ?? text?.Length ?? textSpan.Length 459CharsConsumed = charsConsumed
Model\TiktokenTokenizer.cs (4)
262return new EncodeResults<EncodedToken> { NormalizedText = null, Tokens = [], CharsConsumed = 0 }; 290return new EncodeResults<EncodedToken> { NormalizedText = normalizedText, Tokens = tokens, CharsConsumed = charsConsumed }; 372return new EncodeResults<int> { NormalizedText = null, Tokens = [], CharsConsumed = 0 }; 407return new EncodeResults<int> { NormalizedText = normalizedText, Tokens = ids, CharsConsumed = charsConsumed };
Model\WordPieceTokenizer.cs (4)
276return new EncodeResults<EncodedToken> { NormalizedText = null, Tokens = [], CharsConsumed = 0 }; 304return new EncodeResults<EncodedToken> { NormalizedText = normalizedText, Tokens = tokens, CharsConsumed = charsConsumed }; 399return new EncodeResults<int> { NormalizedText = null, Tokens = [], CharsConsumed = 0 }; 435return new EncodeResults<int> { NormalizedText = normalizedText, Tokens = ids, CharsConsumed = charsConsumed };
Tokenizer.cs (1)
55CharsConsumed = results.CharsConsumed,
Microsoft.ML.Tokenizers.Tests (1)
TokenizerTests.cs (1)
119return new EncodeResults<EncodedToken> { Tokens = tokens, CharsConsumed = count };
3 references to CharsConsumed
Microsoft.ML.Tokenizers (3)
Tokenizer.cs (3)
55CharsConsumed = results.CharsConsumed, 101charsConsumed = result.CharsConsumed; 127charsConsumed = result.CharsConsumed;