23 writes to NormalizedText
Microsoft.ML.Tokenizers (23)
Model\BPETokenizer.cs (4)
317return new EncodeResults<EncodedToken> { Tokens = [], NormalizedText = null, CharsConsumed = 0 }; 346return new EncodeResults<EncodedToken> { Tokens = tokens, NormalizedText = normalizedText, CharsConsumed = charsConsumed }; 366return new EncodeResults<int> { Tokens = [], NormalizedText = null, CharsConsumed = 0 }; 402return new EncodeResults<int> { Tokens = ids, NormalizedText = normalizedText, CharsConsumed = charsConsumed };
Model\CodeGenTokenizer.cs (3)
328return new EncodeResults<EncodedToken> { Tokens = [], NormalizedText = null, CharsConsumed = 0 }; 402return new EncodeResults<EncodedToken> { Tokens = tokens, NormalizedText = normalizedText, CharsConsumed = textSpanToEncode.Length }; 494NormalizedText = normalizedText,
Model\EnglishRobertaTokenizer.cs (5)
318return new EncodeResults<EncodedToken> { Tokens = [], NormalizedText = null, CharsConsumed = 0 }; 343return new EncodeResults<EncodedToken> { Tokens = tokens, NormalizedText = normalizedText, CharsConsumed = charsConsumed }; 347return new EncodeResults<EncodedToken> { Tokens = EncodeInternal(textSpanToEncode), NormalizedText = normalizedText, CharsConsumed = charsConsumed }; 419return new EncodeResults<int> { Tokens = [], NormalizedText = null, CharsConsumed = 0 }; 455return new EncodeResults<int> { Tokens = ids, NormalizedText = normalizedText, CharsConsumed = textLength };
Model\SentencePieceTokenizer.cs (2)
201NormalizedText = normalizedText, 458NormalizedText = normalizedText,
Model\TiktokenTokenizer.cs (4)
262return new EncodeResults<EncodedToken> { NormalizedText = null, Tokens = [], CharsConsumed = 0 }; 290return new EncodeResults<EncodedToken> { NormalizedText = normalizedText, Tokens = tokens, CharsConsumed = charsConsumed }; 372return new EncodeResults<int> { NormalizedText = null, Tokens = [], CharsConsumed = 0 }; 407return new EncodeResults<int> { NormalizedText = normalizedText, Tokens = ids, CharsConsumed = charsConsumed };
Model\WordPieceTokenizer.cs (4)
276return new EncodeResults<EncodedToken> { NormalizedText = null, Tokens = [], CharsConsumed = 0 }; 304return new EncodeResults<EncodedToken> { NormalizedText = normalizedText, Tokens = tokens, CharsConsumed = charsConsumed }; 399return new EncodeResults<int> { NormalizedText = null, Tokens = [], CharsConsumed = 0 }; 435return new EncodeResults<int> { NormalizedText = normalizedText, Tokens = ids, CharsConsumed = charsConsumed };
Tokenizer.cs (1)
56NormalizedText = results.NormalizedText,
9 references to NormalizedText
Microsoft.ML.Tokenizers (9)
Model\CodeGenTokenizer.cs (2)
302normalizedText = result.NormalizedText; 320normalizedText = result.NormalizedText;
Tokenizer.cs (7)
56NormalizedText = results.NormalizedText, 100normalizedText = result.NormalizedText; 126normalizedText = result.NormalizedText; 152normalizedText = result.NormalizedText; 168normalizedText = result.NormalizedText; 236normalizedText = tokens.NormalizedText; 257return tokens.NormalizedText?.Length ?? textSpan.Length;