23 writes to NormalizedText
Microsoft.ML.Tokenizers (23)
Model\BPETokenizer.cs (4)
317
return new EncodeResults<EncodedToken> { Tokens = [],
NormalizedText
= null, CharsConsumed = 0 };
346
return new EncodeResults<EncodedToken> { Tokens = tokens,
NormalizedText
= normalizedText, CharsConsumed = charsConsumed };
366
return new EncodeResults<int> { Tokens = [],
NormalizedText
= null, CharsConsumed = 0 };
402
return new EncodeResults<int> { Tokens = ids,
NormalizedText
= normalizedText, CharsConsumed = charsConsumed };
Model\CodeGenTokenizer.cs (3)
328
return new EncodeResults<EncodedToken> { Tokens = [],
NormalizedText
= null, CharsConsumed = 0 };
402
return new EncodeResults<EncodedToken> { Tokens = tokens,
NormalizedText
= normalizedText, CharsConsumed = textSpanToEncode.Length };
494
NormalizedText
= normalizedText,
Model\EnglishRobertaTokenizer.cs (5)
318
return new EncodeResults<EncodedToken> { Tokens = [],
NormalizedText
= null, CharsConsumed = 0 };
343
return new EncodeResults<EncodedToken> { Tokens = tokens,
NormalizedText
= normalizedText, CharsConsumed = charsConsumed };
347
return new EncodeResults<EncodedToken> { Tokens = EncodeInternal(textSpanToEncode),
NormalizedText
= normalizedText, CharsConsumed = charsConsumed };
419
return new EncodeResults<int> { Tokens = [],
NormalizedText
= null, CharsConsumed = 0 };
455
return new EncodeResults<int> { Tokens = ids,
NormalizedText
= normalizedText, CharsConsumed = textLength };
Model\SentencePieceTokenizer.cs (2)
201
NormalizedText
= normalizedText,
458
NormalizedText
= normalizedText,
Model\TiktokenTokenizer.cs (4)
262
return new EncodeResults<EncodedToken> {
NormalizedText
= null, Tokens = [], CharsConsumed = 0 };
290
return new EncodeResults<EncodedToken> {
NormalizedText
= normalizedText, Tokens = tokens, CharsConsumed = charsConsumed };
372
return new EncodeResults<int> {
NormalizedText
= null, Tokens = [], CharsConsumed = 0 };
407
return new EncodeResults<int> {
NormalizedText
= normalizedText, Tokens = ids, CharsConsumed = charsConsumed };
Model\WordPieceTokenizer.cs (4)
276
return new EncodeResults<EncodedToken> {
NormalizedText
= null, Tokens = [], CharsConsumed = 0 };
304
return new EncodeResults<EncodedToken> {
NormalizedText
= normalizedText, Tokens = tokens, CharsConsumed = charsConsumed };
399
return new EncodeResults<int> {
NormalizedText
= null, Tokens = [], CharsConsumed = 0 };
435
return new EncodeResults<int> {
NormalizedText
= normalizedText, Tokens = ids, CharsConsumed = charsConsumed };
Tokenizer.cs (1)
56
NormalizedText
= results.NormalizedText,
9 references to NormalizedText
Microsoft.ML.Tokenizers (9)
Model\CodeGenTokenizer.cs (2)
302
normalizedText = result.
NormalizedText
;
320
normalizedText = result.
NormalizedText
;
Tokenizer.cs (7)
56
NormalizedText = results.
NormalizedText
,
100
normalizedText = result.
NormalizedText
;
126
normalizedText = result.
NormalizedText
;
152
normalizedText = result.
NormalizedText
;
168
normalizedText = result.
NormalizedText
;
236
normalizedText = tokens.
NormalizedText
;
257
return tokens.
NormalizedText
?.Length ?? textSpan.Length;