1 write to _normalizer
Microsoft.ML.Tokenizers (1)
Model\SentencePieceTokenizer.cs (1)
76
_normalizer
= new SentencePieceNormalizer(modelProto.NormalizerSpec.RemoveExtraWhitespaces, AddDummyPrefix, EscapeWhiteSpaces, modelProto.TrainerSpec.TreatWhitespaceAsSuffix, specialTokens);
10 references to _normalizer
Microsoft.ML.Tokenizers (10)
Model\SentencePieceTokenizer.cs (10)
163
public override Normalizer? Normalizer =>
_normalizer
;
240
if (considerNormalization &&
_normalizer
is not null)
242
normalizedString = text is not null ?
_normalizer
.Normalize(text) :
_normalizer
.Normalize(textSpan);
562
if (considerNormalization &&
_normalizer
is not null)
564
normalizedString =
_normalizer
.Normalize(text);
910
if (considerNormalization &&
_normalizer
is not null)
912
normalizedString =
_normalizer
.Normalize(text);
1263
if (considerNormalization &&
_normalizer
is not null)
1265
normalizedString =
_normalizer
.Normalize(text);