1 write to _normalizer
Microsoft.ML.Tokenizers (1)
Model\SentencePieceTokenizer.cs (1)
76_normalizer = new SentencePieceNormalizer(modelProto.NormalizerSpec.RemoveExtraWhitespaces, AddDummyPrefix, EscapeWhiteSpaces, modelProto.TrainerSpec.TreatWhitespaceAsSuffix, specialTokens);
10 references to _normalizer
Microsoft.ML.Tokenizers (10)
Model\SentencePieceTokenizer.cs (10)
163public override Normalizer? Normalizer => _normalizer; 240if (considerNormalization && _normalizer is not null) 242normalizedString = text is not null ? _normalizer.Normalize(text) : _normalizer.Normalize(textSpan); 562if (considerNormalization && _normalizer is not null) 564normalizedString = _normalizer.Normalize(text); 910if (considerNormalization && _normalizer is not null) 912normalizedString = _normalizer.Normalize(text); 1263if (considerNormalization && _normalizer is not null) 1265normalizedString = _normalizer.Normalize(text);