1 write to _normalizer
Microsoft.ML.Tokenizers (1)
Model\SentencePieceTokenizer.cs (1)
76_normalizer = new SentencePieceNormalizer(modelProto.NormalizerSpec.RemoveExtraWhitespaces, AddDummyPrefix, EscapeWhiteSpaces, modelProto.TrainerSpec.TreatWhitespaceAsSuffix, specialTokens);
10 references to _normalizer
Microsoft.ML.Tokenizers (10)
Model\SentencePieceTokenizer.cs (10)
164public override Normalizer? Normalizer => _normalizer; 241if (considerNormalization && _normalizer is not null) 243normalizedText = text is not null ? _normalizer.Normalize(text) : _normalizer.Normalize(textSpan); 563if (considerNormalization && _normalizer is not null) 565normalizedText = _normalizer.Normalize(text); 911if (considerNormalization && _normalizer is not null) 913normalizedText = _normalizer.Normalize(text); 1264if (considerNormalization && _normalizer is not null) 1266normalizedText = _normalizer.Normalize(text);