1 write to Normalizer
Microsoft.ML.Tokenizers (1)
Model\SentencePieceBaseModel.cs (1)
54Normalizer = new SentencePieceNormalizer(
11 references to Normalizer
Microsoft.ML.Tokenizers (11)
Model\SentencePieceBpeModel.cs (9)
76if (considerNormalization && Normalizer is not null) 78normalizedText = text is not null ? Normalizer.Normalize(text) : Normalizer.Normalize(textSpan); 327if (considerNormalization && Normalizer is not null) 329normalizedText = Normalizer.Normalize(text); 619if (considerNormalization && Normalizer is not null) 621normalizedText = Normalizer.Normalize(textSpan); 869if (considerNormalization && Normalizer is not null) 871normalizedText = Normalizer.Normalize(textSpan);
Model\SentencePieceTokenizer.cs (1)
106public override Normalizer? Normalizer => _model.Normalizer;
Model\SentencePieceUnigramModel.cs (1)
307int normalizationCount = Normalizer!.Normalize(byteSpan.Slice(0, byteCount), ref normalizationSpan, ref normalizedArrayPool);