2 writes to Normalizer
Microsoft.ML.Tokenizers (2)
Model\SentencePieceBaseModel.cs (2)
54Normalizer = new SentencePieceNormalizer( 115Normalizer = new SentencePieceNormalizer(
11 references to Normalizer
Microsoft.ML.Tokenizers (11)
Model\SentencePieceBpeModel.cs (9)
122if (considerNormalization && Normalizer is not null) 124normalizedText = text is not null ? Normalizer.Normalize(text) : Normalizer.Normalize(textSpan); 373if (considerNormalization && Normalizer is not null) 375normalizedText = Normalizer.Normalize(text); 665if (considerNormalization && Normalizer is not null) 667normalizedText = Normalizer.Normalize(textSpan); 915if (considerNormalization && Normalizer is not null) 917normalizedText = Normalizer.Normalize(textSpan);
Model\SentencePieceTokenizer.cs (1)
116public override Normalizer? Normalizer => _model.Normalizer;
Model\SentencePieceUnigramModel.cs (1)
357int normalizationCount = Normalizer!.Normalize(byteSpan.Slice(0, byteCount), ref normalizationSpan, ref normalizedArrayPool);