2 writes to Normalizer
Microsoft.ML.Tokenizers (2)
Model\SentencePieceBaseModel.cs (2)
54Normalizer = new SentencePieceNormalizer( 100Normalizer = new SentencePieceNormalizer(
12 references to Normalizer
Microsoft.ML.Tokenizers (12)
Model\SentencePieceBpeModel.cs (9)
76if (considerNormalization && Normalizer is not null) 78normalizedText = text is not null ? Normalizer.Normalize(text) : Normalizer.Normalize(textSpan); 327if (considerNormalization && Normalizer is not null) 329normalizedText = Normalizer.Normalize(text); 619if (considerNormalization && Normalizer is not null) 621normalizedText = Normalizer.Normalize(textSpan); 869if (considerNormalization && Normalizer is not null) 871normalizedText = Normalizer.Normalize(textSpan);
Model\SentencePieceTokenizer.cs (2)
112public override Normalizer? Normalizer => _model.Normalizer; 754model.Normalizer!.NormalizationChain = chainNormalizer;
Model\SentencePieceUnigramModel.cs (1)
582int normalizationCount = Normalizer!.Normalize(byteSpan.Slice(0, byteCount), ref normalizationSpan, ref normalizedArrayPool);