6 overrides of Normalize
Microsoft.ML.Tokenizers (4)
Normalizer\BertNormalizer.cs (1)
126public override string Normalize(ReadOnlySpan<char> original)
Normalizer\LowerCaseNormalizer.cs (1)
38public override string Normalize(ReadOnlySpan<char> original)
Normalizer\SentencePieceNormalizer.cs (1)
76public override string Normalize(ReadOnlySpan<char> original)
Normalizer\UpperCaseNormalizer.cs (1)
38public override string Normalize(ReadOnlySpan<char> original)
Microsoft.ML.Tokenizers.Tests (2)
NormalizerTests.cs (2)
82public override string Normalize(ReadOnlySpan<char> original) 151public override string Normalize(ReadOnlySpan<char> original)
4 references to Normalize
Microsoft.ML.Tokenizers (4)
Model\SentencePieceTokenizer.cs (4)
243normalizedText = text is not null ? _normalizer.Normalize(text) : _normalizer.Normalize(textSpan); 565normalizedText = _normalizer.Normalize(text); 913normalizedText = _normalizer.Normalize(text); 1266normalizedText = _normalizer.Normalize(text);