6 overrides of Normalize
Microsoft.ML.Tokenizers (4)
Normalizer\BertNormalizer.cs (1)
126
public override string
Normalize
(ReadOnlySpan<char> original)
Normalizer\LowerCaseNormalizer.cs (1)
38
public override string
Normalize
(ReadOnlySpan<char> original)
Normalizer\SentencePieceNormalizer.cs (1)
76
public override string
Normalize
(ReadOnlySpan<char> original)
Normalizer\UpperCaseNormalizer.cs (1)
38
public override string
Normalize
(ReadOnlySpan<char> original)
Microsoft.ML.Tokenizers.Tests (2)
NormalizerTests.cs (2)
82
public override string
Normalize
(ReadOnlySpan<char> original)
151
public override string
Normalize
(ReadOnlySpan<char> original)
4 references to Normalize
Microsoft.ML.Tokenizers (4)
Model\SentencePieceTokenizer.cs (4)
243
normalizedText = text is not null ? _normalizer.Normalize(text) : _normalizer.
Normalize
(textSpan);
565
normalizedText = _normalizer.
Normalize
(text);
913
normalizedText = _normalizer.
Normalize
(text);
1266
normalizedText = _normalizer.
Normalize
(text);