9 overrides of Normalize
Microsoft.ML.Tokenizers (9)
Normalizer\SentencePieceNormalizationStep.cs (9)
254public override string Normalize(string text) 267public override string Normalize(string text) => text.ToLowerInvariant(); 275public override string Normalize(string text) 314public override string Normalize(string text) => text.Length == 0 ? text : ReplaceLoneSurrogates(text).Normalize(_form); 321public override string Normalize(string text) => _prefix.Length == 0 ? text : _prefix + text; 339public override string Normalize(string text) 385public override string Normalize(string text) 455public override string Normalize(string text) 489public override string Normalize(string text) => _charsMap is null ? text : _charsMap.NormalizeUtf8ToString(text);
3 references to Normalize
Microsoft.ML.Tokenizers (3)
Normalizer\SentencePieceNormalizationStep.cs (1)
258text = step.Normalize(text);
Normalizer\SentencePieceNormalizer.cs (2)
146return NormalizeCore(NormalizationChain.Normalize(original.ToString()).AsSpan()); 395string chained = NormalizationChain.Normalize(Helpers.GetString(input));