2 references to ReplaceLoneSurrogates
Microsoft.ML.Tokenizers (2)
Normalizer\SentencePieceNormalizationStep.cs (2)
282
string decomposed =
ReplaceLoneSurrogates
(text).Normalize(NormalizationForm.FormD);
314
public override string Normalize(string text) => text.Length == 0 ? text :
ReplaceLoneSurrogates
(text).Normalize(_form);