1 write to _charsMap
Microsoft.ML.Tokenizers (1)
Normalizer\SentencePieceNormalizationStep.cs (1)
479
_charsMap
= new SentencePieceNormalizer(
2 references to _charsMap
Microsoft.ML.Tokenizers (2)
Normalizer\SentencePieceNormalizationStep.cs (2)
489
public override string Normalize(string text) =>
_charsMap
is null ? text :
_charsMap
.NormalizeUtf8ToString(text);