1 write to TreatWhitespaceAsSuffix
Microsoft.ML.Tokenizers (1)
Normalizer\SentencePieceNormalizer.cs (1)
36TreatWhitespaceAsSuffix = treatWhitespaceAsSuffix;
4 references to TreatWhitespaceAsSuffix
Microsoft.ML.Tokenizers (4)
Normalizer\SentencePieceNormalizer.cs (4)
144if (AddDummyPrefix && !TreatWhitespaceAsSuffix) 179if (AddDummyPrefix && TreatWhitespaceAsSuffix) 379if (!TreatWhitespaceAsSuffix && AddDummyPrefix) 455if (TreatWhitespaceAsSuffix && AddDummyPrefix)