1 write to AddDummyPrefix
Microsoft.ML.Tokenizers (1)
Normalizer\SentencePieceNormalizer.cs (1)
34
AddDummyPrefix
= addDummyPrefix;
5 references to AddDummyPrefix
Microsoft.ML.Tokenizers (5)
Normalizer\SentencePieceNormalizer.cs (5)
132
int spanLength =
AddDummyPrefix
? length + 1 : length;
144
if (
AddDummyPrefix
&& !TreatWhitespaceAsSuffix)
179
if (
AddDummyPrefix
&& TreatWhitespaceAsSuffix)
379
if (!TreatWhitespaceAsSuffix &&
AddDummyPrefix
)
455
if (TreatWhitespaceAsSuffix &&
AddDummyPrefix
)