1 write to AddDummyPrefix
Microsoft.ML.Tokenizers (1)
Normalizer\SentencePieceNormalizer.cs (1)
25AddDummyPrefix = addDummyPrefix;
3 references to AddDummyPrefix
Microsoft.ML.Tokenizers (3)
Normalizer\SentencePieceNormalizer.cs (3)
104int spanLength = AddDummyPrefix ? length + 1 : length; 116if (AddDummyPrefix && !TreatWhitespaceAsSuffix) 151if (AddDummyPrefix && TreatWhitespaceAsSuffix)