1 write to AddDummyPrefix
Microsoft.ML.Tokenizers (1)
Normalizer\SentencePieceNormalizer.cs (1)
41AddDummyPrefix = addDummyPrefix;
5 references to AddDummyPrefix
Microsoft.ML.Tokenizers (5)
Normalizer\SentencePieceNormalizer.cs (5)
180int spanLength = AddDummyPrefix ? length + 1 : length; 192if (AddDummyPrefix && !TreatWhitespaceAsSuffix) 227if (AddDummyPrefix && TreatWhitespaceAsSuffix) 460if (!TreatWhitespaceAsSuffix && AddDummyPrefix) 536if (TreatWhitespaceAsSuffix && AddDummyPrefix)