9 references to DummyPrefix
Microsoft.ML.Tokenizers (9)
Model\SentencePieceBaseModel.cs (4)
209char prefixSuffixChar = EscapeWhiteSpaces ? SentencePieceNormalizer.DummyPrefix : ' '; 301Debug.Assert(sb[suffixIndex] == SentencePieceNormalizer.DummyPrefix); 317return EscapeWhiteSpaces ? sb.ToString(SentencePieceNormalizer.DummyPrefix, ' ') : sb.ToString(); 434char prefixSuffixChar = EscapeWhiteSpaces ? SentencePieceNormalizer.DummyPrefix : ' ';
Normalizer\SentencePieceNormalizer.cs (5)
200span[bufferIndex++] = EscapeWhiteSpaces ? DummyPrefix : ' '; 211span[bufferIndex++] = EscapeWhiteSpaces ? DummyPrefix : c; 236span[bufferIndex++] = EscapeWhiteSpaces ? DummyPrefix : ' '; 275span[bufferIndex++] = EscapeWhiteSpaces ? DummyPrefix : ' '; 314span[currentBufferIndex + 1] = EscapeWhiteSpaces ? DummyPrefix : ' ';