9 references to DummyPrefix
Microsoft.ML.Tokenizers (9)
Model\SentencePieceTokenizer.cs (4)
1558char prefixSuffixChar = EscapeWhiteSpaces ? SentencePieceNormalizer.DummyPrefix : ' '; 1658Debug.Assert(sb[suffixIndex] == SentencePieceNormalizer.DummyPrefix); 1674return EscapeWhiteSpaces ? sb.ToString(SentencePieceNormalizer.DummyPrefix, ' ') : sb.ToString(); 1807char prefixSuffixChar = EscapeWhiteSpaces ? SentencePieceNormalizer.DummyPrefix : ' ';
Normalizer\SentencePieceNormalizer.cs (5)
124span[bufferIndex++] = EscapeWhiteSpaces ? DummyPrefix : ' '; 135span[bufferIndex++] = EscapeWhiteSpaces ? DummyPrefix : c; 160span[bufferIndex++] = EscapeWhiteSpaces ? DummyPrefix : ' '; 199span[bufferIndex++] = EscapeWhiteSpaces ? DummyPrefix : ' '; 238span[currentBufferIndex + 1] = EscapeWhiteSpaces ? DummyPrefix : ' ';