9 references to DummyPrefix
Microsoft.ML.Tokenizers (9)
Model\SentencePieceBaseModel.cs (4)
226char prefixSuffixChar = EscapeWhiteSpaces ? SentencePieceNormalizer.DummyPrefix : ' '; 318Debug.Assert(sb[suffixIndex] == SentencePieceNormalizer.DummyPrefix); 334return EscapeWhiteSpaces ? sb.ToString(SentencePieceNormalizer.DummyPrefix, ' ') : sb.ToString(); 451char prefixSuffixChar = EscapeWhiteSpaces ? SentencePieceNormalizer.DummyPrefix : ' ';
Normalizer\SentencePieceNormalizer.cs (5)
152span[bufferIndex++] = EscapeWhiteSpaces ? DummyPrefix : ' '; 163span[bufferIndex++] = EscapeWhiteSpaces ? DummyPrefix : c; 188span[bufferIndex++] = EscapeWhiteSpaces ? DummyPrefix : ' '; 227span[bufferIndex++] = EscapeWhiteSpaces ? DummyPrefix : ' '; 266span[currentBufferIndex + 1] = EscapeWhiteSpaces ? DummyPrefix : ' ';