1 write to EscapeWhiteSpaces
Microsoft.ML.Tokenizers (1)
Normalizer\SentencePieceNormalizer.cs (1)
42EscapeWhiteSpaces = escapeWhiteSpaces;
8 references to EscapeWhiteSpaces
Microsoft.ML.Tokenizers (8)
Normalizer\SentencePieceNormalizer.cs (8)
200span[bufferIndex++] = EscapeWhiteSpaces ? DummyPrefix : ' '; 211span[bufferIndex++] = EscapeWhiteSpaces ? DummyPrefix : c; 236span[bufferIndex++] = EscapeWhiteSpaces ? DummyPrefix : ' '; 275span[bufferIndex++] = EscapeWhiteSpaces ? DummyPrefix : ' '; 314span[currentBufferIndex + 1] = EscapeWhiteSpaces ? DummyPrefix : ' '; 482if (EscapeWhiteSpaces && sp[n] == ' ') 521Span<byte> space = EscapeWhiteSpaces ? _spaceSymbol : _space; 546if (normalizer.EscapeWhiteSpaces)