1 write to EscapeWhiteSpaces
Microsoft.ML.Tokenizers (1)
Normalizer\SentencePieceNormalizer.cs (1)
35EscapeWhiteSpaces = escapeWhiteSpaces;
8 references to EscapeWhiteSpaces
Microsoft.ML.Tokenizers (8)
Normalizer\SentencePieceNormalizer.cs (8)
152span[bufferIndex++] = EscapeWhiteSpaces ? DummyPrefix : ' '; 163span[bufferIndex++] = EscapeWhiteSpaces ? DummyPrefix : c; 188span[bufferIndex++] = EscapeWhiteSpaces ? DummyPrefix : ' '; 227span[bufferIndex++] = EscapeWhiteSpaces ? DummyPrefix : ' '; 266span[currentBufferIndex + 1] = EscapeWhiteSpaces ? DummyPrefix : ' '; 401if (EscapeWhiteSpaces && sp[n] == ' ') 440Span<byte> space = EscapeWhiteSpaces ? _spaceSymbol : _space; 465if (normalizer.EscapeWhiteSpaces)