1 write to EscapeWhiteSpaces
Microsoft.ML.Tokenizers (1)
Model\SentencePieceTokenizer.cs (1)
71EscapeWhiteSpaces = modelProto.NormalizerSpec.EscapeWhitespaces;
5 references to EscapeWhiteSpaces
Microsoft.ML.Tokenizers (4)
Model\SentencePieceTokenizer.cs (4)
76_normalizer = new SentencePieceNormalizer(modelProto.NormalizerSpec.RemoveExtraWhitespaces, AddDummyPrefix, EscapeWhiteSpaces, modelProto.TrainerSpec.TreatWhitespaceAsSuffix, specialTokens); 1558char prefixSuffixChar = EscapeWhiteSpaces ? SentencePieceNormalizer.DummyPrefix : ' '; 1674return EscapeWhiteSpaces ? sb.ToString(SentencePieceNormalizer.DummyPrefix, ' ') : sb.ToString(); 1807char prefixSuffixChar = EscapeWhiteSpaces ? SentencePieceNormalizer.DummyPrefix : ' ';
Microsoft.ML.Tokenizers.Tests (1)
LlamaTests.cs (1)
373Assert.True(bpe.EscapeWhiteSpaces);