6 references to TreatWhitespaceAsSuffix
Microsoft.ML.Tokenizers.Tests (6)
LlamaTests.cs (2)
374
Assert.False(bpe.
TreatWhitespaceAsSuffix
);
873
Assert.True(tokenizerWithSuffix.
TreatWhitespaceAsSuffix
);
SentencePieceTests.cs (4)
77
Assert.False(tokenizer.
TreatWhitespaceAsSuffix
);
100
Assert.False(tokenizer.
TreatWhitespaceAsSuffix
);
201
Assert.False(tokenizer.
TreatWhitespaceAsSuffix
);
234
Assert.True(tokenizer.
TreatWhitespaceAsSuffix
);