1 write to TreatWhitespaceAsSuffix
Microsoft.ML.Tokenizers (1)
Model\SentencePieceTokenizer.cs (1)
72
TreatWhitespaceAsSuffix
= modelProto.TrainerSpec.TreatWhitespaceAsSuffix;
9 references to TreatWhitespaceAsSuffix
Microsoft.ML.Tokenizers (7)
Model\SentencePieceTokenizer.cs (7)
1581
AppendTokenWithCheckingPrefix(AddDummyPrefix,
TreatWhitespaceAsSuffix
, token, prefixSuffixChar, ref sb, ref prefixRemoved, ref suffixIndex);
1590
AppendTokenWithCheckingPrefix(AddDummyPrefix,
TreatWhitespaceAsSuffix
, token, prefixSuffixChar, ref sb, ref prefixRemoved, ref suffixIndex);
1641
AppendTokenWithCheckingPrefix(AddDummyPrefix,
TreatWhitespaceAsSuffix
, token, prefixSuffixChar, ref sb, ref prefixRemoved, ref suffixIndex);
1655
if (AddDummyPrefix &&
TreatWhitespaceAsSuffix
&& suffixIndex >= 0 && sb.Length > 0)
1837
if (!AppendTokenWithCheckingPrefix(AddDummyPrefix,
TreatWhitespaceAsSuffix
, token, prefixSuffixChar, destination, ref prefixRemoved, ref suffixIndex, ref idsConsumed, ref charsWritten))
1855
if (!AppendTokenWithCheckingPrefix(AddDummyPrefix,
TreatWhitespaceAsSuffix
, token, prefixSuffixChar, destination, ref prefixRemoved, ref suffixIndex, ref idsConsumed, ref charsWritten))
1930
if (!AppendTokenWithCheckingPrefix(AddDummyPrefix,
TreatWhitespaceAsSuffix
, token, prefixSuffixChar, destination, ref prefixRemoved, ref suffixIndex, ref idsConsumed, ref charsWritten))
Microsoft.ML.Tokenizers.Tests (2)
LlamaTests.cs (2)
374
Assert.False(bpe.
TreatWhitespaceAsSuffix
);
873
Assert.True(tokenizerWithSuffix.
TreatWhitespaceAsSuffix
);