1 write to SpecialTokens
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
349
SpecialTokens
= specialTokens;
3 references to SpecialTokens
Microsoft.ML.Tokenizers.Tests (3)
BpeTests.cs (3)
855
Assert.NotNull(tokenizer.
SpecialTokens
);
856
Assert.True(tokenizer.
SpecialTokens
.TryGetValue(thinkStartSpecialToken, out int thinkStartId));
857
Assert.True(tokenizer.
SpecialTokens
.TryGetValue(thinkEndSpecialToken, out int thinkEndId));