1 write to SpecialTokens
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
354
SpecialTokens
= specialTokens;
3 references to SpecialTokens
Microsoft.ML.Tokenizers.Tests (3)
BpeTests.cs (3)
868
Assert.NotNull(tokenizer.
SpecialTokens
);
869
Assert.True(tokenizer.
SpecialTokens
.TryGetValue(thinkStartSpecialToken, out int thinkStartId));
870
Assert.True(tokenizer.
SpecialTokens
.TryGetValue(thinkEndSpecialToken, out int thinkEndId));