2 writes to SpecialTokens
Microsoft.ML.Tokenizers (2)
Model\TiktokenTokenizer.cs (2)
98
SpecialTokens
= specialTokens;
112
SpecialTokens
= specialTokens;
5 references to SpecialTokens
Microsoft.ML.Tokenizers.Tests (5)
TiktokenTests.cs (5)
50
IReadOnlyDictionary<string, int>? specialTokens = (GPT4 as TiktokenTokenizer)!.
SpecialTokens
;
123
TiktokenTokenizer externalTokenizer = TiktokenTokenizer.Create(tokenizerDataFileName, tokenizer.PreTokenizer, null, tiktoken.
SpecialTokens
);
519
Assert.Equal(tiktoken1.
SpecialTokens
, tiktoken.
SpecialTokens
);
818
}, GptOss.
SpecialTokens
);