2 writes to SpecialTokens
Microsoft.ML.Tokenizers (2)
Model\TiktokenTokenizer.cs (2)
98
SpecialTokens
= specialTokens;
112
SpecialTokens
= specialTokens;
5 references to SpecialTokens
Microsoft.ML.Tokenizers.Tests (5)
TiktokenTests.cs (5)
49
IReadOnlyDictionary<string, int>? specialTokens = (GPT4 as TiktokenTokenizer)!.
SpecialTokens
;
122
TiktokenTokenizer externalTokenizer = TiktokenTokenizer.Create(tokenizerDataFileName, tokenizer.PreTokenizer, null, tiktoken.
SpecialTokens
);
516
Assert.Equal(tiktoken1.
SpecialTokens
, tiktoken.
SpecialTokens
);
814
}, GptOss.
SpecialTokens
);