2 writes to SpecialTokens
Microsoft.ML.Tokenizers (2)
Model\TiktokenTokenizer.cs (2)
98
SpecialTokens
= specialTokens;
112
SpecialTokens
= specialTokens;
4 references to SpecialTokens
Microsoft.ML.Tokenizers.Tests (4)
TiktokenTests.cs (4)
47
IReadOnlyDictionary<string, int>? specialTokens = (GPT4 as TiktokenTokenizer)!.
SpecialTokens
;
120
TiktokenTokenizer externalTokenizer = TiktokenTokenizer.Create(tokenizerDataFileName, tokenizer.PreTokenizer, null, tiktoken.
SpecialTokens
);
492
Assert.Equal(tiktoken1.
SpecialTokens
, tiktoken.
SpecialTokens
);