2 writes to SpecialTokens
Microsoft.ML.Tokenizers (2)
Model\TiktokenTokenizer.cs (2)
98
SpecialTokens
= specialTokens;
112
SpecialTokens
= specialTokens;
5 references to SpecialTokens
Microsoft.ML.Tokenizers.Tests (5)
TiktokenTests.cs (5)
52
IReadOnlyDictionary<string, int>? specialTokens = (GPT4 as TiktokenTokenizer)!.
SpecialTokens
;
125
TiktokenTokenizer externalTokenizer = TiktokenTokenizer.Create(tokenizerDataFileName, tokenizer.PreTokenizer, null, tiktoken.
SpecialTokens
);
525
Assert.Equal(tiktoken1.
SpecialTokens
, tiktoken.
SpecialTokens
);
826
}, GptOss.
SpecialTokens
);