2 writes to SpecialTokens
Microsoft.ML.Tokenizers (2)
Model\TiktokenTokenizer.cs (2)
97
SpecialTokens
= specialTokens;
111
SpecialTokens
= specialTokens;
4 references to SpecialTokens
Microsoft.ML.Tokenizers.Tests (4)
TitokenTests.cs (4)
44
IReadOnlyDictionary<string, int>? specialTokensEncoder = (GPT4 as TiktokenTokenizer)!.
SpecialTokens
;
116
TiktokenTokenizer externalTokenizer = TiktokenTokenizer.Create(tokenizerDataFileName, tokenizer.PreTokenizer, null, tiktoken.
SpecialTokens
);
478
Assert.Equal(tiktoken1.
SpecialTokens
, tiktoken.
SpecialTokens
);