1 reference to _specialTokens
Microsoft.ML.Tokenizers.Tests (1)
TiktokenTests.cs (1)
31
public static Tokenizer GPT4 { get; } = TiktokenTokenizer.CreateForModel("gpt-4",
_specialTokens
);