7 types derived from Tokenizer
Microsoft.ML.Tokenizers (6)
Model\BPETokenizer.cs (1)
23public sealed class BpeTokenizer : Tokenizer
Model\CodeGenTokenizer.cs (1)
23public class CodeGenTokenizer : Tokenizer
Model\EnglishRobertaTokenizer.cs (1)
20public sealed class EnglishRobertaTokenizer : Tokenizer
Model\SentencePieceTokenizer.cs (1)
20public class SentencePieceTokenizer : Tokenizer
Model\TiktokenTokenizer.cs (1)
25public sealed partial class TiktokenTokenizer : Tokenizer
Model\WordPieceTokenizer.cs (1)
25public partial class WordPieceTokenizer : Tokenizer
Microsoft.ML.Tokenizers.Tests (1)
TokenizerTests.cs (1)
79private sealed class EnglishAlphabetTokenizer : Tokenizer
95 references to Tokenizer
Microsoft.Extensions.AI.Integration.Tests (3)
ReducingChatClientTests.cs (3)
21private static readonly Tokenizer _gpt4oTokenizer = TiktokenTokenizer.CreateForModel("gpt-4o"); 61private readonly Tokenizer _tokenizer; 64public TokenCountingChatReducer(Tokenizer tokenizer, int tokenLimit)
Microsoft.Extensions.DataIngestion (4)
Chunkers\DocumentTokenChunker.cs (1)
25private readonly Tokenizer _tokenizer;
Chunkers\ElementsChunker.cs (1)
16private readonly Tokenizer _tokenizer;
Chunkers\IngestionChunkerOptions.cs (2)
24public IngestionChunkerOptions(Tokenizer tokenizer) 32public Tokenizer Tokenizer { get; }
Microsoft.Extensions.DataIngestion.Tests (3)
Chunkers\ChunkerOptionsTests.cs (1)
12private static readonly Tokenizer _tokenizer = TiktokenTokenizer.CreateForModel("gpt-4");
Chunkers\HeaderChunkerTests.cs (1)
175Tokenizer tokenizer = TiktokenTokenizer.CreateForModel("gpt-4");
Chunkers\SemanticSimilarityChunkerTests.cs (1)
26Tokenizer tokenizer = TiktokenTokenizer.CreateForModel("gpt-4o");
Microsoft.ML.Tokenizers (5)
Tokenizer.cs (5)
17/// Initializes a new instance of the <see cref="Tokenizer"/> class. 39/// Types derived from <see cref="Tokenizer"/> may override this implementation to provide a more efficient implementation. 180/// Types derived from <see cref="Tokenizer"/> may override this implementation to provide a more efficient implementation. 223/// Types derived from <see cref="Tokenizer"/> may override this implementation to provide a more efficient implementation. 361/// Types derived from <see cref="Tokenizer"/> may override this implementation to provide a more efficient implementation.
Microsoft.ML.Tokenizers.Tests (63)
BpeTests.cs (6)
300Tokenizer tokenizer = bpe; 358private static Tokenizer? _gpt2Tokenizer = null; 360private static Tokenizer GetGpt2Tokenizer() 414Tokenizer tokenizer = GetGpt2Tokenizer(); 418private void ValidateTokenizer(Tokenizer tokenizer) 473Tokenizer tokenizer = GetGpt2Tokenizer();
CodeGenTests.cs (10)
17private static Tokenizer _codegen350MMonoTokenizer = CreateCodegen350MMonoTokenizer(); 18private static Tokenizer _codegen350MMonoTokenizerWithSpace = CreateCodegen350MMonoTokenizer(addPrefixSpace: true); 19private static Tokenizer _codegen350MMonoTokenizerWithBeginningOfSentence = CreateCodegen350MMonoTokenizer(bos: true); 20private static Tokenizer _codegen350MMonoTokenizerWithEndOfSentence = CreateCodegen350MMonoTokenizer(eos: true); 21private static Tokenizer _codegen350MMonoTokenizerWithBeginningAndEndOfSentence = CreateCodegen350MMonoTokenizer(bos: true, eos: true); 23private static Tokenizer CreateCodegen350MMonoTokenizer(bool addPrefixSpace = false, bool bos = false, bool eos = false) 34private static Tokenizer CreateCodegenPhi2Tokenizer() 223Tokenizer phi2Tokenizer = CreateCodegenPhi2Tokenizer(); 248private void TestDecoding(Tokenizer tokenizer, string text) 332Tokenizer tokenizer,
EnglishRobertaTests.cs (5)
79private static Tokenizer? _robertaTokenizer = null; 80private static Tokenizer GetRobertaTokenizer() 113Tokenizer tokenizer = EnglishRobertaTokenizer.Create(vocabFile, mergeFile, translationFile, RobertaPreTokenizer.Instance); 179Tokenizer tokenizer = GetRobertaTokenizer(); 236private void TestTokenizer(Tokenizer tokenizer, CallingOrder callingOrder = CallingOrder.Encode)
LlamaTests.cs (15)
22private static Tokenizer _llamaTokenizer = CreateLlamaTokenizer(); 23private static Tokenizer _llamaMistralTokenizer = CreateLMistralTokenizer(); 24private static Tokenizer _llamaPhi3Tokenizer = CreateLPhi3Tokenizer(); 25private static Tokenizer _llamaPhi3TokenizerWithTreatSpaceSuffix = CreateLPhi3Tokenizer(treatWhitespaceAsSuffix: true); 28private static Tokenizer CreateLlamaTokenizer() 36private static Tokenizer CreateLMistralTokenizer() 43private static Tokenizer CreateLPhi3Tokenizer(bool treatWhitespaceAsSuffix = false) 234public void TestLlamaTokenizer(Tokenizer tokenizer, string input, int[] ids, string[] tokens, (int Index, int Length)[] offsets) 237Tokenizer[] tokenizers = tokenizer == _llamaTokenizer ? new[] { tokenizer, _llamaPhi3Tokenizer } : new[] { tokenizer }; 239foreach (Tokenizer llamaTokenizer in tokenizers) 336public void TestLlamaTokenizerWithEmptyInput(Tokenizer llamaTokenizer) 352public void TestLlamaTokenizerProperties(Tokenizer llamaTokenizer) 495Tokenizer tokenizer = _llamaTokenizer; 551Tokenizer tokenizer = _llamaTokenizer; 628Tokenizer tokenizer = _llamaTokenizer;
NormalizerTests.cs (1)
64Tokenizer tokenizer = BpeTests.CreateEmptyBpe(preTokenizer: null, normalizer);
PreTokenizerTests.cs (1)
64Tokenizer tokenizer = BpeTests.CreateEmptyBpe(normalizer: null, preTokenizer: preTokenizer);
TiktokenTests.cs (24)
32public static Tokenizer GPT4 { get; } = TiktokenTokenizer.CreateForModel("gpt-4", _specialTokens); 33public static Tokenizer GPT2 { get; } = TiktokenTokenizer.CreateForModel("gpt2"); 34public static Tokenizer P50kBase { get; } = TiktokenTokenizer.CreateForModel("text-davinci-003"); 35public static Tokenizer R50kBase { get; } = TiktokenTokenizer.CreateForModel("ada"); 36public static Tokenizer P50kEdit { get; } = TiktokenTokenizer.CreateForModel("text-davinci-edit-001"); 37public static Tokenizer GPT4o { get; } = TiktokenTokenizer.CreateForModel("gpt-4o"); 38public static Tokenizer GPT5 { get; } = TiktokenTokenizer.CreateForModel("gpt-5"); 39public static Tokenizer GPT5_1 { get; } = TiktokenTokenizer.CreateForModel("gpt-5.1"); 40public static Tokenizer GPT5_2 { get; } = TiktokenTokenizer.CreateForModel("gpt-5.2"); 41public static Tokenizer GPT5_3 { get; } = TiktokenTokenizer.CreateForModel("gpt-5.3"); 42public static Tokenizer GPT5_4 { get; } = TiktokenTokenizer.CreateForModel("gpt-5.4"); 43public static Tokenizer GPT5_5 { get; } = TiktokenTokenizer.CreateForModel("gpt-5.5"); 44public static Tokenizer GPT5_6 { get; } = TiktokenTokenizer.CreateForModel("gpt-5.6"); 45public static Tokenizer GPT6 { get; } = TiktokenTokenizer.CreateForModel("gpt-6"); 46public static Tokenizer Phi4 { get; } = TiktokenTokenizer.CreateForModel("phi-4"); 71Tokenizer tokenizer = TiktokenTokenizer.Create(tokenizerDataFileName, GPT4.PreTokenizer, null, specialTokens); 121public async Task TestTokenizerUsingExternalVocab(Tokenizer tokenizer, string url) 147private void TestGPT4TokenizationEncoding(Tokenizer tokenizer) 222private void TestGPT4Tokenizer(Tokenizer gpt4Tokenizer) 498Tokenizer tokenizer = TiktokenTokenizer.CreateForModel(modelName); 512Tokenizer tokenizer = TiktokenTokenizer.CreateForEncoding(encodingName); 527Tokenizer tokenizer1 = TiktokenTokenizer.CreateForModel(modelName); 581Tokenizer tokenizer = TiktokenTokenizer.CreateForModel(name); 640private void TestTokenizerEncodingForTokenizer(Tokenizer tokenizer, string text, string[] expectedTokens, (int Index, int Length)[] expectedOffsets, int[] expectedIds)
TokenizerTests.cs (1)
123internal static void TestTokenLimits(Tokenizer tokenizer)
Microsoft.ML.TorchSharp (17)
Extensions\TokenizerExtensions.cs (4)
17private static Tokenizer _instance; 19internal static Tokenizer GetInstance(IChannel ch) 41internal static EnglishRobertaTokenizer RobertaModel(this Tokenizer tokenizer) 52internal static IReadOnlyList<int> EncodeToConverted(this Tokenizer tokenizer, string sentence)
NasBert\NasBertTrainer.cs (3)
178public Tokenizer Tokenizer; 582private IList<int> PrepInputTokens(ref ReadOnlyMemory<char> sentence1, ref ReadOnlyMemory<char> sentence2, ref ValueGetter<ReadOnlyMemory<char>> getSentence1, ref ValueGetter<ReadOnlyMemory<char>> getSentence2, Tokenizer tokenizer) 612private protected void UpdateCacheIfNeeded(long position, TensorCacher outputCache, ref ReadOnlyMemory<char> sentence1, ref ReadOnlyMemory<char> sentence2, ref ValueGetter<ReadOnlyMemory<char>> getSentence1, ref ValueGetter<ReadOnlyMemory<char>> getSentence2, Tokenizer tokenizer)
NasBert\NerTrainer.cs (3)
329var tokenizer = TokenizerExtensions.GetInstance(ch); 377private void CondenseOutput(ref VBuffer<UInt32> dst, string sentence, Tokenizer tokenizer, TensorCacher outputCacher) 417Tokenizer tokenizer = TokenizerExtensions.GetInstance(ch);
NasBert\SentenceSimilarityTrainer.cs (2)
242var tokenizer = TokenizerExtensions.GetInstance(ch); 275Tokenizer tokenizer = TokenizerExtensions.GetInstance(ch);
NasBert\TextClassificationTrainer.cs (3)
266var tokenizer = TokenizerExtensions.GetInstance(ch); 321Tokenizer tokenizer = TokenizerExtensions.GetInstance(ch); 352Tokenizer tokenizer = TokenizerExtensions.GetInstance(ch);
Roberta\QATrainer.cs (2)
190public Tokenizer Tokenizer; 569public Tokenizer Tokenizer;