Tokenizer

Model\BPETokenizer.cs (1)

23public sealed class BpeTokenizer : Tokenizer

Model\CodeGenTokenizer.cs (1)

23public class CodeGenTokenizer : Tokenizer

Model\EnglishRobertaTokenizer.cs (1)

20public sealed class EnglishRobertaTokenizer : Tokenizer

Model\SentencePieceTokenizer.cs (1)

20public class SentencePieceTokenizer : Tokenizer

Model\TiktokenTokenizer.cs (1)

25public sealed partial class TiktokenTokenizer : Tokenizer

Model\WordPieceTokenizer.cs (1)

25public partial class WordPieceTokenizer : Tokenizer

TokenizerTests.cs (1)

79private sealed class EnglishAlphabetTokenizer : Tokenizer

ReducingChatClientTests.cs (3)

21private static readonly Tokenizer _gpt4oTokenizer = TiktokenTokenizer.CreateForModel("gpt-4o"); 61private readonly Tokenizer _tokenizer; 64public TokenCountingChatReducer(Tokenizer tokenizer, int tokenLimit)

Chunkers\DocumentTokenChunker.cs (1)

25private readonly Tokenizer _tokenizer;

Chunkers\ElementsChunker.cs (1)

16private readonly Tokenizer _tokenizer;

Chunkers\IngestionChunkerOptions.cs (2)

24public IngestionChunkerOptions(Tokenizer tokenizer) 32public Tokenizer Tokenizer { get; }

Chunkers\ChunkerOptionsTests.cs (1)

12private static readonly Tokenizer _tokenizer = TiktokenTokenizer.CreateForModel("gpt-4");

Chunkers\HeaderChunkerTests.cs (1)

175Tokenizer tokenizer = TiktokenTokenizer.CreateForModel("gpt-4");

Chunkers\SemanticSimilarityChunkerTests.cs (1)

26Tokenizer tokenizer = TiktokenTokenizer.CreateForModel("gpt-4o");

CausalLMPipelineChatClient.cs (1)

18where TTokenizer : Tokenizer

Pipeline\CausalLMPipeline.cs (5)

18where TTokenizer : Tokenizer 28Tokenizer Tokenizer { get; } 69where TTokenizer : Tokenizer 97Tokenizer tokenizer, 116public Tokenizer Tokenizer { get; }

Trainer\CausalLMDataset.cs (2)

29Tokenizer tokenizer) 50public static CausalLMDataset Create(IEnumerable<string> inputs, IEnumerable<string> outputs, Tokenizer tokenizer)

CausalLMDatasetTest.cs (2)

21private static Tokenizer CreateLlamaTokenizer() 96var tokenizer = CreateLlamaTokenizer();

Llama3CausalLMChatClient.cs (2)

12public class Llama3CausalLMChatClient : CausalLMPipelineChatClient<Tokenizer, LlamaForCausalLM> 17ICausalLMPipeline<Tokenizer, LlamaForCausalLM> pipeline,

LlamaCausalLMAgent.cs (2)

15private readonly ICausalLMPipeline<Tokenizer, LlamaForCausalLM> _pipeline; 27ICausalLMPipeline<Tokenizer, LlamaForCausalLM> pipeline,

LlamaChatCompletionService.cs (2)

15private readonly ICausalLMPipeline<Tokenizer, LlamaForCausalLM> _pipeline; 24public LlamaChatCompletionService(ICausalLMPipeline<Tokenizer, LlamaForCausalLM> pipeline, ISemanticKernelChatTemplateBuilder? templateBuilder = null)

LlamaTextCompletionService.cs (2)

20private readonly ICausalLMPipeline<Tokenizer, LlamaForCausalLM> _pipeline; 22public LlamaTextCompletionService(ICausalLMPipeline<Tokenizer, LlamaForCausalLM> pipeline)

MistralCausalLMAgent.cs (2)

18private readonly ICausalLMPipeline<Tokenizer, MistralForCausalLM> _pipeline; 31ICausalLMPipeline<Tokenizer, MistralForCausalLM> pipeline,

Extension\SemanticKernelExtension.cs (2)

18ICausalLMPipeline<Tokenizer, Phi3ForCausalLM> pipeline) 27ICausalLMPipeline<Tokenizer, Phi3ForCausalLM> pipeline)

Phi3\Phi3CausalLMAgent.cs (2)

20private readonly ICausalLMPipeline<Tokenizer, Phi3ForCausalLM> _pipeline; 25ICausalLMPipeline<Tokenizer, Phi3ForCausalLM> pipeline,

Phi3\Phi3CausalLMChatClient.cs (2)

17public class Phi3CausalLMChatClient : CausalLMPipelineChatClient<Tokenizer, Phi3ForCausalLM> 22ICausalLMPipeline<Tokenizer, Phi3ForCausalLM> pipeline,

Phi3\Phi3CausalLMChatCompletionService.cs (2)

17private readonly ICausalLMPipeline<Tokenizer, Phi3ForCausalLM> _pipeline; 22ICausalLMPipeline<Tokenizer, Phi3ForCausalLM> pipeline,

Phi3\Phi3CausalLMTextGenerationService.cs (2)

15private readonly ICausalLMPipeline<Tokenizer, Phi3ForCausalLM> _pipeline; 18ICausalLMPipeline<Tokenizer, Phi3ForCausalLM> pipeline)

AutoGenTests.cs (1)

19var pipeline = Mock.Of<ICausalLMPipeline<Tokenizer, Phi3ForCausalLM>>();

SemanticKernelTests.cs (2)

22var pipeline = Mock.Of<ICausalLMPipeline<Tokenizer, Phi3ForCausalLM>>(); 58var pipeline = Mock.Of<ICausalLMPipeline<Tokenizer, Phi3ForCausalLM>>();

Llama\SFT_Llama_3_2_1B.cs (2)

63if (p is not ICausalLMPipeline<Tokenizer, LlamaForCausalLM> llamaPipeline) 101public static CausalLMDataset CreateDataset(IEnumerable<Data> dataset, Tokenizer tokenizer, IMEAIChatTemplateBuilder templateBuilder)

Tokenizer.cs (5)

17/// Initializes a new instance of the <see cref="Tokenizer"/> class. 39/// Types derived from <see cref="Tokenizer"/> may override this implementation to provide a more efficient implementation. 180/// Types derived from <see cref="Tokenizer"/> may override this implementation to provide a more efficient implementation. 223/// Types derived from <see cref="Tokenizer"/> may override this implementation to provide a more efficient implementation. 361/// Types derived from <see cref="Tokenizer"/> may override this implementation to provide a more efficient implementation.

BpeTests.cs (6)

300Tokenizer tokenizer = bpe; 358private static Tokenizer? _gpt2Tokenizer = null; 360private static Tokenizer GetGpt2Tokenizer() 414Tokenizer tokenizer = GetGpt2Tokenizer(); 418private void ValidateTokenizer(Tokenizer tokenizer) 473Tokenizer tokenizer = GetGpt2Tokenizer();

CodeGenTests.cs (10)

17private static Tokenizer _codegen350MMonoTokenizer = CreateCodegen350MMonoTokenizer(); 18private static Tokenizer _codegen350MMonoTokenizerWithSpace = CreateCodegen350MMonoTokenizer(addPrefixSpace: true); 19private static Tokenizer _codegen350MMonoTokenizerWithBeginningOfSentence = CreateCodegen350MMonoTokenizer(bos: true); 20private static Tokenizer _codegen350MMonoTokenizerWithEndOfSentence = CreateCodegen350MMonoTokenizer(eos: true); 21private static Tokenizer _codegen350MMonoTokenizerWithBeginningAndEndOfSentence = CreateCodegen350MMonoTokenizer(bos: true, eos: true); 23private static Tokenizer CreateCodegen350MMonoTokenizer(bool addPrefixSpace = false, bool bos = false, bool eos = false) 34private static Tokenizer CreateCodegenPhi2Tokenizer() 223Tokenizer phi2Tokenizer = CreateCodegenPhi2Tokenizer(); 248private void TestDecoding(Tokenizer tokenizer, string text) 332Tokenizer tokenizer,

EnglishRobertaTests.cs (5)

79private static Tokenizer? _robertaTokenizer = null; 80private static Tokenizer GetRobertaTokenizer() 113Tokenizer tokenizer = EnglishRobertaTokenizer.Create(vocabFile, mergeFile, translationFile, RobertaPreTokenizer.Instance); 179Tokenizer tokenizer = GetRobertaTokenizer(); 236private void TestTokenizer(Tokenizer tokenizer, CallingOrder callingOrder = CallingOrder.Encode)

LlamaTests.cs (15)

22private static Tokenizer _llamaTokenizer = CreateLlamaTokenizer(); 23private static Tokenizer _llamaMistralTokenizer = CreateLMistralTokenizer(); 24private static Tokenizer _llamaPhi3Tokenizer = CreateLPhi3Tokenizer(); 25private static Tokenizer _llamaPhi3TokenizerWithTreatSpaceSuffix = CreateLPhi3Tokenizer(treatWhitespaceAsSuffix: true); 28private static Tokenizer CreateLlamaTokenizer() 36private static Tokenizer CreateLMistralTokenizer() 43private static Tokenizer CreateLPhi3Tokenizer(bool treatWhitespaceAsSuffix = false) 234public void TestLlamaTokenizer(Tokenizer tokenizer, string input, int[] ids, string[] tokens, (int Index, int Length)[] offsets) 237Tokenizer[] tokenizers = tokenizer == _llamaTokenizer ? new[] { tokenizer, _llamaPhi3Tokenizer } : new[] { tokenizer }; 239foreach (Tokenizer llamaTokenizer in tokenizers) 336public void TestLlamaTokenizerWithEmptyInput(Tokenizer llamaTokenizer) 352public void TestLlamaTokenizerProperties(Tokenizer llamaTokenizer) 495Tokenizer tokenizer = _llamaTokenizer; 551Tokenizer tokenizer = _llamaTokenizer; 628Tokenizer tokenizer = _llamaTokenizer;

NormalizerTests.cs (1)

64Tokenizer tokenizer = BpeTests.CreateEmptyBpe(preTokenizer: null, normalizer);

PreTokenizerTests.cs (1)

64Tokenizer tokenizer = BpeTests.CreateEmptyBpe(normalizer: null, preTokenizer: preTokenizer);

TiktokenTests.cs (24)

TokenizerTests.cs (1)

123internal static void TestTokenLimits(Tokenizer tokenizer)

Extensions\TokenizerExtensions.cs (4)

17private static Tokenizer _instance; 19internal static Tokenizer GetInstance(IChannel ch) 41internal static EnglishRobertaTokenizer RobertaModel(this Tokenizer tokenizer) 52internal static IReadOnlyList<int> EncodeToConverted(this Tokenizer tokenizer, string sentence)

NasBert\NasBertTrainer.cs (3)

178public Tokenizer Tokenizer; 582private IList<int> PrepInputTokens(ref ReadOnlyMemory<char> sentence1, ref ReadOnlyMemory<char> sentence2, ref ValueGetter<ReadOnlyMemory<char>> getSentence1, ref ValueGetter<ReadOnlyMemory<char>> getSentence2, Tokenizer tokenizer) 612private protected void UpdateCacheIfNeeded(long position, TensorCacher outputCache, ref ReadOnlyMemory<char> sentence1, ref ReadOnlyMemory<char> sentence2, ref ValueGetter<ReadOnlyMemory<char>> getSentence1, ref ValueGetter<ReadOnlyMemory<char>> getSentence2, Tokenizer tokenizer)

NasBert\NerTrainer.cs (3)

329var tokenizer = TokenizerExtensions.GetInstance(ch); 377private void CondenseOutput(ref VBuffer<UInt32> dst, string sentence, Tokenizer tokenizer, TensorCacher outputCacher) 417Tokenizer tokenizer = TokenizerExtensions.GetInstance(ch);

NasBert\SentenceSimilarityTrainer.cs (2)

242var tokenizer = TokenizerExtensions.GetInstance(ch); 275Tokenizer tokenizer = TokenizerExtensions.GetInstance(ch);

NasBert\TextClassificationTrainer.cs (3)

266var tokenizer = TokenizerExtensions.GetInstance(ch); 321Tokenizer tokenizer = TokenizerExtensions.GetInstance(ch); 352Tokenizer tokenizer = TokenizerExtensions.GetInstance(ch);

Roberta\QATrainer.cs (2)

190public Tokenizer Tokenizer; 569public Tokenizer Tokenizer;