5 references to GetEncoder
Microsoft.ML.Tokenizers.Tests (5)
TiktokenTests.cs (5)
124IReadOnlyDictionary<ReadOnlyMemory<byte>, int> encoder = GetEncoder(tiktoken)!; 125IReadOnlyDictionary<ReadOnlyMemory<byte>, int> externalEncoder = GetEncoder(externalTokenizer)!; 514Assert.Equal(GetEncoder(tiktoken1), GetEncoder(tiktoken)); 558int entriesCount = GetEncoder((tokenizer as TiktokenTokenizer)!)!.Count;