5 references to GetEncoder
Microsoft.ML.Tokenizers.Tests (5)
TiktokenTests.cs (5)
127IReadOnlyDictionary<ReadOnlyMemory<byte>, int> encoder = GetEncoder(tiktoken)!; 128IReadOnlyDictionary<ReadOnlyMemory<byte>, int> externalEncoder = GetEncoder(externalTokenizer)!; 523Assert.Equal(GetEncoder(tiktoken1), GetEncoder(tiktoken)); 570int entriesCount = GetEncoder((tokenizer as TiktokenTokenizer)!)!.Count;