5 references to GetEncoder
Microsoft.ML.Tokenizers.Tests (5)
TiktokenTests.cs (5)
130IReadOnlyDictionary<ReadOnlyMemory<byte>, int> encoder = GetEncoder(tiktoken)!; 131IReadOnlyDictionary<ReadOnlyMemory<byte>, int> externalEncoder = GetEncoder(externalTokenizer)!; 532Assert.Equal(GetEncoder(tiktoken1), GetEncoder(tiktoken)); 582int entriesCount = GetEncoder((tokenizer as TiktokenTokenizer)!)!.Count;