5 references to GetEncoder
Microsoft.ML.Tokenizers.Tests (5)
TiktokenTests.cs (5)
126IReadOnlyDictionary<ReadOnlyMemory<byte>, int> encoder = GetEncoder(tiktoken)!; 127IReadOnlyDictionary<ReadOnlyMemory<byte>, int> externalEncoder = GetEncoder(externalTokenizer)!; 520Assert.Equal(GetEncoder(tiktoken1), GetEncoder(tiktoken)); 566int entriesCount = GetEncoder((tokenizer as TiktokenTokenizer)!)!.Count;