5 references to GetEncoder
Microsoft.ML.Tokenizers.Tests (5)
TiktokenTests.cs (5)
125IReadOnlyDictionary<ReadOnlyMemory<byte>, int> encoder = GetEncoder(tiktoken)!; 126IReadOnlyDictionary<ReadOnlyMemory<byte>, int> externalEncoder = GetEncoder(externalTokenizer)!; 517Assert.Equal(GetEncoder(tiktoken1), GetEncoder(tiktoken)); 562int entriesCount = GetEncoder((tokenizer as TiktokenTokenizer)!)!.Count;