5 references to GetEncoder
Microsoft.ML.Tokenizers.Tests (5)
TiktokenTests.cs (5)
128
IReadOnlyDictionary<ReadOnlyMemory<byte>, int> encoder =
GetEncoder
(tiktoken)!;
129
IReadOnlyDictionary<ReadOnlyMemory<byte>, int> externalEncoder =
GetEncoder
(externalTokenizer)!;
526
Assert.Equal(
GetEncoder
(tiktoken1),
GetEncoder
(tiktoken));
574
int entriesCount =
GetEncoder
((tokenizer as TiktokenTokenizer)!)!.Count;