10 references to EncodeToTokens
Microsoft.ML.Tokenizers.Tests (10)
BpeTests.cs (1)
430
IReadOnlyList<EncodedToken> encoding1 = tokenizer.
EncodeToTokens
(text.AsSpan(), out _);
CodeGenTests.cs (5)
255
encoding = tokenizer.
EncodeToTokens
(text.AsSpan(), out _);
350
encoding = tokenizer.
EncodeToTokens
(text.AsSpan(), out _);
560
encoding = codeGenTokenizer.
EncodeToTokens
(text.AsSpan(), out _);
693
encoding = codeGenTokenizer.
EncodeToTokens
(text.AsSpan(), out _);
829
encoding = codeGenTokenizer.
EncodeToTokens
(text.AsSpan(), out _);
EnglishRobertaTests.cs (1)
182
IReadOnlyList<EncodedToken> encoding1 = tokenizer.
EncodeToTokens
(text.AsSpan(), out _);
LlamaTests.cs (2)
339
Assert.Equal([], llamaTokenizer.
EncodeToTokens
(Span<char>.Empty, out _));
501
IReadOnlyList<EncodedToken> encoding1 = tokenizer.
EncodeToTokens
(text.AsSpan(), out _);
TitokenTests.cs (1)
565
IReadOnlyList<EncodedToken> encoding1 = tokenizer.
EncodeToTokens
(text.AsSpan(), out _);