11 references to EncodeToTokens
Microsoft.ML.Tokenizers.Tests (11)
BpeTests.cs (1)
463
IReadOnlyList<EncodedToken> encoding1 = tokenizer.
EncodeToTokens
(text.AsSpan(), out _);
CodeGenTests.cs (5)
255
encoding = tokenizer.
EncodeToTokens
(text.AsSpan(), out _);
350
encoding = tokenizer.
EncodeToTokens
(text.AsSpan(), out _);
560
encoding = codeGenTokenizer.
EncodeToTokens
(text.AsSpan(), out _);
693
encoding = codeGenTokenizer.
EncodeToTokens
(text.AsSpan(), out _);
829
encoding = codeGenTokenizer.
EncodeToTokens
(text.AsSpan(), out _);
EnglishRobertaTests.cs (1)
182
IReadOnlyList<EncodedToken> encoding1 = tokenizer.
EncodeToTokens
(text.AsSpan(), out _);
LlamaTests.cs (2)
339
Assert.Equal([], llamaTokenizer.
EncodeToTokens
(Span<char>.Empty, out _));
501
IReadOnlyList<EncodedToken> encoding1 = tokenizer.
EncodeToTokens
(text.AsSpan(), out _);
TiktokenTests.cs (1)
588
IReadOnlyList<EncodedToken> encoding1 = tokenizer.
EncodeToTokens
(text.AsSpan(), out _);
UnigramTests.cs (1)
333
result = _unigramTokenizer.
EncodeToTokens
(inputText.AsSpan(), out normalized);