12 references to EncodeToTokens
Microsoft.ML.Tokenizers.Tests (12)
BpeTests.cs (1)
430
IReadOnlyList<EncodedToken> encoding1 = tokenizer.
EncodeToTokens
(text.AsSpan(), out _);
CodeGenTests.cs (5)
255
encoding = tokenizer.
EncodeToTokens
(text.AsSpan(), out _);
350
encoding = tokenizer.
EncodeToTokens
(text.AsSpan(), out _);
560
encoding = codeGenTokenizer.
EncodeToTokens
(text.AsSpan(), out _);
693
encoding = codeGenTokenizer.
EncodeToTokens
(text.AsSpan(), out _);
829
encoding = codeGenTokenizer.
EncodeToTokens
(text.AsSpan(), out _);
EnglishRobertaTests.cs (1)
182
IReadOnlyList<EncodedToken> encoding1 = tokenizer.
EncodeToTokens
(text.AsSpan(), out _);
LlamaTests.cs (2)
339
Assert.Equal([], llamaTokenizer.
EncodeToTokens
(Span<char>.Empty, out _));
501
IReadOnlyList<EncodedToken> encoding1 = tokenizer.
EncodeToTokens
(text.AsSpan(), out _);
TiktokenTests.cs (1)
583
IReadOnlyList<EncodedToken> encoding1 = tokenizer.
EncodeToTokens
(text.AsSpan(), out _);
UnigramTests.cs (2)
412
result = _unigramTokenizer.
EncodeToTokens
(inputText.AsSpan(), out normalized);
416
result = _unigramTokenizerFromJson.
EncodeToTokens
(inputText.AsSpan(), out normalized);