11 references to EncodeToIds
Microsoft.ML.Tokenizers.Tests (11)
BpeTests.cs (1)
441
Assert.Equal(expectedIds, tokenizer.
EncodeToIds
(text.AsSpan()));
CodeGenTests.cs (4)
372
Assert.Equal(ids, tokenizer.
EncodeToIds
(text.AsSpan()));
611
ids = codeGenTokenizer.
EncodeToIds
(text.AsSpan());
744
ids = codeGenTokenizer.
EncodeToIds
(text.AsSpan());
892
ids = codeGenTokenizer.
EncodeToIds
(text.AsSpan());
EnglishRobertaTests.cs (1)
193
Assert.Equal(expectedIds, tokenizer.
EncodeToIds
(text.AsSpan()));
LlamaTests.cs (2)
342
Assert.Equal([], llamaTokenizer.
EncodeToIds
(Span<char>.Empty));
557
Assert.Equal(expectedIds, tokenizer.
EncodeToIds
(text.AsSpan()));
TitokenTests.cs (1)
576
Assert.Equal(expectedIds, tokenizer.
EncodeToIds
(text.AsSpan()));
TokenizerTests.cs (2)
159
prefixIds = tokenizer.
EncodeToIds
(prefixString.AsSpan(), considerNormalization: false);
177
suffixIds = tokenizer.
EncodeToIds
(suffixString.AsSpan(), considerNormalization: false);