12 references to EncodeToIds
Microsoft.ML.Tokenizers (1)
Model\BertTokenizer.cs (1)
263
IReadOnlyList<int> ids = text is null ? base.
EncodeToIds
(textSpan, considerPreTokenization, considerNormalization) : base.EncodeToIds(text, considerPreTokenization, considerNormalization);
Microsoft.ML.Tokenizers.Tests (11)
BpeTests.cs (1)
441
Assert.Equal(expectedIds, tokenizer.
EncodeToIds
(text.AsSpan()));
CodeGenTests.cs (4)
372
Assert.Equal(ids, tokenizer.
EncodeToIds
(text.AsSpan()));
611
ids = codeGenTokenizer.
EncodeToIds
(text.AsSpan());
744
ids = codeGenTokenizer.
EncodeToIds
(text.AsSpan());
892
ids = codeGenTokenizer.
EncodeToIds
(text.AsSpan());
EnglishRobertaTests.cs (1)
193
Assert.Equal(expectedIds, tokenizer.
EncodeToIds
(text.AsSpan()));
LlamaTests.cs (2)
342
Assert.Equal([], llamaTokenizer.
EncodeToIds
(Span<char>.Empty));
557
Assert.Equal(expectedIds, tokenizer.
EncodeToIds
(text.AsSpan()));
TiktokenTests.cs (1)
580
Assert.Equal(expectedIds, tokenizer.
EncodeToIds
(text.AsSpan()));
TokenizerTests.cs (2)
159
prefixIds = tokenizer.
EncodeToIds
(prefixString.AsSpan(), considerNormalization: false);
177
suffixIds = tokenizer.
EncodeToIds
(suffixString.AsSpan(), considerNormalization: false);