10 references to EncodeToIds
Microsoft.ML.Tokenizers.Tests (10)
BpeTests.cs (2)
445
Assert.Equal(expectedIds, tokenizer.
EncodeToIds
(text.AsSpan(), expectedIds.Length, out normalizedString, out length));
453
Assert.Equal(expectedIds.Take(expectedIds.Length - 2), tokenizer.
EncodeToIds
(text.AsSpan(), expectedIds.Length - 2, out normalizedString, out length));
CodeGenTests.cs (3)
382
Assert.Equal(ids, codeGenTokenizer.
EncodeToIds
(text.AsSpan(), ids.Length, out normalizedString, out length));
406
Assert.Equal(ids.Take(ids.Length - expectedTokensToExclude), codeGenTokenizer.
EncodeToIds
(text.AsSpan(), ids.Length - 1, out normalizedString, out length));
1002
ids = _codegen350MMonoTokenizer.
EncodeToIds
(input.AsSpan(), maxTokenCount, out _, out charsConsumed);
EnglishRobertaTests.cs (2)
197
Assert.Equal(expectedIds, tokenizer.
EncodeToIds
(text.AsSpan(), expectedIds.Length, out normalizedString, out length));
205
Assert.Equal(expectedIds.Take(expectedIds.Length - 2), tokenizer.
EncodeToIds
(text.AsSpan(), expectedIds.Length - 2, out normalizedString, out length));
LlamaTests.cs (1)
561
Assert.Equal(expectedIds, tokenizer.
EncodeToIds
(text.AsSpan(), expectedIds.Length, out normalizedString, out length));
TitokenTests.cs (2)
580
Assert.Equal(expectedIds, tokenizer.
EncodeToIds
(text.AsSpan(), expectedIds.Length, out normalizedString, out length));
588
Assert.Equal(expectedIds.Take(expectedIds.Length - 4), tokenizer.
EncodeToIds
(text.AsSpan(), expectedIds.Length - 4, out normalizedString, out length));