12 references to EncodeToIds
Microsoft.ML.Tokenizers.Tests (12)
BpeTests.cs (2)
442
Assert.Equal(expectedIds, tokenizer.
EncodeToIds
(text, expectedIds.Length, out string? normalizedString, out int length));
449
Assert.Equal(expectedIds.Take(expectedIds.Length - 2), tokenizer.
EncodeToIds
(text, expectedIds.Length - 2, out normalizedString, out length));
CodeGenTests.cs (3)
379
Assert.Equal(ids, codeGenTokenizer.
EncodeToIds
(text, ids.Length, out string? normalizedString, out int length));
401
Assert.Equal(ids.Take(ids.Length - expectedTokensToExclude), codeGenTokenizer.
EncodeToIds
(text, ids.Length - 1, out normalizedString, out length));
997
IReadOnlyList<int> ids = _codegen350MMonoTokenizer.
EncodeToIds
(input, maxTokenCount, out _, out int charsConsumed);
EnglishRobertaTests.cs (2)
194
Assert.Equal(expectedIds, tokenizer.
EncodeToIds
(text, expectedIds.Length, out string? normalizedString, out int length));
201
Assert.Equal(expectedIds.Take(expectedIds.Length - 2), tokenizer.
EncodeToIds
(text, expectedIds.Length - 2, out normalizedString, out length));
LlamaTests.cs (1)
558
Assert.Equal(expectedIds, tokenizer.
EncodeToIds
(text, expectedIds.Length, out string? normalizedString, out int length));
TitokenTests.cs (2)
577
Assert.Equal(expectedIds, tokenizer.
EncodeToIds
(text, expectedIds.Length, out string? normalizedString, out int length));
584
Assert.Equal(expectedIds.Take(expectedIds.Length - 4), tokenizer.
EncodeToIds
(text, expectedIds.Length - 4, out normalizedString, out length));
TokenizerTests.cs (2)
36
IReadOnlyList<int> ids = tokenizer.
EncodeToIds
("hello, world", 5, out string? normalizedText, out int charsConsumed);
139
IReadOnlyList<int> partialIdsList = tokenizer.
EncodeToIds
(input, maxTokenCount: i, out string? processedText, out int charsConsumed);