10 references to GetIndexByTokenCount
Microsoft.ML.Tokenizers.Tests (10)
BpeTests.cs (1)
463
Assert.Equal(expectedOffsets[expectedOffsets.Length - 4].Index + expectedOffsets[expectedOffsets.Length - 4].Length, tokenizer.
GetIndexByTokenCount
(text.AsSpan(), expectedIds.Length - 3, out normalizedString, out tokenCount));
CodeGenTests.cs (5)
446
Assert.Equal(offsets[offsets.Length - 1].Index + offsets[offsets.Length - 1].Length, codeGenTokenizer.
GetIndexByTokenCount
(text.AsSpan(), ids.Length, out normalizedString, out tokenCount));
641
length = codeGenTokenizer.
GetIndexByTokenCount
(text.AsSpan(), maxTokenCount: 500, out normalizedString, out count);
774
length = codeGenTokenizer.
GetIndexByTokenCount
(text.AsSpan(), maxTokenCount: 500, out normalizedString, out count);
928
length = codeGenTokenizer.
GetIndexByTokenCount
(text.AsSpan(), maxTokenCount: 500, out normalizedString, out count);
1012
charsConsumed = _codegen350MMonoTokenizer.
GetIndexByTokenCount
(input.AsSpan(), maxTokenCount, out _, out tokenCount);
EnglishRobertaTests.cs (1)
215
Assert.Equal(expectedOffsets[expectedOffsets.Length - 4].Index + expectedOffsets[expectedOffsets.Length - 4].Length, tokenizer.
GetIndexByTokenCount
(text.AsSpan(), expectedIds.Length - 3, out normalizedString, out tokenCount));
LlamaTests.cs (1)
638
Assert.Equal(expectedOffsets[expectedOffsets.Length - 7].Index + expectedOffsets[expectedOffsets.Length - 7].Length, tokenizer.
GetIndexByTokenCount
(text.AsSpan(), expectedIds.Length - 6, out normalizedString, out tokenCount));
TitokenTests.cs (1)
598
Assert.Equal(expectedOffsets[expectedOffsets.Length - 4].Index + expectedOffsets[expectedOffsets.Length - 4].Length, tokenizer.
GetIndexByTokenCount
(text.AsSpan(), expectedIds.Length - 3, out normalizedString, out tokenCount));
TokenizerTests.cs (1)
218
Assert.Equal(0, tokenizer.
GetIndexByTokenCount
(Span<char>.Empty, maxTokenCount: 10, out _, out _));