10 references to GetIndexByTokenCountFromEnd
Microsoft.ML.Tokenizers.Tests (10)
BpeTests.cs (1)
470
Assert.Equal(expectedOffsets[expectedOffsets.Length - 3].Index, tokenizer.
GetIndexByTokenCountFromEnd
(text.AsSpan(), 3, out normalizedString, out tokenCount));
CodeGenTests.cs (5)
473
Assert.Equal(expectedIndex, codeGenTokenizer.
GetIndexByTokenCountFromEnd
(text.AsSpan(), 1, out normalizedString, out tokenCount));
660
index = codeGenTokenizer.
GetIndexByTokenCountFromEnd
(text.AsSpan(), maxTokenCount: 500, out normalizedString, out count);
793
index = codeGenTokenizer.
GetIndexByTokenCountFromEnd
(text.AsSpan(), maxTokenCount: 500, out normalizedString, out count);
947
index = codeGenTokenizer.
GetIndexByTokenCountFromEnd
(text.AsSpan(), maxTokenCount: 500, out normalizedString, out count);
1022
charsConsumed = _codegen350MMonoTokenizer.
GetIndexByTokenCountFromEnd
(input.AsSpan(), maxTokenCount, out _, out tokenCount);
EnglishRobertaTests.cs (1)
222
Assert.Equal(expectedOffsets[expectedOffsets.Length - 3].Index, tokenizer.
GetIndexByTokenCountFromEnd
(text.AsSpan(), 3, out normalizedString, out tokenCount));
LlamaTests.cs (1)
645
Assert.Equal(expectedOffsets[expectedOffsets.Length - 7].Index, tokenizer.
GetIndexByTokenCountFromEnd
(text.AsSpan(), 7, out normalizedString, out tokenCount));
TitokenTests.cs (1)
605
Assert.Equal(expectedOffsets[expectedOffsets.Length - 3].Index, tokenizer.
GetIndexByTokenCountFromEnd
(text.AsSpan(), 3, out normalizedString, out tokenCount));
TokenizerTests.cs (1)
219
Assert.Equal(0, tokenizer.
GetIndexByTokenCountFromEnd
(Span<char>.Empty, maxTokenCount: 10, out _, out _));