2 references to GetIndexByTokenCount
Microsoft.ML.Tokenizers.Tests (2)
LlamaTests.cs (1)
691
int index = tokenizer.
GetIndexByTokenCount
(s, addBeginningOfSentence: false, addEndOfSentence: false, maxTokenCount: i, out normalizedText, out tokenCount);
UnigramTests.cs (1)
497
int index = _unigramTokenizer.
GetIndexByTokenCount
(inputText, addBeginningOfSentence: false, addEndOfSentence: false, maxTokenCount: 1, out string? normalized, out int charConsumed);