2 overrides of GetIndexByTokenCountFromEnd
Microsoft.ML.Tokenizers (2)
Model\SentencePieceBpeModel.cs (1)
898
public override int
GetIndexByTokenCountFromEnd
(string? text, ReadOnlySpan<char> textSpan, bool addBeginningOfSentence, bool addEndOfSentence, int maxTokenCount, bool considerNormalization, out string? normalizedText, out int tokenCount)
Model\SentencePieceUnigramModel.cs (1)
1260
public override int
GetIndexByTokenCountFromEnd
(
3 references to GetIndexByTokenCountFromEnd
Microsoft.ML.Tokenizers (3)
Model\SentencePieceTokenizer.cs (3)
321
return _model.
GetIndexByTokenCountFromEnd
(text, textSpan, AddBeginningOfSentence, AddEndOfSentence, settings.MaxTokenCount, settings.ConsiderNormalization, out normalizedText, out tokenCount);
387
=> _model.
GetIndexByTokenCountFromEnd
(text, ReadOnlySpan<char>.Empty, addBeginningOfSentence, addEndOfSentence, maxTokenCount, considerNormalization, out normalizedText, out tokenCount);
404
=> _model.
GetIndexByTokenCountFromEnd
(null, text, addBeginningOfSentence, addEndOfSentence, maxTokenCount, considerNormalization, out normalizedText, out tokenCount);