2 overrides of CountTokens
Microsoft.ML.Tokenizers (2)
Model\SentencePieceBpeModel.cs (1)
594
public override int
CountTokens
(
Model\SentencePieceUnigramModel.cs (1)
940
public override int
CountTokens
(
8 references to CountTokens
Microsoft.ML.Tokenizers (8)
Model\SentencePieceTokenizer.cs (8)
235
=> _model.
CountTokens
(text, textSpan, AddBeginningOfSentence, AddEndOfSentence, settings.ConsiderNormalization, out _, out _, settings.MaxTokenCount);
247
=> _model.
CountTokens
(text, ReadOnlySpan<char>.Empty, addBeginningOfSentence, addEndOfSentence, considerNormalization, out _, out _, int.MaxValue);
259
=> _model.
CountTokens
(null, text, addBeginningOfSentence, addEndOfSentence, considerNormalization, out _, out _, int.MaxValue);
274
=> _model.
CountTokens
(text, ReadOnlySpan<char>.Empty, addBeginningOfSentence, addEndOfSentence, considerNormalization, out normalizedText, out charsConsumed, maxTokenCount);
289
=> _model.
CountTokens
(null, text, addBeginningOfSentence, addEndOfSentence, considerNormalization, out normalizedText, out charsConsumed, maxTokenCount);
314
tokenCount = _model.
CountTokens
(text, textSpan, AddBeginningOfSentence, AddEndOfSentence, settings.ConsiderNormalization, out normalizedText, out int charsConsumed, settings.MaxTokenCount);
336
tokenCount = _model.
CountTokens
(text, ReadOnlySpan<char>.Empty, addBeginningOfSentence, addEndOfSentence, considerNormalization, out normalizedText, out int charsConsumed, maxTokenCount);
358
tokenCount = _model.
CountTokens
(null, text, addBeginningOfSentence, addEndOfSentence, considerNormalization, out normalizedText, out int charsConsumed, maxTokenCount);