2 overrides of EncodeToIds
Microsoft.ML.Tokenizers (2)
Model\SentencePieceBpeModel.cs (1)
281
public override IReadOnlyList<int>
EncodeToIds
(string? text, ReadOnlySpan<char> textSpan, bool addBeginningOfSentence, bool addEndOfSentence, bool considerNormalization,
Model\SentencePieceUnigramModel.cs (1)
826
public override IReadOnlyList<int>
EncodeToIds
(
5 references to EncodeToIds
Microsoft.ML.Tokenizers (5)
Model\SentencePieceTokenizer.cs (5)
173
Tokens = _model.
EncodeToIds
(text, textSpan, AddBeginningOfSentence, AddEndOfSentence, settings.ConsiderNormalization, out string? normalizedText, out int charsConsumed, settings.MaxTokenCount),
189
=> _model.
EncodeToIds
(text, Span<char>.Empty, addBeginningOfSentence, addEndOfSentence, considerNormalization, out _, out _);
201
=> _model.
EncodeToIds
(null, text, addBeginningOfSentence, addEndOfSentence, considerNormalization, out _, out _);
216
=> _model.
EncodeToIds
(text, Span<char>.Empty, addBeginningOfSentence, addEndOfSentence, considerNormalization, out normalizedText, out charsConsumed, maxTokenCount);
231
=> _model.
EncodeToIds
(null, text, addBeginningOfSentence, addEndOfSentence, considerNormalization, out normalizedText, out charsConsumed, maxTokenCount);