2 overrides of EncodeToIds
Microsoft.ML.Tokenizers (2)
Model\SentencePieceBpeModel.cs (1)
327
public override IReadOnlyList<int>
EncodeToIds
(string? text, ReadOnlySpan<char> textSpan, bool addBeginningOfSentence, bool addEndOfSentence, bool considerNormalization,
Model\SentencePieceUnigramModel.cs (1)
601
public override IReadOnlyList<int>
EncodeToIds
(
5 references to EncodeToIds
Microsoft.ML.Tokenizers (5)
Model\SentencePieceTokenizer.cs (5)
177
Tokens = _model.
EncodeToIds
(text, textSpan, AddBeginningOfSentence, AddEndOfSentence, settings.ConsiderNormalization, out string? normalizedText, out int charsConsumed, settings.MaxTokenCount),
193
=> _model.
EncodeToIds
(text, Span<char>.Empty, addBeginningOfSentence, addEndOfSentence, considerNormalization, out _, out _);
205
=> _model.
EncodeToIds
(null, text, addBeginningOfSentence, addEndOfSentence, considerNormalization, out _, out _);
220
=> _model.
EncodeToIds
(text, Span<char>.Empty, addBeginningOfSentence, addEndOfSentence, considerNormalization, out normalizedText, out charsConsumed, maxTokenCount);
235
=> _model.
EncodeToIds
(null, text, addBeginningOfSentence, addEndOfSentence, considerNormalization, out normalizedText, out charsConsumed, maxTokenCount);