4 references to EncodeToIds
Microsoft.ML.Tokenizers (4)
Model\BertTokenizer.cs (4)
142EncodeToIds(text, ReadOnlySpan<char>.Empty, addSpecialTokens: true, considerPreTokenization, considerNormalization);
152EncodeToIds(null, text, addSpecialTokens: true, considerPreTokenization, considerNormalization);
163EncodeToIds(text, ReadOnlySpan<char>.Empty, addSpecialTokens, considerPreTokenization, considerNormalization);
174EncodeToIds(null, text, addSpecialTokens, considerPreTokenization, considerNormalization);