5 overrides of EncodeToIds
Microsoft.ML.Tokenizers (5)
Model\BPETokenizer.cs (1)
311
protected override EncodeResults<int>
EncodeToIds
(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings)
Model\CodeGenTokenizer.cs (1)
486
protected override EncodeResults<int>
EncodeToIds
(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings)
Model\EnglishRobertaTokenizer.cs (1)
396
protected override EncodeResults<int>
EncodeToIds
(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings)
Model\SentencePieceBpeTokenizer.cs (1)
452
protected override EncodeResults<int>
EncodeToIds
(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings)
Model\TiktokenTokenizer.cs (1)
361
protected override EncodeResults<int>
EncodeToIds
(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings)
4 references to EncodeToIds
Microsoft.ML.Tokenizers (4)
Tokenizer.cs (4)
68
=>
EncodeToIds
(text, text.AsSpan(), new EncodeSettings { ConsiderPreTokenization = considerPreTokenization, ConsiderNormalization = considerNormalization }).Tokens;
78
=>
EncodeToIds
(null, text, new EncodeSettings { ConsiderPreTokenization = considerPreTokenization, ConsiderNormalization = considerNormalization }).Tokens;
92
EncodeResults<int> result =
EncodeToIds
(text, text.AsSpan(),
118
EncodeResults<int> result =
EncodeToIds
(null, text,