6 overrides of EncodeToIds
Microsoft.ML.Tokenizers (6)
Model\BPETokenizer.cs (1)
356
protected override EncodeResults<int>
EncodeToIds
(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings)
Model\CodeGenTokenizer.cs (1)
488
protected override EncodeResults<int>
EncodeToIds
(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings)
Model\EnglishRobertaTokenizer.cs (1)
407
protected override EncodeResults<int>
EncodeToIds
(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings)
Model\SentencePieceTokenizer.cs (1)
453
protected override EncodeResults<int>
EncodeToIds
(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings)
Model\TiktokenTokenizer.cs (1)
362
protected override EncodeResults<int>
EncodeToIds
(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings)
Model\WordPieceTokenizer.cs (1)
389
protected override EncodeResults<int>
EncodeToIds
(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings)
4 references to EncodeToIds
Microsoft.ML.Tokenizers (4)
Tokenizer.cs (4)
68
=>
EncodeToIds
(text, text.AsSpan(), new EncodeSettings { ConsiderPreTokenization = considerPreTokenization, ConsiderNormalization = considerNormalization }).Tokens;
78
=>
EncodeToIds
(null, text, new EncodeSettings { ConsiderPreTokenization = considerPreTokenization, ConsiderNormalization = considerNormalization }).Tokens;
92
EncodeResults<int> result =
EncodeToIds
(text, text.AsSpan(),
118
EncodeResults<int> result =
EncodeToIds
(null, text,