5 overrides of GetIndexByTokenCount
Microsoft.ML.Tokenizers (5)
Model\BPETokenizer.cs (1)
432
protected override int
GetIndexByTokenCount
(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings, bool fromEnd, out string? normalizedString, out int tokenCount)
Model\CodeGenTokenizer.cs (1)
714
protected override int
GetIndexByTokenCount
(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings, bool fromEnd, out string? normalizedString, out int tokenCount)
Model\EnglishRobertaTokenizer.cs (1)
473
protected override int
GetIndexByTokenCount
(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings, bool fromEnd, out string? normalizedString, out int tokenCount)
Model\SentencePieceBpeTokenizer.cs (1)
1159
protected override int
GetIndexByTokenCount
(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings, bool fromEnd, out string? normalizedString, out int tokenCount)
Model\TiktokenTokenizer.cs (1)
645
protected override int
GetIndexByTokenCount
(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings, bool fromEnd, out string? normalizedString, out int tokenCount)
4 references to GetIndexByTokenCount
Microsoft.ML.Tokenizers (4)
Tokenizer.cs (4)
276
=>
GetIndexByTokenCount
(
299
=>
GetIndexByTokenCount
(
322
=>
GetIndexByTokenCount
(
345
=>
GetIndexByTokenCount
(