6 overrides of GetIndexByTokenCount
Microsoft.ML.Tokenizers (6)
Model\BPETokenizer.cs (1)
477
protected override int
GetIndexByTokenCount
(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings, bool fromEnd, out string? normalizedText, out int tokenCount)
Model\CodeGenTokenizer.cs (1)
716
protected override int
GetIndexByTokenCount
(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings, bool fromEnd, out string? normalizedText, out int tokenCount)
Model\EnglishRobertaTokenizer.cs (1)
484
protected override int
GetIndexByTokenCount
(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings, bool fromEnd, out string? normalizedText, out int tokenCount)
Model\SentencePieceTokenizer.cs (1)
1160
protected override int
GetIndexByTokenCount
(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings, bool fromEnd, out string? normalizedText, out int tokenCount)
Model\TiktokenTokenizer.cs (1)
646
protected override int
GetIndexByTokenCount
(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings, bool fromEnd, out string? normalizedText, out int tokenCount)
Model\WordPieceTokenizer.cs (1)
602
protected override int
GetIndexByTokenCount
(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings, bool fromEnd, out string? normalizedText, out int tokenCount)
4 references to GetIndexByTokenCount
Microsoft.ML.Tokenizers (4)
Tokenizer.cs (4)
276
=>
GetIndexByTokenCount
(
299
=>
GetIndexByTokenCount
(
322
=>
GetIndexByTokenCount
(
345
=>
GetIndexByTokenCount
(