6 overrides of CountTokens
Microsoft.ML.Tokenizers (6)
Model\BPETokenizer.cs (1)
412
protected override int
CountTokens
(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings)
Model\CodeGenTokenizer.cs (1)
674
protected override int
CountTokens
(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings)
Model\EnglishRobertaTokenizer.cs (1)
465
protected override int
CountTokens
(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings)
Model\SentencePieceTokenizer.cs (1)
244
protected override int
CountTokens
(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings)
Model\TiktokenTokenizer.cs (1)
528
protected override int
CountTokens
(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings)
Model\WordPieceTokenizer.cs (1)
541
protected override int
CountTokens
(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings)
2 references to CountTokens
Microsoft.ML.Tokenizers (2)
Tokenizer.cs (2)
194
=>
CountTokens
(text, text.AsSpan(), new EncodeSettings { ConsiderPreTokenization = considerPreTokenization, ConsiderNormalization = considerNormalization });
204
=>
CountTokens
(null, text, new EncodeSettings { ConsiderPreTokenization = considerPreTokenization, ConsiderNormalization = considerNormalization });