5 overrides of CountTokens
Microsoft.ML.Tokenizers (5)
Model\BPETokenizer.cs (1)
367
protected override int
CountTokens
(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings)
Model\CodeGenTokenizer.cs (1)
669
protected override int
CountTokens
(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings)
Model\EnglishRobertaTokenizer.cs (1)
454
protected override int
CountTokens
(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings)
Model\SentencePieceBpeTokenizer.cs (1)
836
protected override int
CountTokens
(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings)
Model\TiktokenTokenizer.cs (1)
527
protected override int
CountTokens
(string? text, ReadOnlySpan<char> textSpan, EncodeSettings settings)
2 references to CountTokens
Microsoft.ML.Tokenizers (2)
Tokenizer.cs (2)
194
=>
CountTokens
(text, text.AsSpan(), new EncodeSettings { ConsiderPreTokenization = considerPreTokenization, ConsiderNormalization = considerNormalization });
204
=>
CountTokens
(null, text, new EncodeSettings { ConsiderPreTokenization = considerPreTokenization, ConsiderNormalization = considerNormalization });