3 overrides of PreTokenize
Microsoft.ML.Tokenizers (2)
PreTokenizer\RegexPreTokenizer.cs (1)
96
public override IEnumerable<(int Offset, int Length)>
PreTokenize
(ReadOnlySpan<char> text)
PreTokenizer\RobertaPreTokenizer.cs (1)
40
public override IEnumerable<(int Offset, int Length)>
PreTokenize
(ReadOnlySpan<char> text)
Microsoft.ML.Tokenizers.Tests (1)
PreTokenizerTests.cs (1)
78
public override IEnumerable<(int Offset, int Length)>
PreTokenize
(ReadOnlySpan<char> text)
1 reference to PreTokenize
Microsoft.ML.Tokenizers (1)
Tokenizer.cs (1)
459
splits = preTokenizer.
PreTokenize
(textSpan);