4 overrides of PreTokenize
Microsoft.ML.Tokenizers (3)
PreTokenizer\CompositePreTokenizer.cs (1)
142
public override IEnumerable<(int Offset, int Length)>
PreTokenize
(ReadOnlySpan<char> text)
PreTokenizer\RegexPreTokenizer.cs (1)
96
public override IEnumerable<(int Offset, int Length)>
PreTokenize
(ReadOnlySpan<char> text)
PreTokenizer\RobertaPreTokenizer.cs (1)
40
public override IEnumerable<(int Offset, int Length)>
PreTokenize
(ReadOnlySpan<char> text)
Microsoft.ML.Tokenizers.Tests (1)
PreTokenizerTests.cs (1)
78
public override IEnumerable<(int Offset, int Length)>
PreTokenize
(ReadOnlySpan<char> text)
3 references to PreTokenize
Microsoft.ML.Tokenizers (3)
PreTokenizer\CompositePreTokenizer.cs (2)
96
foreach ((int Offset, int Length) range in preTokenizer.
PreTokenize
(text.AsSpan(offset, length)))
174
foreach ((int Offset, int Length) range in preTokenizer.
PreTokenize
(text.AsSpan(offset, length)))
Tokenizer.cs (1)
459
splits = preTokenizer.
PreTokenize
(textSpan);