4 overrides of PreTokenize
Microsoft.ML.Tokenizers (3)
PreTokenizer\CompositePreTokenizer.cs (1)
144
public override IEnumerable<(int Offset, int Length)>
PreTokenize
(ReadOnlySpan<char> text)
PreTokenizer\RegexPreTokenizer.cs (1)
96
public override IEnumerable<(int Offset, int Length)>
PreTokenize
(ReadOnlySpan<char> text)
PreTokenizer\RobertaPreTokenizer.cs (1)
40
public override IEnumerable<(int Offset, int Length)>
PreTokenize
(ReadOnlySpan<char> text)
Microsoft.ML.Tokenizers.Tests (1)
PreTokenizerTests.cs (1)
78
public override IEnumerable<(int Offset, int Length)>
PreTokenize
(ReadOnlySpan<char> text)
3 references to PreTokenize
Microsoft.ML.Tokenizers (3)
PreTokenizer\CompositePreTokenizer.cs (2)
98
foreach ((int Offset, int Length) range in preTokenizer.
PreTokenize
(text.AsSpan(offset, length)))
176
foreach ((int Offset, int Length) range in preTokenizer.
PreTokenize
(text.AsSpan(offset, length)))
Tokenizer.cs (1)
459
splits = preTokenizer.
PreTokenize
(textSpan);