3 overrides of PreTokenize
Microsoft.ML.Tokenizers (2)
PreTokenizer\RegexPreTokenizer.cs (1)
96public override IEnumerable<(int Offset, int Length)> PreTokenize(ReadOnlySpan<char> text)
PreTokenizer\RobertaPreTokenizer.cs (1)
40public override IEnumerable<(int Offset, int Length)> PreTokenize(ReadOnlySpan<char> text)
Microsoft.ML.Tokenizers.Tests (1)
PreTokenizerTests.cs (1)
78public override IEnumerable<(int Offset, int Length)> PreTokenize(ReadOnlySpan<char> text)
1 reference to PreTokenize
Microsoft.ML.Tokenizers (1)
Tokenizer.cs (1)
459splits = preTokenizer.PreTokenize(textSpan);