3 overrides of PreTokenize
Microsoft.ML.Tokenizers (2)
PreTokenizer\RegexPreTokenizer.cs (1)
48public override IEnumerable<(int Offset, int Length)> PreTokenize(string text)
PreTokenizer\RobertaPreTokenizer.cs (1)
25public override IEnumerable<(int Offset, int Length)> PreTokenize(string text)
Microsoft.ML.Tokenizers.Tests (1)
PreTokenizerTests.cs (1)
116public override IEnumerable<(int Offset, int Length)> PreTokenize(string text)
6 references to PreTokenize
Microsoft.ML.Tokenizers (3)
Tokenizer.cs (3)
450splits = preTokenizer.PreTokenize(normalizedText); 472splits = preTokenizer.PreTokenize(normalizedText); 481splits = preTokenizer.PreTokenize(text);
Microsoft.ML.Tokenizers.Tests (2)
PreTokenizerTests.cs (2)
60(int Offset, int Length)[] splitParts = preTokenizer.PreTokenize(text).ToArray<(int Offset, int Length)>(); 73Assert.Empty(PreTokenizer.CreateWordOrNonWord().PreTokenize((string)null!));
Microsoft.ML.TorchSharp (1)
NasBert\NerTrainer.cs (1)
379var pre = tokenizer.PreTokenizer.PreTokenize(sentence);