4 overrides of PreTokenize
Microsoft.ML.Tokenizers (3)
PreTokenizer\RobertaPreTokenizer.cs (1)
25
public override IEnumerable<(int Offset, int Length)>
PreTokenize
(string text)
PreTokenizer\TiktokenPreTokenizer.cs (1)
47
public override IEnumerable<(int Offset, int Length)>
PreTokenize
(string text)
PreTokenizer\WhiteSpacePreTokenizer.cs (1)
36
public override IEnumerable<(int Offset, int Length)>
PreTokenize
(string text)
Microsoft.ML.Tokenizers.Tests (1)
PreTokenizerTests.cs (1)
109
public override IEnumerable<(int Offset, int Length)>
PreTokenize
(string text)
5 references to PreTokenize
Microsoft.ML.Tokenizers (3)
Tokenizer.cs (3)
450
splits = preTokenizer.
PreTokenize
(normalizedString);
472
splits = preTokenizer.
PreTokenize
(normalizedString);
481
splits = preTokenizer.
PreTokenize
(text);
Microsoft.ML.Tokenizers.Tests (1)
PreTokenizerTests.cs (1)
53
(int Offset, int Length)[] splitParts = preTokenizer.
PreTokenize
(text).ToArray<(int Offset, int Length)>();
Microsoft.ML.TorchSharp (1)
NasBert\NerTrainer.cs (1)
379
var pre = tokenizer.PreTokenizer.
PreTokenize
(sentence);