3 references to SplitText
Microsoft.ML.Tokenizers (3)
PreTokenizer\CompositePreTokenizer.cs (3)
84foreach ((int Offset, int Length) range in SplitText(text, _preTokenizers, preTokenizerIndex: 0, beginning, text.Length - beginning))
103foreach ((int Offset, int Length) subRange in SplitText(text, preTokenizers, preTokenizerIndex + 1, offset + beginning, range.Offset - beginning))
124foreach ((int Offset, int Length) subRange in SplitText(text, preTokenizers, preTokenizerIndex + 1, offset + beginning, length - beginning))