3 references to SplitText
Microsoft.ML.Tokenizers (3)
PreTokenizer\CompositePreTokenizer.cs (3)
164foreach ((int Offset, int Length) range in SplitText(text, _preTokenizers, preTokenizerIndex: 0, beginning, length - beginning))
183foreach ((int Offset, int Length) subRange in SplitText(text, preTokenizers, preTokenizerIndex + 1, offset + beginning, range.Offset - beginning))
204foreach ((int Offset, int Length) subRange in SplitText(text, preTokenizers, preTokenizerIndex + 1, offset + beginning, length - beginning))