6 references to Segment
Microsoft.ML.Tokenizers (6)
Model\SentencePieceTokenizer.cs (6)
721if (!Segment(symbols[index].pieceSpan, text, ref charsConsumed)) 826return Segment((merge.LeftIndex, merge.LeftLen), text, ref charsConsumed) && Segment((merge.RightIndex, merge.RightLen), text, ref charsConsumed); 1042if (!Segment(symbols[index].pieceSpan, text, ref charsConsumed)) 1140return Segment((merge.LeftIndex, merge.LeftLen), text, ref charsConsumed) && Segment((merge.RightIndex, merge.RightLen), text, ref charsConsumed);