Model\SentencePieceTokenizer.cs (6)
720if (!Segment(symbols[index].pieceSpan, text, ref charsConsumed))
825return Segment((merge.LeftIndex, merge.LeftLen), text, ref charsConsumed) && Segment((merge.RightIndex, merge.RightLen), text, ref charsConsumed);
1041if (!Segment(symbols[index].pieceSpan, text, ref charsConsumed))
1139return Segment((merge.LeftIndex, merge.LeftLen), text, ref charsConsumed) && Segment((merge.RightIndex, merge.RightLen), text, ref charsConsumed);