3 references to Segment
Microsoft.ML.Tokenizers (3)
Model\SentencePieceTokenizer.cs (3)
360
Segment
(symbols[index].pieceSpan, text);
440
Segment
((merge.LeftIndex, merge.LeftLen), text);
441
Segment
((merge.RightIndex, merge.RightLen), text);