3 writes to StartsAt
Microsoft.ML.Tokenizers (3)
Model\SentencePieceUnigramModel.cs (3)
508StartsAt = -1; 572targetNode.StartsAt = startsAt; 591targetNode.StartsAt = startsAt;
14 references to StartsAt
Microsoft.ML.Tokenizers (14)
Model\SentencePieceUnigramModel.cs (14)
412string stringToken = node.Id == UnknownId ? Helpers.GetString(normalizationSpan.Slice(node.StartsAt, endsAt - node.StartsAt)) : _vocabReverse[node.Id].Piece; 416endsAt = node.StartsAt; 569if (targetNode.StartsAt == -1 || candidateBestPathScore > targetNode.BestPathScore) 588if (targetNode.StartsAt == -1 || candidateBestPathScore > targetNode.BestPathScore) 865unknownTokensTracking[unknownTokensCount - 1] = (ids.Count - 1, node.StartsAt, endsAt - node.StartsAt); 868endsAt = node.StartsAt; 1183ids[idsIndex--] = (node.Id, node.StartsAt, endsAt - node.StartsAt); 1187endsAt = node.StartsAt; 1430int length = endsAt - node.StartsAt; 1445consumedCharacters += Helpers.GetUtf16LengthFromUtf8Bytes(normalizationSpan.Slice(node.StartsAt, length)); 1453endsAt = node.StartsAt;