3 writes to StartsAt
Microsoft.ML.Tokenizers (3)
Model\SentencePieceUnigramModel.cs (3)
458StartsAt = -1; 522targetNode.StartsAt = startsAt; 541targetNode.StartsAt = startsAt;
14 references to StartsAt
Microsoft.ML.Tokenizers (14)
Model\SentencePieceUnigramModel.cs (14)
362string stringToken = node.Id == UnknownId ? Helpers.GetString(normalizationSpan.Slice(node.StartsAt, endsAt - node.StartsAt)) : _vocabReverse[node.Id].Piece; 366endsAt = node.StartsAt; 519if (targetNode.StartsAt == -1 || candidateBestPathScore > targetNode.BestPathScore) 538if (targetNode.StartsAt == -1 || candidateBestPathScore > targetNode.BestPathScore) 815unknownTokensTracking[unknownTokensCount - 1] = (ids.Count - 1, node.StartsAt, endsAt - node.StartsAt); 818endsAt = node.StartsAt; 1133ids[idsIndex--] = (node.Id, node.StartsAt, endsAt - node.StartsAt); 1137endsAt = node.StartsAt; 1380int length = endsAt - node.StartsAt; 1395consumedCharacters += Helpers.GetUtf16LengthFromUtf8Bytes(normalizationSpan.Slice(node.StartsAt, length)); 1403endsAt = node.StartsAt;