3 writes to Id
Microsoft.ML.Tokenizers (3)
Model\SentencePieceUnigramModel.cs (3)
731
Id
= -1;
798
targetNode.
Id
= ret;
817
targetNode.
Id
= UnknownId;
10 references to Id
Microsoft.ML.Tokenizers (10)
Model\SentencePieceUnigramModel.cs (10)
637
string stringToken = node.
Id
== UnknownId ? Helpers.GetString(normalizationSpan.Slice(node.StartsAt, endsAt - node.StartsAt)) : _vocabReverse[node.
Id
].Piece;
640
tokens.Add(new EncodedToken(node.
Id
, stringToken, new Range(0, tokenLength))); // we will update the range later.
642
unknownEncountered = unknownEncountered || node.
Id
== UnknownId;
1087
ids.Add(node.
Id
);
1089
if (node.
Id
== UnknownId && needToTrackUnknown)
1429
ids[idsIndex--] = (node.
Id
, node.StartsAt, endsAt - node.StartsAt);
1431
unknownEncountered = unknownEncountered || node.
Id
== UnknownId;
1684
if (node.
Id
== UnknownId)
1705
consumedCharacters += _vocabReverse[node.
Id
].Piece.Length;