3 writes to Id
Microsoft.ML.Tokenizers (3)
Model\SentencePieceUnigramModel.cs (3)
506
Id
= -1;
573
targetNode.
Id
= ret;
592
targetNode.
Id
= UnknownId;
10 references to Id
Microsoft.ML.Tokenizers (10)
Model\SentencePieceUnigramModel.cs (10)
412
string stringToken = node.
Id
== UnknownId ? Helpers.GetString(normalizationSpan.Slice(node.StartsAt, endsAt - node.StartsAt)) : _vocabReverse[node.
Id
].Piece;
415
tokens.Add(new EncodedToken(node.
Id
, stringToken, new Range(0, tokenLength))); // we will update the range later.
417
unknownEncountered = unknownEncountered || node.
Id
== UnknownId;
851
ids.Add(node.
Id
);
853
if (node.
Id
== UnknownId && needToTrackUnknown)
1183
ids[idsIndex--] = (node.
Id
, node.StartsAt, endsAt - node.StartsAt);
1185
unknownEncountered = unknownEncountered || node.
Id
== UnknownId;
1428
if (node.
Id
== UnknownId)
1449
consumedCharacters += _vocabReverse[node.
Id
].Piece.Length;