3 writes to Id
Microsoft.ML.Tokenizers (3)
Model\SentencePieceUnigramModel.cs (3)
456
Id
= -1;
523
targetNode.
Id
= ret;
542
targetNode.
Id
= UnknownId;
10 references to Id
Microsoft.ML.Tokenizers (10)
Model\SentencePieceUnigramModel.cs (10)
362
string stringToken = node.
Id
== UnknownId ? Helpers.GetString(normalizationSpan.Slice(node.StartsAt, endsAt - node.StartsAt)) : _vocabReverse[node.
Id
].Piece;
365
tokens.Add(new EncodedToken(node.
Id
, stringToken, new Range(0, tokenLength))); // we will update the range later.
367
unknownEncountered = unknownEncountered || node.
Id
== UnknownId;
801
ids.Add(node.
Id
);
803
if (node.
Id
== UnknownId && needToTrackUnknown)
1133
ids[idsIndex--] = (node.
Id
, node.StartsAt, endsAt - node.StartsAt);
1135
unknownEncountered = unknownEncountered || node.
Id
== UnknownId;
1378
if (node.
Id
== UnknownId)
1399
consumedCharacters += _vocabReverse[node.
Id
].Piece.Length;