1 write to UnknownId
Microsoft.ML.Tokenizers (1)
Model\SentencePieceBaseModel.cs (1)
32UnknownId = Math.Max(0, modelProto.TrainerSpec.UnkId);
21 references to UnknownId
Microsoft.ML.Tokenizers (21)
Model\SentencePieceBaseModel.cs (2)
350else if (id == model.UnknownId) 588else if (id == model.UnknownId)
Model\SentencePieceBpeModel.cs (8)
175id = UnknownId; 182if (id == UnknownId && ByteFallback) 453id = UnknownId; 460if (id == UnknownId && ByteFallback) 721id = UnknownId; 728if (id == UnknownId && ByteFallback) 997id = UnknownId; 1004if (id == UnknownId && ByteFallback)
Model\SentencePieceTokenizer.cs (1)
96public int UnknownId => _model.UnknownId;
Model\SentencePieceUnigramModel.cs (10)
362string stringToken = node.Id == UnknownId ? Helpers.GetString(normalizationSpan.Slice(node.StartsAt, endsAt - node.StartsAt)) : _vocabReverse[node.Id].Piece; 367unknownEncountered = unknownEncountered || node.Id == UnknownId; 418if (tokens[insertionStartPosition].Id == UnknownId) 542targetNode.Id = UnknownId; 745Debug.Assert(ids[IdsIndex] == UnknownId); 803if (node.Id == UnknownId && needToTrackUnknown) 872if (ids[index] == UnknownId) 1135unknownEncountered = unknownEncountered || node.Id == UnknownId; 1168if (ids[i].Id == UnknownId) 1378if (node.Id == UnknownId)