2 writes to UnknownId
Microsoft.ML.Tokenizers (2)
Model\SentencePieceBaseModel.cs (2)
32UnknownId = Math.Max(0, modelProto.TrainerSpec.UnkId); 79UnknownId = unkId;
21 references to UnknownId
Microsoft.ML.Tokenizers (21)
Model\SentencePieceBaseModel.cs (2)
394else if (id == model.UnknownId) 632else if (id == model.UnknownId)
Model\SentencePieceBpeModel.cs (8)
175id = UnknownId; 182if (id == UnknownId && ByteFallback) 453id = UnknownId; 460if (id == UnknownId && ByteFallback) 721id = UnknownId; 728if (id == UnknownId && ByteFallback) 997id = UnknownId; 1004if (id == UnknownId && ByteFallback)
Model\SentencePieceTokenizer.cs (1)
102public int UnknownId => _model.UnknownId;
Model\SentencePieceUnigramModel.cs (10)
637string stringToken = node.Id == UnknownId ? Helpers.GetString(normalizationSpan.Slice(node.StartsAt, endsAt - node.StartsAt)) : _vocabReverse[node.Id].Piece; 642unknownEncountered = unknownEncountered || node.Id == UnknownId; 693if (tokens[insertionStartPosition].Id == UnknownId) 817targetNode.Id = UnknownId; 1031Debug.Assert(ids[IdsIndex] == UnknownId); 1089if (node.Id == UnknownId && needToTrackUnknown) 1158if (ids[index] == UnknownId) 1431unknownEncountered = unknownEncountered || node.Id == UnknownId; 1464if (ids[i].Id == UnknownId) 1684if (node.Id == UnknownId)