3 writes to UnknownId
Microsoft.ML.Tokenizers (3)
Model\SentencePieceBaseModel.cs (1)
32UnknownId = Math.Max(0, modelProto.TrainerSpec.UnkId);
Model\SentencePieceBpeModel.cs (1)
75UnknownId = unknownToken.Id;
Model\SentencePieceUnigramModel.cs (1)
131UnknownId = unknownToken;
21 references to UnknownId
Microsoft.ML.Tokenizers (21)
Model\SentencePieceBaseModel.cs (2)
411else if (id == model.UnknownId) 649else if (id == model.UnknownId)
Model\SentencePieceBpeModel.cs (8)
221id = UnknownId; 228if (id == UnknownId && ByteFallback) 499id = UnknownId; 506if (id == UnknownId && ByteFallback) 767id = UnknownId; 774if (id == UnknownId && ByteFallback) 1043id = UnknownId; 1050if (id == UnknownId && ByteFallback)
Model\SentencePieceTokenizer.cs (1)
106public int UnknownId => _model.UnknownId;
Model\SentencePieceUnigramModel.cs (10)
412string stringToken = node.Id == UnknownId ? Helpers.GetString(normalizationSpan.Slice(node.StartsAt, endsAt - node.StartsAt)) : _vocabReverse[node.Id].Piece; 417unknownEncountered = unknownEncountered || node.Id == UnknownId; 468if (tokens[insertionStartPosition].Id == UnknownId) 592targetNode.Id = UnknownId; 795Debug.Assert(ids[IdsIndex] == UnknownId); 853if (node.Id == UnknownId && needToTrackUnknown) 922if (ids[index] == UnknownId) 1185unknownEncountered = unknownEncountered || node.Id == UnknownId; 1218if (ids[i].Id == UnknownId) 1428if (node.Id == UnknownId)