1 write to _vocabIdToHighestOccurrence
Microsoft.ML.Tokenizers (1)
Model\EnglishRobertaTokenizer.cs (1)
153_vocabIdToHighestOccurrence = GetHighestOccurrenceMapping(highestOccurrenceMappingStream);
6 references to _vocabIdToHighestOccurrence
Microsoft.ML.Tokenizers (6)
Model\EnglishRobertaTokenizer.cs (6)
829list.Add(id <= 0 ? -id : _vocabIdToHighestOccurrence.IdToOccurrenceRank(id)); 851list.Add(id <= 0 ? 0 : _vocabIdToHighestOccurrence.IdToOccurrenceValue(id)); 873list.Add(_vocabIdToHighestOccurrence.ConvertOccurrenceRankToId(rank)); 882public int PadIndex => _vocabIdToHighestOccurrence.PadIndex; 887public int SymbolsCount => _vocabIdToHighestOccurrence.Count; 894public int AddMaskSymbol(string mask = "<mask>") => _vocabIdToHighestOccurrence.AddMaskSymbol(mask);