2 writes to _vocab
Microsoft.ML.Tokenizers (2)
Model\SentencePieceUnigramModel.cs (2)
29_vocab = new SortedDictionary<string, int>(OrdinalUtf8StringComparer.Instance); 90_vocab = new SortedDictionary<string, int>(OrdinalUtf8StringComparer.Instance);
14 references to _vocab
Microsoft.ML.Tokenizers (14)
Model\SentencePieceUnigramModel.cs (14)
44_vocab.Add(piece, i); 59ByteCodeToIdOffset = _vocab.TryGetValue("<0x00>", out int id) ? id : MaxByteId; 63_trie = new DoubleArrayTrie(_vocab); 73_vocab[modelProto.TrainerSpec.UnkPiece] = modelProto.TrainerSpec.UnkId; 74_vocab[modelProto.TrainerSpec.BosPiece] = modelProto.TrainerSpec.BosId; 75_vocab[modelProto.TrainerSpec.EosPiece] = modelProto.TrainerSpec.EosId; 83_vocab[modelProto.TrainerSpec.PadPiece] = modelProto.TrainerSpec.PadId; 102_vocab.Add(item.Token, id++); 112if (!_vocab.TryGetValue("<0x00>", out id)) 122_trie = new DoubleArrayTrie(_vocab); 127if (!_vocab.TryGetValue(options.UnknownToken, out int unknownToken)) 133if (!_vocab.TryGetValue(options.BeginningOfSentenceToken, out int beginOfSentenceToken)) 139if (!_vocab.TryGetValue(options.EndOfSentenceToken, out int endOfSentenceToken)) 146public override IReadOnlyDictionary<string, int> Vocabulary => new ReadOnlyDictionary<string, int>(_vocab);