2 writes to _vocab
Microsoft.ML.Tokenizers (2)
Model\SentencePieceUnigramModel.cs (2)
32_vocab = new SortedDictionary<string, int>(OrdinalUtf8StringComparer.Instance); 173_vocab = new SortedDictionary<string, int>(OrdinalUtf8StringComparer.Instance);
15 references to _vocab
Microsoft.ML.Tokenizers (15)
Model\SentencePieceUnigramModel.cs (15)
55_vocab.Add(piece, i); 70ByteCodeToIdOffset = _vocab.TryGetValue("<0x00>", out int id) ? id : MaxByteId; 74_trie = new DoubleArrayTrie(_vocab); 84_vocab[modelProto.TrainerSpec.UnkPiece] = modelProto.TrainerSpec.UnkId; 85_vocab[modelProto.TrainerSpec.BosPiece] = modelProto.TrainerSpec.BosId; 86_vocab[modelProto.TrainerSpec.EosPiece] = modelProto.TrainerSpec.EosId; 94_vocab[modelProto.TrainerSpec.PadPiece] = modelProto.TrainerSpec.PadId; 206_vocab.Add(piece, i); 217ByteCodeToIdOffset = _vocab.TryGetValue("<0x00>", out int id) ? id : MaxByteId; 218if (!_vocab.ContainsKey("<0x00>") || !_vocab.TryGetValue("<0xFF>", out int maxByteId) || maxByteId - ByteCodeToIdOffset != 0xFF) 230_trie = new DoubleArrayTrie(_vocab); 238_vocab[unkToken] = unkId; 252_vocab[piece] = controlId; 371public override IReadOnlyDictionary<string, int> Vocabulary => new ReadOnlyDictionary<string, int>(_vocab);