1 write to _vocab
Microsoft.ML.Tokenizers (1)
Model\CodeGenTokenizer.cs (1)
127_vocab = GetVocabulary(vocabularyStream);
8 references to _vocab
Microsoft.ML.Tokenizers (8)
Model\CodeGenTokenizer.cs (8)
128_vocabReverse = _vocab.ToDictionary(kvp => kvp.Value.Id, kvp => kvp.Value.Token); 151if (!_vocab.TryGetValue(UnknownToken!, out (int unknownId, string token) value)) 161if (!_vocab.TryGetValue(BeginningOfSentenceToken!, out (int beggingOfSentenceId, string token) value)) 171if (!_vocab.TryGetValue(EndOfSentenceToken!, out (int endOfSentenceId, string token) value)) 269var vocab = new ReadOnlyDictionary<string, int>(_vocab.ToDictionary(kvp => kvp.Value.Token, kvp => kvp.Value.Id)); 1554if (_vocab.TryGetValue(token, out (int Id, string Token) value)) 1610return new List<EncodedToken> { new EncodedToken(_vocab[new StringSpanOrdinalKey(tokenValue)].Id, tokenValue, new Range(mapping[0], mapping[0] + 1)) }; 1663if (_vocab.TryGetValue(text.Slice(symbols[index].pieceSpan.Index, symbols[index].pieceSpan.Length), out (int Id, string Token) value))