1 write to _vocab
Microsoft.ML.Tokenizers (1)
Model\CodeGenTokenizer.cs (1)
124_vocab = GetVocabulary(vocabularyStream);
8 references to _vocab
Microsoft.ML.Tokenizers (8)
Model\CodeGenTokenizer.cs (8)
125_vocabReverse = _vocab.ToDictionary(kvp => kvp.Value.Id, kvp => kvp.Value.Token); 148if (!_vocab.TryGetValue(UnknownToken!, out (int unknownId, string token) value)) 158if (!_vocab.TryGetValue(BeginningOfSentenceToken!, out (int beggingOfSentenceId, string token) value)) 168if (!_vocab.TryGetValue(EndOfSentenceToken!, out (int endOfSentenceId, string token) value)) 266var vocab = new ReadOnlyDictionary<string, int>(_vocab.ToDictionary(kvp => kvp.Value.Token, kvp => kvp.Value.Id)); 1551if (_vocab.TryGetValue(token, out (int Id, string Token) value)) 1628return new List<EncodedToken> { new EncodedToken(_vocab[new StringSpanOrdinalKey(tokenValue)].Id, tokenValue, new Range(mapping[0], mapping[0] + 1)) }; 1681if (_vocab.TryGetValue(text.Slice(symbols[index].pieceSpan.Index, symbols[index].pieceSpan.Length), out (int Id, string Token) value))