1 write to _vocabReverse
Microsoft.ML.Tokenizers (1)
Model\WordPieceTokenizer.cs (1)
40_vocabReverse = vocabReverse!;
4 references to _vocabReverse
Microsoft.ML.Tokenizers (4)
Model\WordPieceTokenizer.cs (4)
351Debug.Assert(_vocabReverse.ContainsKey(id)); 352curToken = new EncodedToken(id, _vocabReverse[id], new Range(offset + start, offset + end)); 692if (_vocabReverse.TryGetValue(id, out string? token)) 751if (_vocabReverse.TryGetValue(id, out string? token))