1 write to _vocabReverse
Microsoft.ML.Tokenizers (1)
Model\WordPieceTokenizer.cs (1)
48_vocabReverse = vocabReverse!;
4 references to _vocabReverse
Microsoft.ML.Tokenizers (4)
Model\WordPieceTokenizer.cs (4)
406Debug.Assert(_vocabReverse.ContainsKey(id)); 407curToken = new EncodedToken(id, _vocabReverse[id], new Range(offset + start, offset + end)); 747if (_vocabReverse.TryGetValue(id, out string? token)) 806if (_vocabReverse.TryGetValue(id, out string? token))