1 write to VocabReverse
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
340VocabReverse = new();
3 references to VocabReverse
Microsoft.ML.Tokenizers (3)
Model\BPETokenizer.cs (3)
344VocabReverse.Add(kvp.Value, kvp.Key.Data!); 750if (VocabReverse.TryGetValue(id, out string? value)) 1301internal void WordToTokens(ref Word word, List<EncodedToken> tokens, int offset, ReadOnlySpan<int> mapping) => word.ToTokens(VocabReverse, tokens, offset, mapping);