1 write to VocabReverse
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
249VocabReverse = new();
3 references to VocabReverse
Microsoft.ML.Tokenizers (3)
Model\BPETokenizer.cs (3)
253VocabReverse.Add(kvp.Value, kvp.Key.Data!); 598if (VocabReverse.TryGetValue(id, out string? value)) 979internal void WordToTokens(ref Word word, List<EncodedToken> tokens, int offset) => word.ToTokens(VocabReverse, tokens, offset);