1 write to SpecialTokensReverse
Microsoft.ML.Tokenizers (1)
Model\WordPieceTokenizer.cs (1)
45SpecialTokensReverse = options.SpecialTokens is not null ? options.SpecialTokens.GroupBy(kvp => kvp.Value).ToDictionary(g => g.Key, g => g.First().Key) : null;
4 references to SpecialTokensReverse
Microsoft.ML.Tokenizers (4)
Model\WordPieceTokenizer.cs (4)
683bool ignoreSpecialTokens = skipSpecialTokens && SpecialTokensReverse is not null; 687if (ignoreSpecialTokens && SpecialTokensReverse!.TryGetValue(id, out _)) 742bool ignoreSpecialTokens = SpecialTokensReverse is not null && skipSpecialTokens; 746if (ignoreSpecialTokens && SpecialTokensReverse!.TryGetValue(id, out _))