1 write to SpecialTokensReverse
Microsoft.ML.Tokenizers (1)
Model\WordPieceTokenizer.cs (1)
50SpecialTokensReverse = specialTokens is not null ? specialTokens.ToDictionary(kvp => kvp.Value, kvp => kvp.Key) : null;
4 references to SpecialTokensReverse
Microsoft.ML.Tokenizers (4)
Model\WordPieceTokenizer.cs (4)
738bool ignoreSpecialTokens = skipSpecialTokens && SpecialTokensReverse is not null; 742if (ignoreSpecialTokens && SpecialTokensReverse!.TryGetValue(id, out _)) 797bool ignoreSpecialTokens = SpecialTokensReverse is not null && skipSpecialTokens; 801if (ignoreSpecialTokens && SpecialTokensReverse!.TryGetValue(id, out _))