4 references to GetSpecialTokensMask
Microsoft.ML.Tokenizers.Tests (4)
BertTokenizerTests.cs (4)
476
bertTokenizer.
GetSpecialTokensMask
(ids1, additionalTokenIds: null, alreadyHasSpecialTokens: true));
491
bertTokenizer.
GetSpecialTokensMask
(ids1, additionalTokenIds: ids2, alreadyHasSpecialTokens: true));
511
bertTokenizer.
GetSpecialTokensMask
(ids1, additionalTokenIds: null, alreadyHasSpecialTokens: false));
526
bertTokenizer.
GetSpecialTokensMask
(ids1, additionalTokenIds: ids2, alreadyHasSpecialTokens: false));