3 references to MaskingToken
Microsoft.ML.Tokenizers (3)
Model\BertTokenizer.cs (3)
47
MaskingToken = options.
MaskingToken
;
48
MaskingTokenId = vocab[new StringSpanOrdinalKey(options.
MaskingToken
)];
800
AddSpecialToken(vocab, tempSpecialTokens, options.
MaskingToken
, lowerCase, notNormalizedSpecialTokens);