3 references to PaddingToken
Microsoft.ML.Tokenizers (3)
Model\BertTokenizer.cs (3)
41
PaddingToken = options.
PaddingToken
;
42
PaddingTokenId = vocab[new StringSpanOrdinalKey(options.
PaddingToken
)];
798
AddSpecialToken(vocab, tempSpecialTokens, options.
PaddingToken
, lowerCase, notNormalizedSpecialTokens);