3 references to PaddingToken
Microsoft.ML.Tokenizers (3)
Model\BertTokenizer.cs (3)
41
PaddingToken = options.
PaddingToken
;
42
PaddingTokenId = vocab[new StringSpanOrdinalKey(options.
PaddingToken
)];
795
AddSpecialToken(vocab, specialTokens, options.
PaddingToken
, lowerCase);