3 references to SeparatorToken
Microsoft.ML.Tokenizers (3)
Model\BertTokenizer.cs (3)
38
SeparatorToken = options.
SeparatorToken
;
39
SeparatorTokenId = vocab[new StringSpanOrdinalKey(options.
SeparatorToken
)];
797
AddSpecialToken(vocab, tempSpecialTokens, options.
SeparatorToken
, lowerCase, notNormalizedSpecialTokens);