6 references to UnknownToken
Microsoft.ML.Tokenizers (6)
Model\BertTokenizer.cs (1)
793
AddSpecialToken(vocab, specialTokens, options.
UnknownToken
, lowerCase);
Model\WordPieceTokenizer.cs (5)
47
if (options.
UnknownToken
is null)
49
throw new ArgumentNullException(nameof(options.
UnknownToken
));
62
if (!vocab!.TryGetValue(options.
UnknownToken
, out int id))
64
throw new ArgumentException($"The unknown token '{options.
UnknownToken
}' is not in the vocabulary.");
67
UnknownToken = options.
UnknownToken
;