1 write to UnknownToken
Microsoft.ML.Tokenizers.Tests (1)
UnigramTests.cs (1)
47
options.
UnknownToken
= "<unk>";
7 references to UnknownToken
Microsoft.ML.Tokenizers (7)
Model\SentencePieceBaseModel.cs (3)
84
if (options.
UnknownToken
is null)
86
throw new ArgumentNullException(nameof(options.
UnknownToken
));
93
UnknownToken = options.
UnknownToken
;
Model\SentencePieceBpeModel.cs (2)
71
if (!_vocab.TryGetValue(options.
UnknownToken
, out (int Id, float Score, byte Type) unknownToken))
73
throw new ArgumentException($"The vocabulary must include the unknown token '{options.
UnknownToken
}'.");
Model\SentencePieceUnigramModel.cs (2)
127
if (!_vocab.TryGetValue(options.
UnknownToken
, out int unknownToken))
129
throw new ArgumentException($"The vocabulary must include the unknown token '{options.
UnknownToken
}'.");