1 write to UnknownToken
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
263
UnknownToken
= unknownToken;
3 references to UnknownToken
Microsoft.ML.Tokenizers (3)
Model\BPETokenizer.cs (3)
937
else if (
UnknownToken
is not null)
950
if (!_vocab.TryGetValue(
UnknownToken
, out int value))
959
if (!_vocab.TryGetValue(
UnknownToken
, out int value))