1 write to UnknownToken
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
359
UnknownToken
= unknownToken;
5 references to UnknownToken
Microsoft.ML.Tokenizers (5)
Model\BPETokenizer.cs (5)
810
Debug.Assert(
UnknownToken
is not null);
811
sb.Append(
UnknownToken
);
1263
else if (
UnknownToken
is not null)
1276
if (!_vocab.TryGetValue(
UnknownToken
, out int value))
1285
if (!_vocab.TryGetValue(
UnknownToken
, out int value))