1 write to UnknownToken
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
359UnknownToken = unknownToken;
5 references to UnknownToken
Microsoft.ML.Tokenizers (5)
Model\BPETokenizer.cs (5)
810Debug.Assert(UnknownToken is not null); 811sb.Append(UnknownToken); 1263else if (UnknownToken is not null) 1276if (!_vocab.TryGetValue(UnknownToken, out int value)) 1285if (!_vocab.TryGetValue(UnknownToken, out int value))