1 write to UnknownToken
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
218UnknownToken = unknownToken;
3 references to UnknownToken
Microsoft.ML.Tokenizers (3)
Model\BPETokenizer.cs (3)
893else if (UnknownToken is not null) 906if (!_vocab.TryGetValue(UnknownToken, out int value)) 915if (!_vocab.TryGetValue(UnknownToken, out int value))