1 write to UnknownToken
Microsoft.ML.Tokenizers (1)
Model\BPETokenizer.cs (1)
263UnknownToken = unknownToken;
3 references to UnknownToken
Microsoft.ML.Tokenizers (3)
Model\BPETokenizer.cs (3)
937else if (UnknownToken is not null) 950if (!_vocab.TryGetValue(UnknownToken, out int value)) 959if (!_vocab.TryGetValue(UnknownToken, out int value))