2 writes to _unknownTokenId
Microsoft.ML.Tokenizers (2)
Model\BPETokenizer.cs (2)
48_unknownTokenId = null; 57_unknownTokenId = id;
4 references to _unknownTokenId
Microsoft.ML.Tokenizers (4)
Model\BPETokenizer.cs (4)
588bool decodeUnknownToken = _unknownTokenId.HasValue && considerSpecialTokens; 604if (id == _unknownTokenId) 663bool skipUnknownToken = !_unknownTokenId.HasValue || !considerSpecialTokens; 673if (skipUnknownToken && id == _unknownTokenId)