2 writes to _unknownTokenId
Microsoft.ML.Tokenizers (2)
Model\BPETokenizer.cs (2)
56_unknownTokenId = null; 65_unknownTokenId = id;
4 references to _unknownTokenId
Microsoft.ML.Tokenizers (4)
Model\BPETokenizer.cs (4)
790bool decodeUnknownToken = _unknownTokenId.HasValue && considerSpecialTokens; 806if (id == _unknownTokenId) 912bool skipUnknownToken = !_unknownTokenId.HasValue || !considerSpecialTokens; 922if (skipUnknownToken && id == _unknownTokenId)