2 writes to _unknownTokenId
Microsoft.ML.Tokenizers (2)
Model\BPETokenizer.cs (2)
55_unknownTokenId = null; 64_unknownTokenId = id;
4 references to _unknownTokenId
Microsoft.ML.Tokenizers (4)
Model\BPETokenizer.cs (4)
633bool decodeUnknownToken = _unknownTokenId.HasValue && considerSpecialTokens; 649if (id == _unknownTokenId) 708bool skipUnknownToken = !_unknownTokenId.HasValue || !considerSpecialTokens; 718if (skipUnknownToken && id == _unknownTokenId)