2 writes to _unknownTokenId
Microsoft.ML.Tokenizers (2)
Model\BPETokenizer.cs (2)
56_unknownTokenId = null; 65_unknownTokenId = id;
3 references to _unknownTokenId
Microsoft.ML.Tokenizers (3)
Model\BPETokenizer.cs (3)
806if (id == _unknownTokenId) 916bool skipUnknownToken = !_unknownTokenId.HasValue || !considerSpecialTokens; 926if (skipUnknownToken && id == _unknownTokenId)