2 writes to _unknownTokenId
Microsoft.ML.Tokenizers (2)
Model\BPETokenizer.cs (2)
56
_unknownTokenId
= null;
65
_unknownTokenId
= id;
3 references to _unknownTokenId
Microsoft.ML.Tokenizers (3)
Model\BPETokenizer.cs (3)
806
if (id ==
_unknownTokenId
)
916
bool skipUnknownToken = !
_unknownTokenId
.HasValue || !considerSpecialTokens;
926
if (skipUnknownToken && id ==
_unknownTokenId
)