1 write to MaskingTokenId
Microsoft.ML.Tokenizers (1)
Model\BertTokenizer.cs (1)
48
MaskingTokenId
= vocab[new StringSpanOrdinalKey(options.MaskingToken)];
4 references to MaskingTokenId
Microsoft.ML.Tokenizers (4)
Model\BertTokenizer.cs (4)
442
mask.Add(id == ClassificationTokenId || id == SeparatorTokenId || id == PaddingTokenId || id ==
MaskingTokenId
|| id == UnknownTokenId ? 1 : 0);
449
mask.Add(id == ClassificationTokenId || id == SeparatorTokenId || id == PaddingTokenId || id ==
MaskingTokenId
|| id == UnknownTokenId ? 1 : 0);
529
destination[valuesWritten++] = id == ClassificationTokenId || id == SeparatorTokenId || id == PaddingTokenId || id ==
MaskingTokenId
|| id == UnknownTokenId ? 1 : 0;
541
destination[valuesWritten++] = id == ClassificationTokenId || id == SeparatorTokenId || id == PaddingTokenId || id ==
MaskingTokenId
|| id == UnknownTokenId ? 1 : 0;