1 write to MaskingTokenId
Microsoft.ML.Tokenizers (1)
Model\BertTokenizer.cs (1)
48MaskingTokenId = vocab[new StringSpanOrdinalKey(options.MaskingToken)];
4 references to MaskingTokenId
Microsoft.ML.Tokenizers (4)
Model\BertTokenizer.cs (4)
442mask.Add(id == ClassificationTokenId || id == SeparatorTokenId || id == PaddingTokenId || id == MaskingTokenId || id == UnknownTokenId ? 1 : 0); 449mask.Add(id == ClassificationTokenId || id == SeparatorTokenId || id == PaddingTokenId || id == MaskingTokenId || id == UnknownTokenId ? 1 : 0); 529destination[valuesWritten++] = id == ClassificationTokenId || id == SeparatorTokenId || id == PaddingTokenId || id == MaskingTokenId || id == UnknownTokenId ? 1 : 0; 541destination[valuesWritten++] = id == ClassificationTokenId || id == SeparatorTokenId || id == PaddingTokenId || id == MaskingTokenId || id == UnknownTokenId ? 1 : 0;