1 write to ClassificationTokenId
Microsoft.ML.Tokenizers (1)
Model\BertTokenizer.cs (1)
45
ClassificationTokenId
= vocab[new StringSpanOrdinalKey(options.ClassificationToken)];
34 references to ClassificationTokenId
Microsoft.ML.Tokenizers (9)
Model\BertTokenizer.cs (9)
250
list.Insert(0,
ClassificationTokenId
);
272
list.Insert(0,
ClassificationTokenId
);
308
ids = new(capacity) {
ClassificationTokenId
};
313
ids = new List<int>(10) {
ClassificationTokenId
};
352
destination[valuesWritten++] =
ClassificationTokenId
;
442
mask.Add(id ==
ClassificationTokenId
|| id == SeparatorTokenId || id == PaddingTokenId || id == MaskingTokenId || id == UnknownTokenId ? 1 : 0);
449
mask.Add(id ==
ClassificationTokenId
|| id == SeparatorTokenId || id == PaddingTokenId || id == MaskingTokenId || id == UnknownTokenId ? 1 : 0);
529
destination[valuesWritten++] = id ==
ClassificationTokenId
|| id == SeparatorTokenId || id == PaddingTokenId || id == MaskingTokenId || id == UnknownTokenId ? 1 : 0;
541
destination[valuesWritten++] = id ==
ClassificationTokenId
|| id == SeparatorTokenId || id == PaddingTokenId || id == MaskingTokenId || id == UnknownTokenId ? 1 : 0;
Microsoft.ML.Tokenizers.Tests (25)
BertTokenizerTests.cs (25)
71
Assert.Equal([tokenizer.
ClassificationTokenId
, 8, 6, 10, 11, 12, 13, 7, tokenizer.SeparatorTokenId], ids);
93
Assert.Equal([tokenizer.
ClassificationTokenId
, tokenizer.
ClassificationTokenId
, 8, 6, 10, 11, 12, 13, 7, tokenizer.SeparatorTokenId, tokenizer.SeparatorTokenId], ids);
143
Assert.Equal([tokenizer.
ClassificationTokenId
, 8, 6, 10, 11, 12, 7, tokenizer.SeparatorTokenId], ids);
164
Assert.Equal([tokenizer.
ClassificationTokenId
, tokenizer.
ClassificationTokenId
, 8, 6, 10, 11, 12, 7, tokenizer.SeparatorTokenId, tokenizer.SeparatorTokenId], ids);
211
Assert.Equal([tokenizer.
ClassificationTokenId
, 1, 6, 1, 11, 12, 7, tokenizer.SeparatorTokenId], ids);
369
Assert.Equal([bertTokenizer.
ClassificationTokenId
, 8, 6, 10, 11, 12, 7, bertTokenizer.SeparatorTokenId], ids1);
372
Assert.Equal([bertTokenizer.
ClassificationTokenId
, 13, 14, 15, 5, bertTokenizer.SeparatorTokenId], ids2);
375
[bertTokenizer.
ClassificationTokenId
, bertTokenizer.
ClassificationTokenId
, 8, 6, 10, 11, 12, 7, bertTokenizer.SeparatorTokenId, bertTokenizer.SeparatorTokenId],
387
Assert.Equal(new int[] { bertTokenizer.
ClassificationTokenId
, bertTokenizer.
ClassificationTokenId
, 8, 6, 10, 11, 12, 7, bertTokenizer.SeparatorTokenId, bertTokenizer.SeparatorTokenId }, ids1Span.ToArray());
390
[bertTokenizer.
ClassificationTokenId
, bertTokenizer.
ClassificationTokenId
, 8, 6, 10, 11, 12, 7, bertTokenizer.SeparatorTokenId, bertTokenizer.SeparatorTokenId, bertTokenizer.
ClassificationTokenId
, 13, 14, 15, 5, bertTokenizer.SeparatorTokenId, bertTokenizer.SeparatorTokenId],
403
new int[] { bertTokenizer.
ClassificationTokenId
, bertTokenizer.
ClassificationTokenId
, 8, 6, 10, 11, 12, 7, bertTokenizer.SeparatorTokenId, bertTokenizer.SeparatorTokenId, bertTokenizer.
ClassificationTokenId
, 13, 14, 15, 5, bertTokenizer.SeparatorTokenId, bertTokenizer.SeparatorTokenId },
413
[bertTokenizer.
ClassificationTokenId
, 8, 6, 10, 11, 12, 7, bertTokenizer.SeparatorTokenId],
426
new int[] { bertTokenizer.
ClassificationTokenId
, 8, 6, 10, 11, 12, 7, bertTokenizer.SeparatorTokenId },
430
[bertTokenizer.
ClassificationTokenId
, 8, 6, 10, 11, 12, 7, bertTokenizer.SeparatorTokenId, 13, 14, 15, 5, bertTokenizer.SeparatorTokenId],
443
new int[] { bertTokenizer.
ClassificationTokenId
, 8, 6, 10, 11, 12, 7, bertTokenizer.SeparatorTokenId, 13, 14, 15, 5, bertTokenizer.SeparatorTokenId },
469
Assert.Equal([bertTokenizer.
ClassificationTokenId
, 8, 6, 10, 11, 12, 7, bertTokenizer.SeparatorTokenId], ids1);
472
Assert.Equal([bertTokenizer.
ClassificationTokenId
, 13, 14, 15, 5, bertTokenizer.SeparatorTokenId], ids2);