1 write to SeparatorTokenId
Microsoft.ML.Tokenizers (1)
Model\BertTokenizer.cs (1)
39SeparatorTokenId = vocab[new StringSpanOrdinalKey(options.SeparatorToken)];
36 references to SeparatorTokenId
Microsoft.ML.Tokenizers (10)
Model\BertTokenizer.cs (10)
251list.Add(SeparatorTokenId); 273list.Add(SeparatorTokenId); 317ids.Add(SeparatorTokenId); 322ids.Add(SeparatorTokenId); 369destination[valuesWritten++] = SeparatorTokenId; 388destination[valuesWritten++] = SeparatorTokenId; 442mask.Add(id == ClassificationTokenId || id == SeparatorTokenId || id == PaddingTokenId || id == MaskingTokenId || id == UnknownTokenId ? 1 : 0); 449mask.Add(id == ClassificationTokenId || id == SeparatorTokenId || id == PaddingTokenId || id == MaskingTokenId || id == UnknownTokenId ? 1 : 0); 529destination[valuesWritten++] = id == ClassificationTokenId || id == SeparatorTokenId || id == PaddingTokenId || id == MaskingTokenId || id == UnknownTokenId ? 1 : 0; 541destination[valuesWritten++] = id == ClassificationTokenId || id == SeparatorTokenId || id == PaddingTokenId || id == MaskingTokenId || id == UnknownTokenId ? 1 : 0;
Microsoft.ML.Tokenizers.Tests (26)
BertTokenizerTests.cs (26)
54Assert.Equal([tokenizer.ClassificationTokenId, 8, 6, 10, 11, 12, 7, tokenizer.SeparatorTokenId], ids); 75Assert.Equal([tokenizer.ClassificationTokenId, tokenizer.ClassificationTokenId, 8, 6, 10, 11, 12, 7, tokenizer.SeparatorTokenId, tokenizer.SeparatorTokenId], ids); 122Assert.Equal([tokenizer.ClassificationTokenId, 1, 6, 1, 11, 12, 7, tokenizer.SeparatorTokenId], ids); 280Assert.Equal([bertTokenizer.ClassificationTokenId, 8, 6, 10, 11, 12, 7, bertTokenizer.SeparatorTokenId], ids1); 283Assert.Equal([bertTokenizer.ClassificationTokenId, 13, 14, 15, 5, bertTokenizer.SeparatorTokenId], ids2); 286[bertTokenizer.ClassificationTokenId, bertTokenizer.ClassificationTokenId, 8, 6, 10, 11, 12, 7, bertTokenizer.SeparatorTokenId, bertTokenizer.SeparatorTokenId], 298Assert.Equal(new int[] { bertTokenizer.ClassificationTokenId, bertTokenizer.ClassificationTokenId, 8, 6, 10, 11, 12, 7, bertTokenizer.SeparatorTokenId, bertTokenizer.SeparatorTokenId }, ids1Span.ToArray()); 301[bertTokenizer.ClassificationTokenId, bertTokenizer.ClassificationTokenId, 8, 6, 10, 11, 12, 7, bertTokenizer.SeparatorTokenId, bertTokenizer.SeparatorTokenId, bertTokenizer.ClassificationTokenId, 13, 14, 15, 5, bertTokenizer.SeparatorTokenId, bertTokenizer.SeparatorTokenId], 314new int[] { bertTokenizer.ClassificationTokenId, bertTokenizer.ClassificationTokenId, 8, 6, 10, 11, 12, 7, bertTokenizer.SeparatorTokenId, bertTokenizer.SeparatorTokenId, bertTokenizer.ClassificationTokenId, 13, 14, 15, 5, bertTokenizer.SeparatorTokenId, bertTokenizer.SeparatorTokenId }, 324[bertTokenizer.ClassificationTokenId, 8, 6, 10, 11, 12, 7, bertTokenizer.SeparatorTokenId], 337new int[] { bertTokenizer.ClassificationTokenId, 8, 6, 10, 11, 12, 7, bertTokenizer.SeparatorTokenId }, 341[bertTokenizer.ClassificationTokenId, 8, 6, 10, 11, 12, 7, bertTokenizer.SeparatorTokenId, 13, 14, 15, 5, bertTokenizer.SeparatorTokenId], 354new int[] { bertTokenizer.ClassificationTokenId, 8, 6, 10, 11, 12, 7, bertTokenizer.SeparatorTokenId, 13, 14, 15, 5, bertTokenizer.SeparatorTokenId }, 380Assert.Equal([bertTokenizer.ClassificationTokenId, 8, 6, 10, 11, 12, 7, bertTokenizer.SeparatorTokenId], ids1); 383Assert.Equal([bertTokenizer.ClassificationTokenId, 13, 14, 15, 5, bertTokenizer.SeparatorTokenId], ids2);