1 write to UnknownTokenId
Microsoft.ML.Tokenizers (1)
Model\WordPieceTokenizer.cs (1)
68
UnknownTokenId
= id;
12 references to UnknownTokenId
Microsoft.ML.Tokenizers (8)
Model\BertTokenizer.cs (4)
442
mask.Add(id == ClassificationTokenId || id == SeparatorTokenId || id == PaddingTokenId || id == MaskingTokenId || id ==
UnknownTokenId
? 1 : 0);
449
mask.Add(id == ClassificationTokenId || id == SeparatorTokenId || id == PaddingTokenId || id == MaskingTokenId || id ==
UnknownTokenId
? 1 : 0);
529
destination[valuesWritten++] = id == ClassificationTokenId || id == SeparatorTokenId || id == PaddingTokenId || id == MaskingTokenId || id ==
UnknownTokenId
? 1 : 0;
541
destination[valuesWritten++] = id == ClassificationTokenId || id == SeparatorTokenId || id == PaddingTokenId || id == MaskingTokenId || id ==
UnknownTokenId
? 1 : 0;
Model\WordPieceTokenizer.cs (4)
319
tokens.Add(new EncodedToken(
UnknownTokenId
, UnknownToken, new Range(offset, offset + text.Length)));
373
tokens.Add(new EncodedToken(
UnknownTokenId
, UnknownToken, new Range(offset, offset + textLength)));
458
accumulatedIds?.Add(
UnknownTokenId
);
522
accumulatedIds?.Add(
UnknownTokenId
);
Microsoft.ML.Tokenizers.Tests (4)
BertTokenizerTests.cs (3)
48
Assert.Equal(1, tokenizer.
UnknownTokenId
);
119
Assert.Equal(1, tokenizer.
UnknownTokenId
);
191
Assert.Equal(1, tokenizer.
UnknownTokenId
);
WordPieceTests.cs (1)
39
Assert.Equal(0, tokenizer.
UnknownTokenId
);