1 write to UnknownTokenId
Microsoft.ML.Tokenizers (1)
Model\WordPieceTokenizer.cs (1)
73UnknownTokenId = id;
11 references to UnknownTokenId
Microsoft.ML.Tokenizers (8)
Model\BertTokenizer.cs (4)
438mask.Add(id == ClsTokenId || id == SepTokenId || id == PadTokenId || id == MaskTokenId || id == UnknownTokenId ? 1 : 0); 445mask.Add(id == ClsTokenId || id == SepTokenId || id == PadTokenId || id == MaskTokenId || id == UnknownTokenId ? 1 : 0); 525buffer[written++] = id == ClsTokenId || id == SepTokenId || id == PadTokenId || id == MaskTokenId || id == UnknownTokenId ? 1 : 0; 537buffer[written++] = id == ClsTokenId || id == SepTokenId || id == PadTokenId || id == MaskTokenId || id == UnknownTokenId ? 1 : 0;
Model\WordPieceTokenizer.cs (4)
374tokens.Add(new EncodedToken(UnknownTokenId, UnknownToken, new Range(offset, offset + text.Length))); 428tokens.Add(new EncodedToken(UnknownTokenId, UnknownToken, new Range(offset, offset + textLength))); 513accumulatedIds?.Add(UnknownTokenId); 577accumulatedIds?.Add(UnknownTokenId);
Microsoft.ML.Tokenizers.Tests (3)
BertTokenizerTests.cs (2)
34Assert.Equal(1, tokenizer.UnknownTokenId); 101Assert.Equal(1, tokenizer.UnknownTokenId);
WordPieceTests.cs (1)
39Assert.Equal(0, tokenizer.UnknownTokenId);