1 write to UnknownToken
Microsoft.ML.Tokenizers (1)
Model\WordPieceTokenizer.cs (1)
72
UnknownToken
= unknownToken;
5 references to UnknownToken
Microsoft.ML.Tokenizers (2)
Model\WordPieceTokenizer.cs (2)
374
tokens.Add(new EncodedToken(UnknownTokenId,
UnknownToken
, new Range(offset, offset + text.Length)));
428
tokens.Add(new EncodedToken(UnknownTokenId,
UnknownToken
, new Range(offset, offset + textLength)));
Microsoft.ML.Tokenizers.Tests (3)
BertTokenizerTests.cs (2)
33
Assert.Equal("[UNK]", tokenizer.
UnknownToken
);
100
Assert.Equal("[UNK]", tokenizer.
UnknownToken
);
WordPieceTests.cs (1)
38
Assert.Equal("[UNK]", tokenizer.
UnknownToken
);