1 write to UnknownToken
Microsoft.ML.Tokenizers (1)
Model\WordPieceTokenizer.cs (1)
67
UnknownToken
= options.UnknownToken;
7 references to UnknownToken
Microsoft.ML.Tokenizers (2)
Model\WordPieceTokenizer.cs (2)
319
tokens.Add(new EncodedToken(UnknownTokenId,
UnknownToken
, new Range(offset, offset + text.Length)));
373
tokens.Add(new EncodedToken(UnknownTokenId,
UnknownToken
, new Range(offset, offset + textLength)));
Microsoft.ML.Tokenizers.Tests (5)
BertTokenizerTests.cs (4)
47
Assert.Equal("[UNK]", tokenizer.
UnknownToken
);
118
Assert.Equal("[UNK]", tokenizer.
UnknownToken
);
124
Assert.True(tokenizer.SpecialTokens!.ContainsKey(tokenizer.
UnknownToken
));
190
Assert.Equal("[UNK]", tokenizer.
UnknownToken
);
WordPieceTests.cs (1)
38
Assert.Equal("[UNK]", tokenizer.
UnknownToken
);