1 write to UnknownToken
Microsoft.ML.Tokenizers (1)
Model\WordPieceTokenizer.cs (1)
67UnknownToken = options.UnknownToken;
7 references to UnknownToken
Microsoft.ML.Tokenizers (2)
Model\WordPieceTokenizer.cs (2)
319tokens.Add(new EncodedToken(UnknownTokenId, UnknownToken, new Range(offset, offset + text.Length))); 373tokens.Add(new EncodedToken(UnknownTokenId, UnknownToken, new Range(offset, offset + textLength)));
Microsoft.ML.Tokenizers.Tests (5)
BertTokenizerTests.cs (4)
47Assert.Equal("[UNK]", tokenizer.UnknownToken); 118Assert.Equal("[UNK]", tokenizer.UnknownToken); 124Assert.True(tokenizer.SpecialTokens!.ContainsKey(tokenizer.UnknownToken)); 190Assert.Equal("[UNK]", tokenizer.UnknownToken);
WordPieceTests.cs (1)
38Assert.Equal("[UNK]", tokenizer.UnknownToken);