10 references to UnknownToken
Microsoft.ML.Tokenizers.Tests (10)
LlamaTests.cs (1)
364
Assert.Equal("<unk>", bpe.
UnknownToken
);
SentencePieceTests.cs (8)
79
Assert.Equal("<unk>", tokenizer.
UnknownToken
);
202
Assert.Equal("<unk>", tokenizer.
UnknownToken
);
235
Assert.Equal("[UNK]", tokenizer.
UnknownToken
);
309
Assert.Equal("[UNK]", tokenizer.
UnknownToken
);
329
Assert.Equal("second", tokenizer.
UnknownToken
);
456
Assert.Equal("", tokenizer.
UnknownToken
);
709
Assert.Equal("second", tokenizer.
UnknownToken
);
734
Assert.Equal("[UNK]", tokenizer.
UnknownToken
);
UnigramTests.cs (1)
558
Assert.Equal("<unk>", _unigramTokenizer.
UnknownToken
);