3 references to UnknownToken
Microsoft.ML.Tokenizers.Tests (3)
LlamaTests.cs (1)
364
Assert.Equal("<unk>", bpe.
UnknownToken
);
UnigramTests.cs (2)
803
Assert.Equal("<unk>", _unigramTokenizer.
UnknownToken
);
814
Assert.Equal("<unk>", _unigramTokenizerFromJson.
UnknownToken
);