2 references to UnknownToken
Microsoft.ML.Tokenizers.Tests (2)
LlamaTests.cs (1)
364
Assert.Equal("<unk>", bpe.
UnknownToken
);
UnigramTests.cs (1)
558
Assert.Equal("<unk>", _unigramTokenizer.
UnknownToken
);