1 write to UnknownToken
Microsoft.ML.Tokenizers (1)
Model\SentencePieceTokenizer.cs (1)
67
UnknownToken
= modelProto.TrainerSpec.UnkPiece ?? "<unk>";
3 references to UnknownToken
Microsoft.ML.Tokenizers (2)
Model\SentencePieceTokenizer.cs (2)
1755
sb.Append(tokenizer.
UnknownToken
);
1996
specialToken = tokenizer.
UnknownToken
;
Microsoft.ML.Tokenizers.Tests (1)
LlamaTests.cs (1)
364
Assert.Equal("<unk>", bpe.
UnknownToken
);