2 writes to UnknownToken
Microsoft.ML.Tokenizers (2)
Model\SentencePieceBaseModel.cs (2)
31
UnknownToken
= modelProto.TrainerSpec.UnkPiece ?? "<unk>";
78
UnknownToken
= unkToken;
3 references to UnknownToken
Microsoft.ML.Tokenizers (3)
Model\SentencePieceBaseModel.cs (2)
396
token = model.
UnknownToken
;
634
specialToken = model.
UnknownToken
;
Model\SentencePieceTokenizer.cs (1)
87
public string UnknownToken => _model.
UnknownToken
;