1 write to UnknownToken
Microsoft.ML.Tokenizers (1)
Model\SentencePieceBaseModel.cs (1)
31
UnknownToken
= modelProto.TrainerSpec.UnkPiece ?? "<unk>";
3 references to UnknownToken
Microsoft.ML.Tokenizers (3)
Model\SentencePieceBaseModel.cs (2)
352
token = model.
UnknownToken
;
590
specialToken = model.
UnknownToken
;
Model\SentencePieceTokenizer.cs (1)
81
public string UnknownToken => _model.
UnknownToken
;