1 write to UnknownId
Microsoft.ML.Tokenizers (1)
Model\SentencePieceTokenizer.cs (1)
68
UnknownId
= modelProto.TrainerSpec.UnkId < 0 ? 0 : modelProto.TrainerSpec.UnkId;
11 references to UnknownId
Microsoft.ML.Tokenizers (10)
Model\SentencePieceTokenizer.cs (10)
339
id =
UnknownId
;
346
if (id ==
UnknownId
&& ByteFallback)
688
id =
UnknownId
;
695
if (id ==
UnknownId
&& ByteFallback)
1012
id =
UnknownId
;
1019
if (id ==
UnknownId
&& ByteFallback)
1392
id =
UnknownId
;
1399
if (id ==
UnknownId
&& ByteFallback)
1753
else if (id == tokenizer.
UnknownId
)
1994
else if (id == tokenizer.
UnknownId
)
Microsoft.ML.Tokenizers.Tests (1)
LlamaTests.cs (1)
363
Assert.Equal(0, bpe.
UnknownId
);