1 write to UnknownId
Microsoft.ML.Tokenizers (1)
Model\SentencePieceTokenizer.cs (1)
68UnknownId = modelProto.TrainerSpec.UnkId < 0 ? 0 : modelProto.TrainerSpec.UnkId;
11 references to UnknownId
Microsoft.ML.Tokenizers (10)
Model\SentencePieceTokenizer.cs (10)
340id = UnknownId; 347if (id == UnknownId && ByteFallback) 689id = UnknownId; 696if (id == UnknownId && ByteFallback) 1013id = UnknownId; 1020if (id == UnknownId && ByteFallback) 1393id = UnknownId; 1400if (id == UnknownId && ByteFallback) 1754else if (id == tokenizer.UnknownId) 1995else if (id == tokenizer.UnknownId)
Microsoft.ML.Tokenizers.Tests (1)
LlamaTests.cs (1)
363Assert.Equal(0, bpe.UnknownId);