1 write to UnknownId
Microsoft.ML.Tokenizers (1)
Model\SentencePieceBpeTokenizer.cs (1)
68UnknownId = modelProto.TrainerSpec.UnkId < 0 ? 0 : modelProto.TrainerSpec.UnkId;
11 references to UnknownId
Microsoft.ML.Tokenizers (10)
Model\SentencePieceBpeTokenizer.cs (10)
339id = UnknownId; 346if (id == UnknownId && ByteFallback) 688id = UnknownId; 695if (id == UnknownId && ByteFallback) 1012id = UnknownId; 1019if (id == UnknownId && ByteFallback) 1392id = UnknownId; 1399if (id == UnknownId && ByteFallback) 1753else if (id == tokenizer.UnknownId) 1994else if (id == tokenizer.UnknownId)
Microsoft.ML.Tokenizers.Tests (1)
LlamaTests.cs (1)
363Assert.Equal(0, bpe.UnknownId);