2 writes to MaxByteId
Microsoft.ML.Tokenizers (2)
Model\SentencePieceBpeModel.cs (1)
36MaxByteId = i;
Model\SentencePieceUnigramModel.cs (1)
50MaxByteId = i;
8 references to MaxByteId
Microsoft.ML.Tokenizers (8)
Model\SentencePieceBaseModel.cs (6)
229if (current <= MaxByteId && ByteFallback) 247if (current <= MaxByteId && ByteFallback) 279if (current <= MaxByteId && ByteFallback) 454if (current <= MaxByteId && ByteFallback) 476if (current <= MaxByteId && ByteFallback) 552if (current <= MaxByteId && ByteFallback)
Model\SentencePieceBpeModel.cs (1)
40ByteCodeToIdOffset = _vocab.TryGetValue("<0x00>", out (int Id, float Score, byte Type) value) ? value.Id : MaxByteId;
Model\SentencePieceUnigramModel.cs (1)
59ByteCodeToIdOffset = _vocab.TryGetValue("<0x00>", out int id) ? id : MaxByteId;