3 writes to MaxByteId
Microsoft.ML.Tokenizers (3)
Model\SentencePieceBpeModel.cs (1)
36MaxByteId = i;
Model\SentencePieceUnigramModel.cs (2)
61MaxByteId = i; 223MaxByteId = maxByteId;
9 references to MaxByteId
Microsoft.ML.Tokenizers (9)
Model\SentencePieceBaseModel.cs (6)
212if (current <= MaxByteId && ByteFallback) 230if (current <= MaxByteId && ByteFallback) 262if (current <= MaxByteId && ByteFallback) 437if (current <= MaxByteId && ByteFallback) 459if (current <= MaxByteId && ByteFallback) 535if (current <= MaxByteId && ByteFallback)
Model\SentencePieceBpeModel.cs (1)
40ByteCodeToIdOffset = _vocab.TryGetValue("<0x00>", out (int Id, float Score, byte Type) value) ? value.Id : MaxByteId;
Model\SentencePieceUnigramModel.cs (2)
70ByteCodeToIdOffset = _vocab.TryGetValue("<0x00>", out int id) ? id : MaxByteId; 217ByteCodeToIdOffset = _vocab.TryGetValue("<0x00>", out int id) ? id : MaxByteId;