2 writes to MaxByteId
Microsoft.ML.Tokenizers (2)
Model\SentencePieceBpeModel.cs (1)
36MaxByteId = i;
Model\SentencePieceUnigramModel.cs (1)
58MaxByteId = i;
8 references to MaxByteId
Microsoft.ML.Tokenizers (8)
Model\SentencePieceBaseModel.cs (6)
168if (current <= MaxByteId && ByteFallback) 186if (current <= MaxByteId && ByteFallback) 218if (current <= MaxByteId && ByteFallback) 393if (current <= MaxByteId && ByteFallback) 415if (current <= MaxByteId && ByteFallback) 491if (current <= MaxByteId && ByteFallback)
Model\SentencePieceBpeModel.cs (1)
40ByteCodeToIdOffset = _vocab.TryGetValue("<0x00>", out (int Id, float Score, byte Type) value) ? value.Id : MaxByteId;
Model\SentencePieceUnigramModel.cs (1)
67ByteCodeToIdOffset = _vocab.TryGetValue("<0x00>", out int id) ? id : MaxByteId;