3 writes to ByteCodeToIdOffset
Microsoft.ML.Tokenizers (3)
Model\SentencePieceBpeModel.cs (1)
40ByteCodeToIdOffset = _vocab.TryGetValue("<0x00>", out (int Id, float Score, byte Type) value) ? value.Id : MaxByteId;
Model\SentencePieceUnigramModel.cs (2)
70ByteCodeToIdOffset = _vocab.TryGetValue("<0x00>", out int id) ? id : MaxByteId; 217ByteCodeToIdOffset = _vocab.TryGetValue("<0x00>", out int id) ? id : MaxByteId;
23 references to ByteCodeToIdOffset
Microsoft.ML.Tokenizers (23)
Model\SentencePieceBaseModel.cs (10)
216while (current < ByteCodeToIdOffset) 232EncodeByte(current, OneByteUtf8EncodingMaxId, ByteCodeToIdOffset, ref bytesCount, ref bytesPoolArray, ref sb); 249if (current < ByteCodeToIdOffset) 273bytesPoolArray![bytesCount++] = (byte)(current - ByteCodeToIdOffset); 277EncodeByte(current, OneByteUtf8EncodingMaxId, ByteCodeToIdOffset, ref bytesCount, ref bytesPoolArray, ref sb); 440while (current < ByteCodeToIdOffset) 461if (!EncodeByte(enumerator.Current, OneByteUtf8EncodingMaxId, ByteCodeToIdOffset, ref bytesCount, buffer, ref charsWritten, ref idsConsumed, ref bytesPoolArray)) 515if (current < ByteCodeToIdOffset) 546bytesPoolArray![bytesCount++] = (byte)(current - ByteCodeToIdOffset); 550if (!EncodeByte(current, OneByteUtf8EncodingMaxId, ByteCodeToIdOffset, ref bytesCount, buffer, ref charsWritten, ref idsConsumed, ref bytesPoolArray))
Model\SentencePieceBpeModel.cs (5)
41OneByteUtf8EncodingMaxId = ByteCodeToIdOffset + 0x7F; // 0x7F is the maximum value of the one byte UTF-8 character. 216int id = (int)c + ByteCodeToIdOffset; // byte code is mapped to the to the Ids starting from 4. 240int id = (int)utf8Bytes[j] + ByteCodeToIdOffset; // byte code is mapped to the to the Ids starting from 4. 515accumulatedIds.Add((int)c + ByteCodeToIdOffset); // byte code is mapped to the to the Ids starting from 4. 543accumulatedIds.Add((int)utf8Bytes[j] + ByteCodeToIdOffset); // byte code is mapped to the to the Ids starting from 4.
Model\SentencePieceUnigramModel.cs (8)
71OneByteUtf8EncodingMaxId = ByteCodeToIdOffset + 0x7F; // 0x7F is the maximum value of the one byte UTF-8 character. 72MaxIdByteFallbackId = ByteCodeToIdOffset + 0xFF; // from <0x00> to <0xFF>. 218if (!_vocab.ContainsKey("<0x00>") || !_vocab.TryGetValue("<0xFF>", out int maxByteId) || maxByteId - ByteCodeToIdOffset != 0xFF) 224OneByteUtf8EncodingMaxId = ByteCodeToIdOffset + 0x7F; 225MaxIdByteFallbackId = ByteCodeToIdOffset + 0xFF; 708int id = ByteCodeToIdOffset + destination[0]; 713id = ByteCodeToIdOffset + destination[j]; 1038ids.Insert(IdsIndex, ByteCodeToIdOffset + normalizationSpan[Utf8Index + j]);