// Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. using System.Buffers.Binary; using System.Diagnostics; using System.Numerics; using System.Runtime.CompilerServices; namespace System.Text.Unicode { internal static partial class Utf8Utility { /// <summary> /// Given a machine-endian DWORD which four bytes of UTF-8 data, interprets the /// first three bytes as a three-byte UTF-8 subsequence and returns the UTF-16 representation. /// </summary> [MethodImpl(MethodImplOptions.AggressiveInlining)] private static uint ExtractCharFromFirstThreeByteSequence(uint value) { Debug.Assert(UInt32BeginsWithUtf8ThreeByteMask(value)); if (BitConverter.IsLittleEndian) { // value = [ ######## | 10xxxxxx 10yyyyyy 1110zzzz ] return ((value & 0x003F0_000u) >> 16) | ((value & 0x0000_3F00u) >> 2) | ((value & 0x0000_000Fu) << 12); } else { // value = [ 1110zzzz 10yyyyyy 10xxxxxx | ######## ] return ((value & 0x0F00_0000u) >> 12) | ((value & 0x003F_0000u) >> 10) | ((value & 0x0000_3F00u) >> 8); } } /// <summary> /// Given a machine-endian DWORD which four bytes of UTF-8 data, interprets the /// first two bytes as a two-byte UTF-8 subsequence and returns the UTF-16 representation. /// </summary> [MethodImpl(MethodImplOptions.AggressiveInlining)] private static uint ExtractCharFromFirstTwoByteSequence(uint value) { Debug.Assert(UInt32BeginsWithUtf8TwoByteMask(value) && !UInt32BeginsWithOverlongUtf8TwoByteSequence(value)); if (BitConverter.IsLittleEndian) { // value = [ ######## ######## | 10xxxxxx 110yyyyy ] uint leadingByte = (uint)(byte)value << 6; return (uint)(byte)(value >> 8) + leadingByte - (0xC0u << 6) - 0x80u; // remove header bits } else { // value = [ 110yyyyy 10xxxxxx | ######## ######## ] return (char)(((value & 0x1F00_0000u) >> 18) | ((value & 0x003F_0000u) >> 16)); } } /// <summary> /// Given a machine-endian DWORD which represents four bytes of UTF-8 data, interprets the input as a /// four-byte UTF-8 sequence and returns the machine-endian DWORD of the UTF-16 representation. /// </summary> [MethodImpl(MethodImplOptions.AggressiveInlining)] private static uint ExtractCharsFromFourByteSequence(uint value) { if (BitConverter.IsLittleEndian) { // input is UTF8 [ 10xxxxxx 10yyyyyy 10uuzzzz 11110uuu ] = scalar 000uuuuu zzzzyyyy yyxxxxxx // want to return UTF16 scalar 000uuuuuzzzzyyyyyyxxxxxx = [ 110111yy yyxxxxxx 110110ww wwzzzzyy ] // where wwww = uuuuu - 1 uint retVal = (uint)(byte)value << 8; // retVal = [ 00000000 00000000 11110uuu 00000000 ] retVal |= (value & 0x0000_3F00u) >> 6; // retVal = [ 00000000 00000000 11110uuu uuzzzz00 ] retVal |= (value & 0x0030_0000u) >> 20; // retVal = [ 00000000 00000000 11110uuu uuzzzzyy ] retVal |= (value & 0x3F00_0000u) >> 8; // retVal = [ 00000000 00xxxxxx 11110uuu uuzzzzyy ] retVal |= (value & 0x000F_0000u) << 6; // retVal = [ 000000yy yyxxxxxx 11110uuu uuzzzzyy ] retVal -= 0x0000_0040u; // retVal = [ 000000yy yyxxxxxx 111100ww wwzzzzyy ] retVal -= 0x0000_2000u; // retVal = [ 000000yy yyxxxxxx 110100ww wwzzzzyy ] retVal += 0x0000_0800u; // retVal = [ 000000yy yyxxxxxx 110110ww wwzzzzyy ] retVal += 0xDC00_0000u; // retVal = [ 110111yy yyxxxxxx 110110ww wwzzzzyy ] return retVal; } else { // input is UTF8 [ 11110uuu 10uuzzzz 10yyyyyy 10xxxxxx ] = scalar 000uuuuu zzzzyyyy yyxxxxxx // want to return UTF16 scalar 000uuuuuxxxxxxxxxxxxxxxx = [ 110110wwwwxxxxxx 110111xxxxxxxxx ] // where wwww = uuuuu - 1 uint retVal = value & 0xFF00_0000u; // retVal = [ 11110uuu 00000000 00000000 00000000 ] retVal |= (value & 0x003F_0000u) << 2; // retVal = [ 11110uuu uuzzzz00 00000000 00000000 ] retVal |= (value & 0x0000_3000u) << 4; // retVal = [ 11110uuu uuzzzzyy 00000000 00000000 ] retVal |= (value & 0x0000_0F00u) >> 2; // retVal = [ 11110uuu uuzzzzyy 000000yy yy000000 ] retVal |= (value & 0x0000_003Fu); // retVal = [ 11110uuu uuzzzzyy 000000yy yyxxxxxx ] retVal -= 0x2000_0000u; // retVal = [ 11010uuu uuzzzzyy 000000yy yyxxxxxx ] retVal -= 0x0040_0000u; // retVal = [ 110100ww wwzzzzyy 000000yy yyxxxxxx ] retVal += 0x0000_DC00u; // retVal = [ 110100ww wwzzzzyy 110111yy yyxxxxxx ] retVal += 0x0800_0000u; // retVal = [ 110110ww wwzzzzyy 110111yy yyxxxxxx ] return retVal; } } /// <summary> /// Given a 32-bit integer that represents a valid packed UTF-16 surrogate pair, all in machine-endian order, /// returns the packed 4-byte UTF-8 representation of this scalar value, also in machine-endian order. /// </summary> [MethodImpl(MethodImplOptions.AggressiveInlining)] private static uint ExtractFourUtf8BytesFromSurrogatePair(uint value) { Debug.Assert(IsWellFormedUtf16SurrogatePair(value)); if (BitConverter.IsLittleEndian) { // input = [ 110111yyyyxxxxxx 110110wwwwzzzzyy ] = scalar (000uuuuu zzzzyyyy yyxxxxxx) // must return [ 10xxxxxx 10yyyyyy 10uuzzzz 11110uuu ], where wwww = uuuuu - 1 value += 0x0000_0040u; // = [ 110111yyyyxxxxxx 11011uuuuuzzzzyy ] uint tempA = BinaryPrimitives.ReverseEndianness(value & 0x003F_0700u); // = [ 00000000 00000uuu 00xxxxxx 00000000 ] tempA = BitOperations.RotateLeft(tempA, 16); // = [ 00xxxxxx 00000000 00000000 00000uuu ] uint tempB = (value & 0x00FCu) << 6; // = [ 00000000 00000000 00uuzzzz 00000000 ] uint tempC = (value >> 6) & 0x000F_0000u; // = [ 00000000 0000yyyy 00000000 00000000 ] tempC |= tempB; uint tempD = (value & 0x03u) << 20; // = [ 00000000 00yy0000 00000000 00000000 ] tempD |= 0x8080_80F0u; return tempD | tempA | tempC; // = [ 10xxxxxx 10yyyyyy 10uuzzzz 11110uuu ] } else { // input = [ 110110wwwwzzzzyy 110111yyyyxxxxxx ], where wwww = uuuuu - 1 // must return [ 11110uuu 10uuzzzz 10yyyyyy 10xxxxxx ], where wwww = uuuuu - 1 value -= 0xD800_DC00u; // = [ 000000wwwwzzzzyy 000000yyyyxxxxxx ] value += 0x0040_0000u; // = [ 00000uuuuuzzzzyy 000000yyyyxxxxxx ] uint tempA = value & 0x0700_0000u; // = [ 00000uuu 00000000 00000000 00000000 ] uint tempB = (value >> 2) & 0x003F_0000u; // = [ 00000000 00uuzzzz 00000000 00000000 ] tempB |= tempA; uint tempC = (value << 2) & 0x0000_0F00u; // = [ 00000000 00000000 0000yyyy 00000000 ] uint tempD = (value >> 4) & 0x0000_3000u; // = [ 00000000 00000000 00yy0000 00000000 ] tempD |= tempC; uint tempE = (value & 0x3Fu) + 0xF080_8080u; // = [ 11110000 10000000 10000000 10xxxxxx ] return tempE | tempB | tempD; // = [ 11110uuu 10uuzzzz 10yyyyyy 10xxxxxx ] } } /// <summary> /// Given a machine-endian DWORD which represents two adjacent UTF-8 two-byte sequences, /// returns the machine-endian DWORD representation of that same data as two adjacent /// UTF-16 byte sequences. /// </summary> /// <param name="value"></param> /// <returns></returns> [MethodImpl(MethodImplOptions.AggressiveInlining)] private static uint ExtractTwoCharsPackedFromTwoAdjacentTwoByteSequences(uint value) { // We don't want to swap the position of the high and low WORDs, // as the buffer was read in machine order and will be written in // machine order. if (BitConverter.IsLittleEndian) { // value = [ 10xxxxxx 110yyyyy | 10xxxxxx 110yyyyy ] return ((value & 0x3F003F00u) >> 8) | ((value & 0x001F001Fu) << 6); } else { // value = [ 110yyyyy 10xxxxxx | 110yyyyy 10xxxxxx ] return ((value & 0x1F001F00u) >> 2) | (value & 0x003F003Fu); } } /// <summary> /// Given a machine-endian DWORD which represents two adjacent UTF-16 sequences, /// returns the machine-endian DWORD representation of that same data as two /// adjacent UTF-8 two-byte sequences. /// </summary> [MethodImpl(MethodImplOptions.AggressiveInlining)] private static uint ExtractTwoUtf8TwoByteSequencesFromTwoPackedUtf16Chars(uint value) { // stays in machine endian Debug.Assert(IsFirstCharTwoUtf8Bytes(value) && IsSecondCharTwoUtf8Bytes(value)); if (BitConverter.IsLittleEndian) { // value = [ 00000YYY YYXXXXXX 00000yyy yyxxxxxx ] // want to return [ 10XXXXXX 110YYYYY 10xxxxxx 110yyyyy ] return ((value >> 6) & 0x001F_001Fu) + ((value << 8) & 0x3F00_3F00u) + 0x80C0_80C0u; } else { // value = [ 00000YYY YYXXXXXX 00000yyy yyxxxxxx ] // want to return [ 110YYYYY 10XXXXXX 110yyyyy 10xxxxxx ] return ((value << 2) & 0x1F00_1F00u) + (value & 0x003F_003Fu) + 0xC080_C080u; } } /// <summary> /// Given a machine-endian DWORD which represents two adjacent UTF-16 sequences, /// returns the machine-endian DWORD representation of the first UTF-16 char /// as a UTF-8 two-byte sequence packed into a WORD and zero-extended to DWORD. /// </summary> [MethodImpl(MethodImplOptions.AggressiveInlining)] private static uint ExtractUtf8TwoByteSequenceFromFirstUtf16Char(uint value) { // stays in machine endian Debug.Assert(IsFirstCharTwoUtf8Bytes(value)); if (BitConverter.IsLittleEndian) { // value = [ ######## ######## 00000yyy yyxxxxxx ] // want to return [ ######## ######## 10xxxxxx 110yyyyy ] uint temp = (value << 2) & 0x1F00u; // [ 00000000 00000000 000yyyyy 00000000 ] value &= 0x3Fu; // [ 00000000 00000000 00000000 00xxxxxx ] return BinaryPrimitives.ReverseEndianness((ushort)(temp + value + 0xC080u)); // [ 00000000 00000000 10xxxxxx 110yyyyy ] } else { // value = [ 00000yyy yyxxxxxx ######## ######## ] // want to return [ ######## ######## 110yyyyy 10xxxxxx ] uint temp = (value >> 16) & 0x3Fu; // [ 00000000 00000000 00000000 00xxxxxx ] value = (value >> 14) & 0x1F00u; // [ 00000000 00000000 000yyyyy 0000000 ] return value + temp + 0xC080u; } } /// <summary> /// Given a 32-bit integer that represents two packed UTF-16 characters, all in machine-endian order, /// returns true iff the first UTF-16 character is ASCII. /// </summary> [MethodImpl(MethodImplOptions.AggressiveInlining)] private static bool IsFirstCharAscii(uint value) { // Little-endian: Given [ #### AAAA ], return whether AAAA is in range [ 0000..007F ]. // Big-endian: Given [ AAAA #### ], return whether AAAA is in range [ 0000..007F ]. // Return statement is written this way to work around https://github.com/dotnet/runtime/issues/4207. return (BitConverter.IsLittleEndian && (value & 0xFF80u) == 0) || (!BitConverter.IsLittleEndian && value < 0x0080_0000u); } /// <summary> /// Given a 32-bit integer that represents two packed UTF-16 characters, all in machine-endian order, /// returns true iff the first UTF-16 character requires *at least* 3 bytes to encode in UTF-8. /// This also returns true if the first UTF-16 character is a surrogate character (well-formedness is not validated). /// </summary> [MethodImpl(MethodImplOptions.AggressiveInlining)] private static bool IsFirstCharAtLeastThreeUtf8Bytes(uint value) { // Little-endian: Given [ #### AAAA ], return whether AAAA is in range [ 0800..FFFF ]. // Big-endian: Given [ AAAA #### ], return whether AAAA is in range [ 0800..FFFF ]. // Return statement is written this way to work around https://github.com/dotnet/runtime/issues/4207. return (BitConverter.IsLittleEndian && (value & 0xF800u) != 0) || (!BitConverter.IsLittleEndian && value >= 0x0800_0000u); } /// <summary> /// Given a 32-bit integer that represents two packed UTF-16 characters, all in machine-endian order, /// returns true iff the first UTF-16 character is a surrogate character (either high or low). /// </summary> [MethodImpl(MethodImplOptions.AggressiveInlining)] private static bool IsFirstCharSurrogate(uint value) { // Little-endian: Given [ #### AAAA ], return whether AAAA is in range [ D800..DFFF ]. // Big-endian: Given [ AAAA #### ], return whether AAAA is in range [ D800..DFFF ]. // Return statement is written this way to work around https://github.com/dotnet/runtime/issues/4207. return (BitConverter.IsLittleEndian && ((value - 0xD800u) & 0xF800u) == 0) || (!BitConverter.IsLittleEndian && (value - 0xD800_0000u) < 0x0800_0000u); } /// <summary> /// Given a 32-bit integer that represents two packed UTF-16 characters, all in machine-endian order, /// returns true iff the first UTF-16 character would be encoded as exactly 2 bytes in UTF-8. /// </summary> [MethodImpl(MethodImplOptions.AggressiveInlining)] private static bool IsFirstCharTwoUtf8Bytes(uint value) { // Little-endian: Given [ #### AAAA ], return whether AAAA is in range [ 0080..07FF ]. // Big-endian: Given [ AAAA #### ], return whether AAAA is in range [ 0080..07FF ]. // TODO: I'd like to be able to write "(ushort)(value - 0x0080u) < 0x0780u" for the little-endian // case, but the JIT doesn't currently emit 16-bit comparisons efficiently. // Tracked as https://github.com/dotnet/runtime/issues/10337. // Return statement is written this way to work around https://github.com/dotnet/runtime/issues/4207. return (BitConverter.IsLittleEndian && ((value - 0x0080u) & 0xFFFFu) < 0x0780u) || (!BitConverter.IsLittleEndian && UnicodeUtility.IsInRangeInclusive(value, 0x0080_0000u, 0x07FF_FFFFu)); } /// <summary> /// Returns <see langword="true"/> iff the low byte of <paramref name="value"/> /// is a UTF-8 continuation byte. /// </summary> [MethodImpl(MethodImplOptions.AggressiveInlining)] private static bool IsLowByteUtf8ContinuationByte(uint value) { // The JIT won't emit a single 8-bit signed cmp instruction (see IsUtf8ContinuationByte), // so the best we can do for now is the lea / cmp pair. // Tracked as https://github.com/dotnet/runtime/issues/10337. return (byte)(value - 0x80u) <= 0x3Fu; } /// <summary> /// Given a 32-bit integer that represents two packed UTF-16 characters, all in machine-endian order, /// returns true iff the second UTF-16 character is ASCII. /// </summary> [MethodImpl(MethodImplOptions.AggressiveInlining)] private static bool IsSecondCharAscii(uint value) { // Little-endian: Given [ BBBB #### ], return whether BBBB is in range [ 0000..007F ]. // Big-endian: Given [ #### BBBB ], return whether BBBB is in range [ 0000..007F ]. // Return statement is written this way to work around https://github.com/dotnet/runtime/issues/4207. return (BitConverter.IsLittleEndian && value < 0x0080_0000u) || (!BitConverter.IsLittleEndian && (value & 0xFF80u) == 0); } /// <summary> /// Given a 32-bit integer that represents two packed UTF-16 characters, all in machine-endian order, /// returns true iff the second UTF-16 character requires *at least* 3 bytes to encode in UTF-8. /// This also returns true if the second UTF-16 character is a surrogate character (well-formedness is not validated). /// </summary> [MethodImpl(MethodImplOptions.AggressiveInlining)] private static bool IsSecondCharAtLeastThreeUtf8Bytes(uint value) { // Little-endian: Given [ BBBB #### ], return whether BBBB is in range [ 0800..FFFF ]. // Big-endian: Given [ #### BBBB ], return whether ABBBBAAA is in range [ 0800..FFFF ]. // Return statement is written this way to work around https://github.com/dotnet/runtime/issues/4207. return (BitConverter.IsLittleEndian && (value & 0xF800_0000u) != 0) || (!BitConverter.IsLittleEndian && (value & 0xF800u) != 0); } /// <summary> /// Given a 32-bit integer that represents two packed UTF-16 characters, all in machine-endian order, /// returns true iff the second UTF-16 character is a surrogate character (either high or low). /// </summary> [MethodImpl(MethodImplOptions.AggressiveInlining)] private static bool IsSecondCharSurrogate(uint value) { // Little-endian: Given [ BBBB #### ], return whether BBBB is in range [ D800..DFFF ]. // Big-endian: Given [ #### BBBB ], return whether BBBB is in range [ D800..DFFF ]. // Return statement is written this way to work around https://github.com/dotnet/runtime/issues/4207. return (BitConverter.IsLittleEndian && (value - 0xD800_0000u) < 0x0800_0000u) || (!BitConverter.IsLittleEndian && ((value - 0xD800u) & 0xF800u) == 0); } /// <summary> /// Given a 32-bit integer that represents two packed UTF-16 characters, all in machine-endian order, /// returns true iff the second UTF-16 character would be encoded as exactly 2 bytes in UTF-8. /// </summary> [MethodImpl(MethodImplOptions.AggressiveInlining)] private static bool IsSecondCharTwoUtf8Bytes(uint value) { // Little-endian: Given [ BBBB #### ], return whether BBBB is in range [ 0080..07FF ]. // Big-endian: Given [ #### BBBB ], return whether BBBB is in range [ 0080..07FF ]. // TODO: I'd like to be able to write "(ushort)(value - 0x0080u) < 0x0780u" for the big-endian // case, but the JIT doesn't currently emit 16-bit comparisons efficiently. // Tracked as https://github.com/dotnet/runtime/issues/10337. // Return statement is written this way to work around https://github.com/dotnet/runtime/issues/4207. return (BitConverter.IsLittleEndian && UnicodeUtility.IsInRangeInclusive(value, 0x0080_0000u, 0x07FF_FFFFu)) || (!BitConverter.IsLittleEndian && ((value - 0x0080u) & 0xFFFFu) < 0x0780u); } /// <summary> /// Returns <see langword="true"/> iff <paramref name="value"/> is a UTF-8 continuation byte; /// i.e., has binary representation 10xxxxxx, where x is any bit. /// </summary> [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static bool IsUtf8ContinuationByte(in byte value) { // This API takes its input as a readonly ref so that the JIT can emit "cmp ModRM" statements // directly rather than bounce a temporary through a register. That is, we want the JIT to be // able to emit a single "cmp byte ptr [data], C0h" statement if we're querying a memory location // to see if it's a continuation byte. Data that's already enregistered will go through the // normal "cmp reg, C0h" code paths, perhaps with some extra unnecessary "movzx" instructions. // // The below check takes advantage of the two's complement representation of negative numbers. // [ 0b1000_0000, 0b1011_1111 ] is [ -127 (sbyte.MinValue), -65 ] return (sbyte)value < -64; } /// <summary> /// Given a 32-bit integer that represents two packed UTF-16 characters, all in machine-endian order, /// returns true iff the two characters represent a well-formed UTF-16 surrogate pair. /// </summary> [MethodImpl(MethodImplOptions.AggressiveInlining)] private static bool IsWellFormedUtf16SurrogatePair(uint value) { // Little-endian: Given [ LLLL HHHH ], validate that LLLL in [ DC00..DFFF ] and HHHH in [ D800..DBFF ]. // Big-endian: Given [ HHHH LLLL ], validate that HHHH in [ D800..DBFF ] and LLLL in [ DC00..DFFF ]. // // We're essentially performing a range check on each component of the input in parallel. The allowed range // ends up being "< 0x0400" after the beginning of the allowed range is subtracted from each element. We // can't perform the equivalent of two CMPs in parallel, but we can take advantage of the fact that 0x0400 // is a whole power of 2, which means that a CMP is really just a glorified TEST operation. Two TESTs *can* // be performed in parallel. The logic below then becomes 3 operations: "add/lea; test; jcc". // Return statement is written this way to work around https://github.com/dotnet/runtime/issues/4207. return (BitConverter.IsLittleEndian && ((value - 0xDC00_D800u) & 0xFC00_FC00u) == 0) || (!BitConverter.IsLittleEndian && ((value - 0xD800_DC00u) & 0xFC00_FC00u) == 0); } /// <summary> /// Converts a DWORD from machine-endian to little-endian. /// </summary> [MethodImpl(MethodImplOptions.AggressiveInlining)] private static uint ToLittleEndian(uint value) { if (BitConverter.IsLittleEndian) { return value; } else { return BinaryPrimitives.ReverseEndianness(value); } } /// <summary> /// Given a UTF-8 buffer which has been read into a DWORD in machine endianness, /// returns <see langword="true"/> iff the first two bytes of the buffer are /// an overlong representation of a sequence that should be represented as one byte. /// This method *does not* validate that the sequence matches the appropriate /// 2-byte sequence mask (see <see cref="UInt32BeginsWithUtf8TwoByteMask"/>). /// </summary> [MethodImpl(MethodImplOptions.AggressiveInlining)] private static bool UInt32BeginsWithOverlongUtf8TwoByteSequence(uint value) { // ASSUMPTION: Caller has already checked the '110yyyyy 10xxxxxx' mask of the input. Debug.Assert(UInt32BeginsWithUtf8TwoByteMask(value)); // Per Table 3-7, first byte of two-byte sequence must be within range C2 .. DF. // Since we already validated it's 80 <= ?? <= DF (per mask check earlier), now only need // to check that it's < C2. // Return statement is written this way to work around https://github.com/dotnet/runtime/issues/4207. return (BitConverter.IsLittleEndian && ((byte)value < 0xC2u)) || (!BitConverter.IsLittleEndian && (value < 0xC200_0000u)); } /// <summary> /// Given a UTF-8 buffer which has been read into a DWORD in machine endianness, /// returns <see langword="true"/> iff the first four bytes of the buffer match /// the UTF-8 4-byte sequence mask [ 11110www 10zzzzzz 10yyyyyy 10xxxxxx ]. This /// method *does not* validate that the sequence is well-formed; the caller must /// still perform overlong form or out-of-range checking. /// </summary> [MethodImpl(MethodImplOptions.AggressiveInlining)] private static bool UInt32BeginsWithUtf8FourByteMask(uint value) { // The code in this method is equivalent to the code // below but is slightly more optimized. // // if (BitConverter.IsLittleEndian) // { // const uint mask = 0xC0C0C0F8U; // const uint comparand = 0x808080F0U; // return ((value & mask) == comparand); // } // else // { // const uint mask = 0xF8C0C0C0U; // const uint comparand = 0xF0808000U; // return ((value & mask) == comparand); // } // Return statement is written this way to work around https://github.com/dotnet/runtime/issues/4207. return (BitConverter.IsLittleEndian && (((value - 0x8080_80F0u) & 0xC0C0_C0F8u) == 0)) || (!BitConverter.IsLittleEndian && (((value - 0xF080_8080u) & 0xF8C0_C0C0u) == 0)); } /// <summary> /// Given a UTF-8 buffer which has been read into a DWORD in machine endianness, /// returns <see langword="true"/> iff the first three bytes of the buffer match /// the UTF-8 3-byte sequence mask [ 1110zzzz 10yyyyyy 10xxxxxx ]. This method *does not* /// validate that the sequence is well-formed; the caller must still perform /// overlong form or surrogate checking. /// </summary> [MethodImpl(MethodImplOptions.AggressiveInlining)] private static bool UInt32BeginsWithUtf8ThreeByteMask(uint value) { // The code in this method is equivalent to the code // below but is slightly more optimized. // // if (BitConverter.IsLittleEndian) // { // const uint mask = 0x00C0C0F0U; // const uint comparand = 0x008080E0U; // return ((value & mask) == comparand); // } // else // { // const uint mask = 0xF0C0C000U; // const uint comparand = 0xE0808000U; // return ((value & mask) == comparand); // } // Return statement is written this way to work around https://github.com/dotnet/runtime/issues/4207. return (BitConverter.IsLittleEndian && (((value - 0x0080_80E0u) & 0x00C0_C0F0u) == 0)) || (!BitConverter.IsLittleEndian && (((value - 0xE080_8000u) & 0xF0C0_C000u) == 0)); } /// <summary> /// Given a UTF-8 buffer which has been read into a DWORD in machine endianness, /// returns <see langword="true"/> iff the first two bytes of the buffer match /// the UTF-8 2-byte sequence mask [ 110yyyyy 10xxxxxx ]. This method *does not* /// validate that the sequence is well-formed; the caller must still perform /// overlong form checking. /// </summary> [MethodImpl(MethodImplOptions.AggressiveInlining)] private static bool UInt32BeginsWithUtf8TwoByteMask(uint value) { // The code in this method is equivalent to the code // below but is slightly more optimized. // // if (BitConverter.IsLittleEndian) // { // const uint mask = 0x0000C0E0U; // const uint comparand = 0x000080C0U; // return ((value & mask) == comparand); // } // else // { // const uint mask = 0xE0C00000U; // const uint comparand = 0xC0800000U; // return ((value & mask) == comparand); // } // Return statement is written this way to work around https://github.com/dotnet/runtime/issues/4207. return (BitConverter.IsLittleEndian && (((value - 0x0000_80C0u) & 0x0000_C0E0u) == 0)) || (!BitConverter.IsLittleEndian && (((value - 0xC080_0000u) & 0xE0C0_0000u) == 0)); } /// <summary> /// Given a UTF-8 buffer which has been read into a DWORD in machine endianness, /// returns <see langword="true"/> iff the first two bytes of the buffer are /// an overlong representation of a sequence that should be represented as one byte. /// This method *does not* validate that the sequence matches the appropriate /// 2-byte sequence mask (see <see cref="UInt32BeginsWithUtf8TwoByteMask"/>). /// </summary> [MethodImpl(MethodImplOptions.AggressiveInlining)] private static bool UInt32EndsWithOverlongUtf8TwoByteSequence(uint value) { // ASSUMPTION: Caller has already checked the '110yyyyy 10xxxxxx' mask of the input. Debug.Assert(UInt32EndsWithUtf8TwoByteMask(value)); // Per Table 3-7, first byte of two-byte sequence must be within range C2 .. DF. // We already validated that it's 80 .. DF (per mask check earlier). // C2 = 1100 0010 // DF = 1101 1111 // This means that we can AND the leading byte with the mask 0001 1110 (1E), // and if the result is zero the sequence is overlong. // Return statement is written this way to work around https://github.com/dotnet/runtime/issues/4207. return (BitConverter.IsLittleEndian && ((value & 0x001E_0000u) == 0)) || (!BitConverter.IsLittleEndian && ((value & 0x1E00u) == 0)); } /// <summary> /// Given a UTF-8 buffer which has been read into a DWORD in machine endianness, /// returns <see langword="true"/> iff the last two bytes of the buffer match /// the UTF-8 2-byte sequence mask [ 110yyyyy 10xxxxxx ]. This method *does not* /// validate that the sequence is well-formed; the caller must still perform /// overlong form checking. /// </summary> [MethodImpl(MethodImplOptions.AggressiveInlining)] private static bool UInt32EndsWithUtf8TwoByteMask(uint value) { // The code in this method is equivalent to the code // below but is slightly more optimized. // // if (BitConverter.IsLittleEndian) // { // const uint mask = 0xC0E00000U; // const uint comparand = 0x80C00000U; // return ((value & mask) == comparand); // } // else // { // const uint mask = 0x0000E0C0U; // const uint comparand = 0x0000C080U; // return ((value & mask) == comparand); // } // Return statement is written this way to work around https://github.com/dotnet/runtime/issues/4207. return (BitConverter.IsLittleEndian && (((value - 0x80C0_0000u) & 0xC0E0_0000u) == 0)) || (!BitConverter.IsLittleEndian && (((value - 0x0000_C080u) & 0x0000_E0C0u) == 0)); } /// <summary> /// Given a UTF-8 buffer which has been read into a DWORD on a little-endian machine, /// returns <see langword="true"/> iff the first two bytes of the buffer are a well-formed /// UTF-8 two-byte sequence. This wraps the mask check and the overlong check into a /// single operation. Returns <see langword="false"/> if running on a big-endian machine. /// </summary> [MethodImpl(MethodImplOptions.AggressiveInlining)] private static bool UInt32BeginsWithValidUtf8TwoByteSequenceLittleEndian(uint value) { // Per Table 3-7, valid 2-byte sequences are [ C2..DF ] [ 80..BF ]. // In little-endian, that would be represented as: // [ ######## ######## 10xxxxxx 110yyyyy ]. // Due to the little-endian representation we can perform a trick by ANDing the low // WORD with the bitmask [ 11000000 11111111 ] and checking that the value is within // the range [ 10000000_11000010, 10000000_11011111 ]. This performs both the // 2-byte-sequence bitmask check and overlong form validation with one comparison. Debug.Assert(BitConverter.IsLittleEndian); // Return statement is written this way to work around https://github.com/dotnet/runtime/issues/4207. return (BitConverter.IsLittleEndian && UnicodeUtility.IsInRangeInclusive(value & 0xC0FFu, 0x80C2u, 0x80DFu)) || (!BitConverter.IsLittleEndian && false); } /// <summary> /// Given a UTF-8 buffer which has been read into a DWORD on a little-endian machine, /// returns <see langword="true"/> iff the last two bytes of the buffer are a well-formed /// UTF-8 two-byte sequence. This wraps the mask check and the overlong check into a /// single operation. Returns <see langword="false"/> if running on a big-endian machine. /// </summary> [MethodImpl(MethodImplOptions.AggressiveInlining)] private static bool UInt32EndsWithValidUtf8TwoByteSequenceLittleEndian(uint value) { // See comments in UInt32BeginsWithValidUtf8TwoByteSequenceLittleEndian. Debug.Assert(BitConverter.IsLittleEndian); // Return statement is written this way to work around https://github.com/dotnet/runtime/issues/4207. return (BitConverter.IsLittleEndian && UnicodeUtility.IsInRangeInclusive(value & 0xC0FF_0000u, 0x80C2_0000u, 0x80DF_0000u)) || (!BitConverter.IsLittleEndian && false); } /// <summary> /// Given a UTF-8 buffer which has been read into a DWORD in machine endianness, /// returns <see langword="true"/> iff the first byte of the buffer is ASCII. /// </summary> [MethodImpl(MethodImplOptions.AggressiveInlining)] private static bool UInt32FirstByteIsAscii(uint value) { // Return statement is written this way to work around https://github.com/dotnet/runtime/issues/4207. return (BitConverter.IsLittleEndian && ((value & 0x80u) == 0)) || (!BitConverter.IsLittleEndian && ((int)value >= 0)); } /// <summary> /// Given a UTF-8 buffer which has been read into a DWORD in machine endianness, /// returns <see langword="true"/> iff the fourth byte of the buffer is ASCII. /// </summary> [MethodImpl(MethodImplOptions.AggressiveInlining)] private static bool UInt32FourthByteIsAscii(uint value) { // Return statement is written this way to work around https://github.com/dotnet/runtime/issues/4207. return (BitConverter.IsLittleEndian && ((int)value >= 0)) || (!BitConverter.IsLittleEndian && ((value & 0x80u) == 0)); } /// <summary> /// Given a UTF-8 buffer which has been read into a DWORD in machine endianness, /// returns <see langword="true"/> iff the second byte of the buffer is ASCII. /// </summary> [MethodImpl(MethodImplOptions.AggressiveInlining)] private static bool UInt32SecondByteIsAscii(uint value) { // Return statement is written this way to work around https://github.com/dotnet/runtime/issues/4207. return (BitConverter.IsLittleEndian && ((value & 0x8000u) == 0)) || (!BitConverter.IsLittleEndian && ((value & 0x0080_0000u) == 0)); } /// <summary> /// Given a UTF-8 buffer which has been read into a DWORD in machine endianness, /// returns <see langword="true"/> iff the third byte of the buffer is ASCII. /// </summary> [MethodImpl(MethodImplOptions.AggressiveInlining)] private static bool UInt32ThirdByteIsAscii(uint value) { // Return statement is written this way to work around https://github.com/dotnet/runtime/issues/4207. return (BitConverter.IsLittleEndian && ((value & 0x0080_0000u) == 0)) || (!BitConverter.IsLittleEndian && ((value & 0x8000u) == 0)); } /// <summary> /// Given a DWORD which represents a buffer of 2 packed UTF-16 values in machine endianness, /// converts those scalar values to their 3-byte UTF-8 representation and writes the /// resulting 6 bytes to the destination buffer. /// </summary> [MethodImpl(MethodImplOptions.AggressiveInlining)] private static void WriteTwoUtf16CharsAsTwoUtf8ThreeByteSequences(ref byte outputBuffer, uint value) { Debug.Assert(IsFirstCharAtLeastThreeUtf8Bytes(value) && !IsFirstCharSurrogate(value), "First half of value should've been 0800..D7FF or E000..FFFF"); Debug.Assert(IsSecondCharAtLeastThreeUtf8Bytes(value) && !IsSecondCharSurrogate(value), "Second half of value should've been 0800..D7FF or E000..FFFF"); if (BitConverter.IsLittleEndian) { // value = [ ZZZZYYYY YYXXXXXX zzzzyyyy yyxxxxxx ] // want to write [ 1110ZZZZ 10xxxxxx 10yyyyyy 1110zzzz ] [ 10XXXXXX 10YYYYYY ] uint tempA = ((value << 2) & 0x3F00u) | ((value & 0x3Fu) << 16); // = [ 00000000 00xxxxxx 00yyyyyy 00000000 ] uint tempB = ((value >> 4) & 0x0F00_0000u) | ((value >> 12) & 0x0Fu); // = [ 0000ZZZZ 00000000 00000000 0000zzzz ] Unsafe.WriteUnaligned(ref outputBuffer, tempA + tempB + 0xE080_80E0u); // = [ 1110ZZZZ 10xxxxxx 10yyyyyy 1110zzzz ] Unsafe.WriteUnaligned(ref Unsafe.Add(ref outputBuffer, 4), (ushort)(((value >> 22) & 0x3Fu) + ((value >> 8) & 0x3F00u) + 0x8080u)); // = [ 10XXXXXX 10YYYYYY ] } else { // value = [ zzzzyyyy yyxxxxxx ZZZZYYYY YYXXXXXX ] // want to write [ 1110zzzz ] [ 10yyyyyy ] [ 10xxxxxx ] [ 1110ZZZZ ] [ 10YYYYYY ] [ 10XXXXXX ] Unsafe.Add(ref outputBuffer, 5) = (byte)((value & 0x3Fu) | 0x80u); Unsafe.Add(ref outputBuffer, 4) = (byte)(((value >>= 6) & 0x3Fu) | 0x80u); Unsafe.Add(ref outputBuffer, 3) = (byte)(((value >>= 6) & 0x0Fu) | 0xE0u); Unsafe.Add(ref outputBuffer, 2) = (byte)(((value >>= 4) & 0x3Fu) | 0x80u); Unsafe.Add(ref outputBuffer, 1) = (byte)(((value >>= 6) & 0x3Fu) | 0x80u); outputBuffer = (byte)((value >>= 6) | 0xE0u); } } /// <summary> /// Given a DWORD which represents a buffer of 2 packed UTF-16 values in machine endianness, /// converts the first UTF-16 value to its 3-byte UTF-8 representation and writes the /// resulting 3 bytes to the destination buffer. /// </summary> [MethodImpl(MethodImplOptions.AggressiveInlining)] private static void WriteFirstUtf16CharAsUtf8ThreeByteSequence(ref byte outputBuffer, uint value) { Debug.Assert(IsFirstCharAtLeastThreeUtf8Bytes(value) && !IsFirstCharSurrogate(value), "First half of value should've been 0800..D7FF or E000..FFFF"); if (BitConverter.IsLittleEndian) { // value = [ ######## ######## zzzzyyyy yyxxxxxx ] // want to write [ 10yyyyyy 1110zzzz ] [ 10xxxxxx ] uint tempA = (value << 2) & 0x3F00u; // [ 00yyyyyy 00000000 ] uint tempB = ((uint)(ushort)value >> 12); // [ 00000000 0000zzzz ] Unsafe.WriteUnaligned(ref outputBuffer, (ushort)(tempA + tempB + 0x80E0u)); // [ 10yyyyyy 1110zzzz ] Unsafe.Add(ref outputBuffer, 2) = (byte)((value & 0x3Fu) | ~0x7Fu); // [ 10xxxxxx ] } else { // value = [ zzzzyyyy yyxxxxxx ######## ######## ] // want to write [ 1110zzzz ] [ 10yyyyyy ] [ 10xxxxxx ] Unsafe.Add(ref outputBuffer, 2) = (byte)(((value >>= 16) & 0x3Fu) | 0x80u); Unsafe.Add(ref outputBuffer, 1) = (byte)(((value >>= 6) & 0x3Fu) | 0x80u); outputBuffer = (byte)((value >>= 6) | 0xE0u); } } } }