1 /**
2 * Copyright (c) 2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #ifndef FOUNDATION_ACE_FRAMEWORKS_BASE_UTILS_UTF_H
17 #define FOUNDATION_ACE_FRAMEWORKS_BASE_UTILS_UTF_H
18
19 #include <cstddef>
20 #include <cstdint>
21 #include <utility>
22 #include <string>
23
24 namespace OHOS::Ace {
25
26 /*
27 * https://en.wikipedia.org/wiki/UTF-8
28 *
29 * N Bits for First Last Byte 1 Byte 2 Byte 3 Byte 4
30 * code point code point code point
31 * 1 7 U+0000 U+007F 0xxxxxxx
32 * 2 11 U+0080 U+07FF 110xxxxx 10xxxxxx
33 * 3 16 U+0800 U+FFFF 1110xxxx 10xxxxxx 10xxxxxx
34 * 4 21 U+10000 U+10FFFF 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
35 */
36 constexpr size_t MASK1 = 0x80;
37 constexpr size_t MASK2 = 0x20;
38 constexpr size_t MASK3 = 0x10;
39
40 constexpr size_t MASK_4BIT = 0x0f;
41 constexpr size_t MASK_5BIT = 0x1f;
42 constexpr size_t MASK_6BIT = 0x3f;
43 constexpr size_t MASK_10BIT = 0x03ff;
44 constexpr size_t MASK_16BIT = 0xffff;
45
46 constexpr size_t DATA_WIDTH = 6;
47 constexpr size_t PAIR_ELEMENT_WIDTH = 16;
48
49 constexpr size_t HI_SURROGATE_MIN = 0xd800;
50 constexpr size_t HI_SURROGATE_MAX = 0xdbff;
51 constexpr size_t LO_SURROGATE_MIN = 0xdc00;
52 constexpr size_t LO_SURROGATE_MAX = 0xdfff;
53
54 constexpr size_t LO_SUPPLEMENTS_MIN = 0x10000;
55
56 constexpr size_t U16_LEAD = 0xd7c0;
57 constexpr size_t U16_TAIL = 0xdc00;
58
59 constexpr uint8_t MUTF8_1B_MAX = 0x7f;
60
61 constexpr uint16_t MUTF8_2B_MAX = 0x7ff;
62 constexpr uint8_t MUTF8_2B_FIRST = 0xc0;
63 constexpr uint8_t MUTF8_2B_SECOND = 0x80;
64
65 constexpr uint8_t MUTF8_3B_FIRST = 0xe0;
66 constexpr uint8_t MUTF8_3B_SECOND = 0x80;
67 constexpr uint8_t MUTF8_3B_THIRD = 0x80;
68
69 constexpr uint8_t MUTF8_4B_FIRST = 0xf0;
70
71 constexpr size_t MAX_U16 = 0xffff;
72 constexpr size_t CONST_2 = 2;
73 constexpr size_t CONST_3 = 3;
74 constexpr size_t CONST_4 = 4;
75 constexpr size_t CONST_6 = 6;
76 constexpr size_t CONST_12 = 12;
77
78 constexpr uint16_t DECODE_LEAD_LOW = 0xD800;
79 constexpr uint16_t DECODE_LEAD_HIGH = 0xDBFF;
80 constexpr uint16_t DECODE_TRAIL_LOW = 0xDC00;
81 constexpr uint16_t DECODE_TRAIL_HIGH = 0xDFFF;
82 constexpr uint32_t DECODE_FIRST_FACTOR = 0x400;
83 constexpr uint32_t DECODE_SECOND_FACTOR = 0x10000;
84 constexpr uint32_t UTF8_OFFSET = 6;
85 constexpr uint32_t UTF16_OFFSET = 10;
86 constexpr uint16_t SURROGATE_MASK = 0xF800;
87 constexpr uint16_t UTF16_REPLACEMENT_CHARACTER = 0xFFFD;
88
89 constexpr uint8_t UTF8_1B_MAX = 0x7f;
90
91 constexpr uint16_t UTF8_2B_MAX = 0x7ff;
92 constexpr uint8_t UTF8_2B_FIRST = 0xc0;
93 constexpr uint8_t UTF8_2B_SECOND = 0x80;
94 constexpr uint8_t UTF8_2B_THIRD = 0x3f;
95 constexpr uint8_t UTF8_2B_FIRST_MIN = 0xc2; // the minimum for 2 bytes is 128, which is 0xc280
96
97 constexpr uint16_t UTF8_3B_MAX = 0xffff;
98 constexpr uint8_t UTF8_3B_FIRST = 0xe0;
99 constexpr uint8_t UTF8_3B_SECOND = 0x80;
100 constexpr uint8_t UTF8_3B_THIRD = 0x80;
101 constexpr uint8_t UTF8_3B_SECOND_MIN = 0xa0; // the minimum for 3 bytes is 2048, which is 0xe0a080
102
103 constexpr uint8_t UTF8_4B_FIRST = 0xf0;
104 constexpr uint8_t UTF8_4B_SECOND_MIN = 0x90; // the minimum for 4 bytes is 65536, which is 0xf0908080
105
106 constexpr uint8_t BYTE_MASK = 0xbf;
107 constexpr uint8_t BYTE_MARK = 0x80;
108
109 enum UTF8BytePatterns {
110 UTF8_TWO_BYTE_MASK = 0xE0,
111 UTF8_TWO_BYTE_PATTERN = 0xC0,
112 UTF8_THREE_BYTE_MASK = 0xF0,
113 UTF8_THREE_BYTE_PATTERN = 0xE0,
114 UTF8_FOUR_BYTE_MASK = 0xF8,
115 UTF8_FOUR_BYTE_PATTERN = 0xF0,
116 UTF8_MULTIBYTE_FOLLOWER = 0x80,
117 UTF8_HIGH_BIT = 0x80
118 };
119
120 enum UTF16LEPatterns {
121 UTF16LE_BOM_FF = 0xFF,
122 UTF16LE_BOM_FE = 0xFE,
123 UTF16LE_ZERO_BYTE = 0x00
124 };
125
126 enum INDEX {
127 INDEX_ONE = 1,
128 INDEX_TWO = 2,
129 INDEX_THREE = 3
130 };
131
132 enum UtfLength : uint8_t { ONE = 1, TWO = 2, THREE = 3, FOUR = 4 };
133
134 const unsigned char FIRST_BYTE_MARK[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };
135
136 std::pair<uint32_t, size_t> ConvertMUtf8ToUtf16Pair(const uint8_t* data, size_t maxBytes = 4);
137
138 size_t MUtf8ToUtf16Size(const uint8_t* mutf8, size_t mutf8Len);
139
140 size_t ConvertRegionUtf8ToUtf16(
141 const uint8_t* utf8In, uint16_t* utf16Out, size_t utf8Len, size_t utf16Len, size_t start);
142
143 size_t DebuggerConvertRegionUtf16ToUtf8(const uint16_t* utf16In, uint8_t* utf8Out, size_t utf16Len, size_t utf8Len,
144 size_t start);
145
146 void ConvertIllegalStr(std::string& str);
147
148 bool IsUTF8(std::string& data);
149
SplitUtf16Pair(uint32_t pair)150 inline std::pair<uint16_t, uint16_t> SplitUtf16Pair(uint32_t pair)
151 {
152 constexpr size_t P1_MASK = 0xffff;
153 constexpr size_t P2_SHIFT = 16;
154 return { pair >> P2_SHIFT, pair & P1_MASK };
155 }
156
157 } // namespace OHOS::Ace
158
159 #endif