simdutf 9.0.0
Unicode at GB/s.
Loading...
Searching...
No Matches
utf16_to_utf8.h
1#ifndef SIMDUTF_UTF16_TO_UTF8_H
2#define SIMDUTF_UTF16_TO_UTF8_H
3
4#include <cstring>
5
6namespace simdutf {
7namespace scalar {
8namespace {
9namespace utf16_to_utf8 {
10
11template <endianness big_endian, typename InputPtr, typename OutputPtr>
12#if SIMDUTF_CPLUSPLUS20
13 requires simdutf::detail::indexes_into_utf16<InputPtr>
14// FIXME constrain output as well
15#endif
16simdutf_constexpr23 size_t convert(InputPtr data, size_t len,
17 OutputPtr utf8_output) {
18 size_t pos = 0;
19 const auto start = utf8_output;
20 while (pos < len) {
21#if SIMDUTF_CPLUSPLUS23
22 if !consteval
23#endif
24 {
25 // try to convert the next block of 8 bytes
26 if (pos + 4 <= len) { // if it is safe to read 8 more bytes, check that
27 // they are ascii
28 uint64_t v;
29 ::memcpy(&v, data + pos, sizeof(uint64_t));
30 if constexpr (!match_system(big_endian)) {
31 v = (v >> 8) | (v << (64 - 8));
32 }
33 if ((v & 0xFF80FF80FF80FF80) == 0) {
34 size_t final_pos = pos + 4;
35 while (pos < final_pos) {
36 *utf8_output++ = !match_system(big_endian)
37 ? char(u16_swap_bytes(data[pos]))
38 : char(data[pos]);
39 pos++;
40 }
41 continue;
42 }
43 }
44 }
45 uint16_t word =
46 !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos];
47 if ((word & 0xFF80) == 0) {
48 // will generate one UTF-8 bytes
49 *utf8_output++ = char(word);
50 pos++;
51 } else if ((word & 0xF800) == 0) {
52 // will generate two UTF-8 bytes
53 // we have 0b110XXXXX 0b10XXXXXX
54 *utf8_output++ = char((word >> 6) | 0b11000000);
55 *utf8_output++ = char((word & 0b111111) | 0b10000000);
56 pos++;
57 } else if ((word & 0xF800) != 0xD800) {
58 // will generate three UTF-8 bytes
59 // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX
60 *utf8_output++ = char((word >> 12) | 0b11100000);
61 *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000);
62 *utf8_output++ = char((word & 0b111111) | 0b10000000);
63 pos++;
64 } else {
65 // must be a surrogate pair
66 if (pos + 1 >= len) {
67 return 0;
68 }
69 uint16_t diff = uint16_t(word - 0xD800);
70 if (diff > 0x3FF) {
71 return 0;
72 }
73 uint16_t next_word = !match_system(big_endian)
74 ? u16_swap_bytes(data[pos + 1])
75 : data[pos + 1];
76 uint16_t diff2 = uint16_t(next_word - 0xDC00);
77 if (diff2 > 0x3FF) {
78 return 0;
79 }
80 uint32_t value = (diff << 10) + diff2 + 0x10000;
81 // will generate four UTF-8 bytes
82 // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX
83 *utf8_output++ = char((value >> 18) | 0b11110000);
84 *utf8_output++ = char(((value >> 12) & 0b111111) | 0b10000000);
85 *utf8_output++ = char(((value >> 6) & 0b111111) | 0b10000000);
86 *utf8_output++ = char((value & 0b111111) | 0b10000000);
87 pos += 2;
88 }
89 }
90 return utf8_output - start;
91}
92
93template <endianness big_endian, bool check_output = false, typename InputPtr,
94 typename OutputPtr>
95#if SIMDUTF_CPLUSPLUS20
96 requires(simdutf::detail::indexes_into_utf16<InputPtr> &&
97 simdutf::detail::index_assignable_from_char<OutputPtr>)
98#endif
99simdutf_constexpr23 full_result convert_with_errors(InputPtr data, size_t len,
100 OutputPtr utf8_output,
101 size_t utf8_len = 0) {
102 if (check_output && utf8_len == 0) {
103 return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, 0, 0);
104 }
105
106 size_t pos = 0;
107 auto start = utf8_output;
108 auto end = utf8_output + utf8_len;
109
110 while (pos < len) {
111#if SIMDUTF_CPLUSPLUS23
112 if !consteval
113#endif
114 {
115 // try to convert the next block of 8 bytes
116 if (pos + 4 <= len) { // if it is safe to read 8 more bytes, check that
117 // they are ascii
118 uint64_t v;
119 ::memcpy(&v, data + pos, sizeof(uint64_t));
120 if constexpr (!match_system(big_endian))
121 v = (v >> 8) | (v << (64 - 8));
122 if ((v & 0xFF80FF80FF80FF80) == 0) {
123 size_t final_pos = pos + 4;
124 while (pos < final_pos) {
125 if (check_output && size_t(end - utf8_output) < 1) {
126 return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos,
127 utf8_output - start);
128 }
129 *utf8_output++ = !match_system(big_endian)
130 ? char(u16_swap_bytes(data[pos]))
131 : char(data[pos]);
132 pos++;
133 }
134 continue;
135 }
136 }
137 }
138
139 uint16_t word =
140 !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos];
141 if ((word & 0xFF80) == 0) {
142 // will generate one UTF-8 bytes
143 if (check_output && size_t(end - utf8_output) < 1) {
144 return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos,
145 utf8_output - start);
146 }
147 *utf8_output++ = char(word);
148 pos++;
149 } else if ((word & 0xF800) == 0) {
150 // will generate two UTF-8 bytes
151 // we have 0b110XXXXX 0b10XXXXXX
152 if (check_output && size_t(end - utf8_output) < 2) {
153 return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos,
154 utf8_output - start);
155 }
156 *utf8_output++ = char((word >> 6) | 0b11000000);
157 *utf8_output++ = char((word & 0b111111) | 0b10000000);
158 pos++;
159
160 } else if ((word & 0xF800) != 0xD800) {
161 // will generate three UTF-8 bytes
162 // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX
163 if (check_output && size_t(end - utf8_output) < 3) {
164 return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos,
165 utf8_output - start);
166 }
167 *utf8_output++ = char((word >> 12) | 0b11100000);
168 *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000);
169 *utf8_output++ = char((word & 0b111111) | 0b10000000);
170 pos++;
171 } else {
172
173 if (check_output && size_t(end - utf8_output) < 4) {
174 return full_result(error_code::OUTPUT_BUFFER_TOO_SMALL, pos,
175 utf8_output - start);
176 }
177 // must be a surrogate pair
178 if (pos + 1 >= len) {
179 return full_result(error_code::SURROGATE, pos, utf8_output - start);
180 }
181 uint16_t diff = uint16_t(word - 0xD800);
182 if (diff > 0x3FF) {
183 return full_result(error_code::SURROGATE, pos, utf8_output - start);
184 }
185 uint16_t next_word = !match_system(big_endian)
186 ? u16_swap_bytes(data[pos + 1])
187 : data[pos + 1];
188 uint16_t diff2 = uint16_t(next_word - 0xDC00);
189 if (diff2 > 0x3FF) {
190 return full_result(error_code::SURROGATE, pos, utf8_output - start);
191 }
192 uint32_t value = (diff << 10) + diff2 + 0x10000;
193 // will generate four UTF-8 bytes
194 // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX
195 *utf8_output++ = char((value >> 18) | 0b11110000);
196 *utf8_output++ = char(((value >> 12) & 0b111111) | 0b10000000);
197 *utf8_output++ = char(((value >> 6) & 0b111111) | 0b10000000);
198 *utf8_output++ = char((value & 0b111111) | 0b10000000);
199 pos += 2;
200 }
201 }
202 return full_result(error_code::SUCCESS, pos, utf8_output - start);
203}
204
205template <endianness big_endian>
206inline result simple_convert_with_errors(const char16_t *buf, size_t len,
207 char *utf8_output) {
208 return convert_with_errors<big_endian, false>(buf, len, utf8_output, 0);
209}
210
211template <endianness big_endian>
212simdutf_constexpr23 size_t convert_with_replacement(const char16_t *data,
213 size_t len,
214 char *utf8_output) {
215 size_t pos = 0;
216 char *start = utf8_output;
217 while (pos < len) {
218#if SIMDUTF_CPLUSPLUS23
219 if !consteval
220#endif
221 {
222 // try to convert the next block of 8 bytes
223 if (pos + 4 <= len) { // if it is safe to read 8 more bytes, check that
224 // they are ascii
225 uint64_t v;
226 ::memcpy(&v, data + pos, sizeof(uint64_t));
227 if constexpr (!match_system(big_endian)) {
228 v = (v >> 8) | (v << (64 - 8));
229 }
230 if ((v & 0xFF80FF80FF80FF80) == 0) {
231 size_t final_pos = pos + 4;
232 while (pos < final_pos) {
233 *utf8_output++ = !match_system(big_endian)
234 ? char(u16_swap_bytes(data[pos]))
235 : char(data[pos]);
236 pos++;
237 }
238 continue;
239 }
240 }
241 }
242 uint16_t word =
243 !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos];
244 if ((word & 0xFF80) == 0) {
245 // will generate one UTF-8 bytes
246 *utf8_output++ = char(word);
247 pos++;
248 } else if ((word & 0xF800) == 0) {
249 // will generate two UTF-8 bytes
250 // we have 0b110XXXXX 0b10XXXXXX
251 *utf8_output++ = char((word >> 6) | 0b11000000);
252 *utf8_output++ = char((word & 0b111111) | 0b10000000);
253 pos++;
254 } else if ((word & 0xF800) != 0xD800) {
255 // will generate three UTF-8 bytes
256 // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX
257 *utf8_output++ = char((word >> 12) | 0b11100000);
258 *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000);
259 *utf8_output++ = char((word & 0b111111) | 0b10000000);
260 pos++;
261 } else {
262 // surrogate range
263 uint16_t diff = uint16_t(word - 0xD800);
264 if (diff <= 0x3FF && pos + 1 < len) {
265 // high surrogate, check for valid pair
266 uint16_t next_word = !match_system(big_endian)
267 ? u16_swap_bytes(data[pos + 1])
268 : data[pos + 1];
269 uint16_t diff2 = uint16_t(next_word - 0xDC00);
270 if (diff2 <= 0x3FF) {
271 // valid surrogate pair
272 uint32_t value = (diff << 10) + diff2 + 0x10000;
273 // will generate four UTF-8 bytes
274 *utf8_output++ = char((value >> 18) | 0b11110000);
275 *utf8_output++ = char(((value >> 12) & 0b111111) | 0b10000000);
276 *utf8_output++ = char(((value >> 6) & 0b111111) | 0b10000000);
277 *utf8_output++ = char((value & 0b111111) | 0b10000000);
278 pos += 2;
279 continue;
280 }
281 }
282 // unpaired surrogate: replace with U+FFFD (0xEF 0xBF 0xBD)
283 *utf8_output++ = char(0xef);
284 *utf8_output++ = char(0xbf);
285 *utf8_output++ = char(0xbd);
286 pos++;
287 }
288 }
289 return utf8_output - start;
290}
291
292} // namespace utf16_to_utf8
293} // unnamed namespace
294} // namespace scalar
295} // namespace simdutf
296
297#endif
simdutf
helpers placed in namespace detail are not a part of the public API
Definition base64_implementation.h:8