simdutf 9.2.1
Unicode at GB/s.
Loading...
Searching...
No Matches
simdutf_c.h
1/***
2 * simdutf_c.h.h - C API for simdutf
3 * This is currently experimental.
4 * We are committed to keeping the C API, but there might be mistakes in our
5 * implementation. Please report any issues you find.
6 */
7
8#ifndef SIMDUTF_C_H
9#define SIMDUTF_C_H
10
11#include <stddef.h>
12#include <stdbool.h>
13#include <stdint.h>
14
15#ifdef __has_include
16 #if __has_include(<uchar.h>)
17 #include <uchar.h>
18 #else // __has_include(<uchar.h>)
19 #define char16_t uint16_t
20 #define char32_t uint32_t
21 #endif // __has_include(<uchar.h>)
22#else // __has_include(<uchar.h>)
23 #define char16_t uint16_t
24 #define char32_t uint32_t
25#endif // __has_include
26
27#ifdef __cplusplus
28extern "C" {
29#endif
30
31/* C-friendly subset of simdutf errors */
32typedef enum simdutf_error_code {
33 SIMDUTF_ERROR_SUCCESS = 0,
34 SIMDUTF_ERROR_HEADER_BITS,
35 SIMDUTF_ERROR_TOO_SHORT,
36 SIMDUTF_ERROR_TOO_LONG,
37 SIMDUTF_ERROR_OVERLONG,
38 SIMDUTF_ERROR_TOO_LARGE,
39 SIMDUTF_ERROR_SURROGATE,
40 SIMDUTF_ERROR_INVALID_BASE64_CHARACTER,
41 SIMDUTF_ERROR_BASE64_INPUT_REMAINDER,
42 SIMDUTF_ERROR_BASE64_EXTRA_BITS,
43 SIMDUTF_ERROR_OUTPUT_BUFFER_TOO_SMALL,
44 SIMDUTF_ERROR_OTHER
45} simdutf_error_code;
46
47typedef struct simdutf_result {
48 simdutf_error_code error;
49 size_t count; /* position of error or number of code units validated */
50} simdutf_result;
51
52typedef struct simdutf_full_result {
53 simdutf_error_code error;
54 size_t input_count; /* number of input units consumed */
55 size_t output_count; /* number of output bytes written */
56} simdutf_full_result;
57
58typedef enum simdutf_encoding_type {
59 SIMDUTF_ENCODING_UNSPECIFIED = 0,
60 SIMDUTF_ENCODING_UTF8 = 1,
61 SIMDUTF_ENCODING_UTF16_LE = 2,
62 SIMDUTF_ENCODING_UTF16_BE = 4,
63 SIMDUTF_ENCODING_UTF32_LE = 8,
64 SIMDUTF_ENCODING_UTF32_BE = 16
65} simdutf_encoding_type;
66
67/* Validate UTF-8: returns true iff input is valid UTF-8 */
68bool simdutf_validate_utf8(const char *buf, size_t len);
69
70/* Validate UTF-8 with detailed result */
71simdutf_result simdutf_validate_utf8_with_errors(const char *buf, size_t len);
72
73/* Encoding detection */
74simdutf_encoding_type simdutf_autodetect_encoding(const char *input,
75 size_t length);
76int simdutf_detect_encodings(const char *input, size_t length);
77
78/* ASCII validation */
79bool simdutf_validate_ascii(const char *buf, size_t len);
80simdutf_result simdutf_validate_ascii_with_errors(const char *buf, size_t len);
81
82/* UTF-16 ASCII checks */
83bool simdutf_validate_utf16_as_ascii(const char16_t *buf, size_t len);
84bool simdutf_validate_utf16be_as_ascii(const char16_t *buf, size_t len);
85bool simdutf_validate_utf16le_as_ascii(const char16_t *buf, size_t len);
86
87/* UTF-16/UTF-8/UTF-32 validation (native/endian-specific) */
88bool simdutf_validate_utf16(const char16_t *buf, size_t len);
89bool simdutf_validate_utf16le(const char16_t *buf, size_t len);
90bool simdutf_validate_utf16be(const char16_t *buf, size_t len);
91simdutf_result simdutf_validate_utf16_with_errors(const char16_t *buf,
92 size_t len);
93simdutf_result simdutf_validate_utf16le_with_errors(const char16_t *buf,
94 size_t len);
95simdutf_result simdutf_validate_utf16be_with_errors(const char16_t *buf,
96 size_t len);
97
98bool simdutf_validate_utf32(const char32_t *buf, size_t len);
99simdutf_result simdutf_validate_utf32_with_errors(const char32_t *buf,
100 size_t len);
101
102/* to_well_formed UTF-16 helpers */
103void simdutf_to_well_formed_utf16le(const char16_t *input, size_t len,
104 char16_t *output);
105void simdutf_to_well_formed_utf16be(const char16_t *input, size_t len,
106 char16_t *output);
107void simdutf_to_well_formed_utf16(const char16_t *input, size_t len,
108 char16_t *output);
109
110/* Counting */
111size_t simdutf_count_utf16(const char16_t *input, size_t length);
112size_t simdutf_count_utf16le(const char16_t *input, size_t length);
113size_t simdutf_count_utf16be(const char16_t *input, size_t length);
114size_t simdutf_count_utf8(const char *input, size_t length);
115
116/* Length estimators */
117size_t simdutf_utf8_length_from_latin1(const char *input, size_t length);
118size_t simdutf_latin1_length_from_utf8(const char *input, size_t length);
119size_t simdutf_latin1_length_from_utf16(size_t length);
120size_t simdutf_latin1_length_from_utf32(size_t length);
121size_t simdutf_utf16_length_from_utf8(const char *input, size_t length);
122size_t simdutf_utf32_length_from_utf8(const char *input, size_t length);
123size_t simdutf_utf8_length_from_utf16(const char16_t *input, size_t length);
124size_t simdutf_utf8_length_from_utf32(const char32_t *input, size_t length);
125simdutf_result
126simdutf_utf8_length_from_utf16_with_replacement(const char16_t *input,
127 size_t length);
128size_t simdutf_utf8_length_from_utf16le(const char16_t *input, size_t length);
129size_t simdutf_utf8_length_from_utf16be(const char16_t *input, size_t length);
130simdutf_result
131simdutf_utf8_length_from_utf16le_with_replacement(const char16_t *input,
132 size_t length);
133simdutf_result
134simdutf_utf8_length_from_utf16be_with_replacement(const char16_t *input,
135 size_t length);
136
137/* Conversions: latin1 <-> utf8, utf8 <-> utf16/utf32, utf16 <-> utf8, etc. */
138size_t simdutf_convert_latin1_to_utf8(const char *input, size_t length,
139 char *output);
140size_t simdutf_convert_latin1_to_utf8_safe(const char *input, size_t length,
141 char *output, size_t utf8_len);
142simdutf_full_result simdutf_convert_latin1_to_utf8_safe_with_details(
143 const char *input, size_t length, char *output, size_t utf8_len);
144size_t simdutf_convert_latin1_to_utf16le(const char *input, size_t length,
145 char16_t *output);
146size_t simdutf_convert_latin1_to_utf16be(const char *input, size_t length,
147 char16_t *output);
148size_t simdutf_convert_latin1_to_utf16(const char *input, size_t length,
149 char16_t *output);
150size_t simdutf_convert_latin1_to_utf32(const char *input, size_t length,
151 char32_t *output);
152
153size_t simdutf_convert_utf8_to_latin1(const char *input, size_t length,
154 char *output);
155size_t simdutf_convert_utf8_to_utf16le(const char *input, size_t length,
156 char16_t *output);
157size_t simdutf_convert_utf8_to_utf16be(const char *input, size_t length,
158 char16_t *output);
159size_t simdutf_convert_utf8_to_utf16(const char *input, size_t length,
160 char16_t *output);
161
162size_t simdutf_convert_utf8_to_utf32(const char *input, size_t length,
163 char32_t *output);
164simdutf_result simdutf_convert_utf8_to_latin1_with_errors(const char *input,
165 size_t length,
166 char *output);
167simdutf_result simdutf_convert_utf8_to_utf16_with_errors(const char *input,
168 size_t length,
169 char16_t *output);
170simdutf_result simdutf_convert_utf8_to_utf16le_with_errors(const char *input,
171 size_t length,
172 char16_t *output);
173simdutf_result simdutf_convert_utf8_to_utf16be_with_errors(const char *input,
174 size_t length,
175 char16_t *output);
176simdutf_result simdutf_convert_utf8_to_utf32_with_errors(const char *input,
177 size_t length,
178 char32_t *output);
179
180/* Conversions assuming valid input */
181size_t simdutf_convert_valid_utf8_to_latin1(const char *input, size_t length,
182 char *output);
183size_t simdutf_convert_valid_utf8_to_utf16le(const char *input, size_t length,
184 char16_t *output);
185size_t simdutf_convert_valid_utf8_to_utf16be(const char *input, size_t length,
186 char16_t *output);
187size_t simdutf_convert_valid_utf8_to_utf32(const char *input, size_t length,
188 char32_t *output);
189
190/* UTF-16 -> UTF-8 and related conversions */
191size_t simdutf_convert_utf16_to_utf8(const char16_t *input, size_t length,
192 char *output);
193size_t simdutf_convert_utf16le_to_utf8(const char16_t *input, size_t length,
194 char *output);
195size_t simdutf_convert_utf16be_to_utf8(const char16_t *input, size_t length,
196 char *output);
197size_t simdutf_convert_utf16_to_utf8_safe(const char16_t *input, size_t length,
198 char *output, size_t utf8_len);
199simdutf_full_result simdutf_convert_utf16_to_utf8_safe_with_details(
200 const char16_t *input, size_t length, char *output, size_t utf8_len);
201size_t simdutf_convert_utf16_to_latin1(const char16_t *input, size_t length,
202 char *output);
203size_t simdutf_convert_utf16le_to_latin1(const char16_t *input, size_t length,
204 char *output);
205size_t simdutf_convert_utf16be_to_latin1(const char16_t *input, size_t length,
206 char *output);
207simdutf_result
208simdutf_convert_utf16_to_latin1_with_errors(const char16_t *input,
209 size_t length, char *output);
210simdutf_result
211simdutf_convert_utf16le_to_latin1_with_errors(const char16_t *input,
212 size_t length, char *output);
213simdutf_result
214simdutf_convert_utf16be_to_latin1_with_errors(const char16_t *input,
215 size_t length, char *output);
216
217simdutf_result simdutf_convert_utf16_to_utf8_with_errors(const char16_t *input,
218 size_t length,
219 char *output);
220simdutf_result
221simdutf_convert_utf16le_to_utf8_with_errors(const char16_t *input,
222 size_t length, char *output);
223simdutf_result
224simdutf_convert_utf16be_to_utf8_with_errors(const char16_t *input,
225 size_t length, char *output);
226
227/* Convert possibly broken UTF-16 to UTF-8, replacing each unpaired surrogate
228 with U+FFFD (EF BF BD). These always succeed and return the number of bytes
229 written. Size the output buffer with the matching
230 simdutf_utf8_length_from_utf16*_with_replacement function. */
231size_t simdutf_convert_utf16_to_utf8_with_replacement(const char16_t *input,
232 size_t length,
233 char *output);
234simdutf_full_result simdutf_convert_utf16_to_utf8_with_replacement_safe(
235 const char16_t *input, size_t length, char *output, size_t utf8_len);
236size_t simdutf_convert_utf16le_to_utf8_with_replacement(const char16_t *input,
237 size_t length,
238 char *output);
239size_t simdutf_convert_utf16be_to_utf8_with_replacement(const char16_t *input,
240 size_t length,
241 char *output);
242
243size_t simdutf_convert_valid_utf16_to_utf8(const char16_t *input, size_t length,
244 char *output);
245size_t simdutf_convert_valid_utf16_to_latin1(const char16_t *input,
246 size_t length, char *output);
247size_t simdutf_convert_valid_utf16le_to_latin1(const char16_t *input,
248 size_t length, char *output);
249size_t simdutf_convert_valid_utf16be_to_latin1(const char16_t *input,
250 size_t length, char *output);
251
252size_t simdutf_convert_valid_utf16le_to_utf8(const char16_t *input,
253 size_t length, char *output);
254size_t simdutf_convert_valid_utf16be_to_utf8(const char16_t *input,
255 size_t length, char *output);
256
257/* UTF-16 <-> UTF-32 conversions */
258size_t simdutf_convert_utf16_to_utf32(const char16_t *input, size_t length,
259 char32_t *output);
260size_t simdutf_convert_utf16le_to_utf32(const char16_t *input, size_t length,
261 char32_t *output);
262size_t simdutf_convert_utf16be_to_utf32(const char16_t *input, size_t length,
263 char32_t *output);
264simdutf_result simdutf_convert_utf16_to_utf32_with_errors(const char16_t *input,
265 size_t length,
266 char32_t *output);
267simdutf_result
268simdutf_convert_utf16le_to_utf32_with_errors(const char16_t *input,
269 size_t length, char32_t *output);
270simdutf_result
271simdutf_convert_utf16be_to_utf32_with_errors(const char16_t *input,
272 size_t length, char32_t *output);
273
274/* Valid UTF-16 conversions */
275size_t simdutf_convert_valid_utf16_to_utf32(const char16_t *input,
276 size_t length, char32_t *output);
277size_t simdutf_convert_valid_utf16le_to_utf32(const char16_t *input,
278 size_t length, char32_t *output);
279size_t simdutf_convert_valid_utf16be_to_utf32(const char16_t *input,
280 size_t length, char32_t *output);
281
282/* UTF-32 -> ... conversions */
283size_t simdutf_convert_utf32_to_utf8(const char32_t *input, size_t length,
284 char *output);
285simdutf_result simdutf_convert_utf32_to_utf8_with_errors(const char32_t *input,
286 size_t length,
287 char *output);
288size_t simdutf_convert_valid_utf32_to_utf8(const char32_t *input, size_t length,
289 char *output);
290
291size_t simdutf_convert_utf32_to_utf16(const char32_t *input, size_t length,
292 char16_t *output);
293size_t simdutf_convert_utf32_to_utf16le(const char32_t *input, size_t length,
294 char16_t *output);
295size_t simdutf_convert_utf32_to_utf16be(const char32_t *input, size_t length,
296 char16_t *output);
297simdutf_result
298simdutf_convert_utf32_to_latin1_with_errors(const char32_t *input,
299 size_t length, char *output);
300
301/* --- Find helpers --- */
302const char *simdutf_find(const char *start, const char *end, char character);
303const char16_t *simdutf_find_utf16(const char16_t *start, const char16_t *end,
304 char16_t character);
305
306/* --- Base64 enums and helpers --- */
307typedef enum simdutf_base64_options {
308 SIMDUTF_BASE64_DEFAULT = 0,
309 SIMDUTF_BASE64_URL = 1,
310 SIMDUTF_BASE64_DEFAULT_NO_PADDING = 2,
311 SIMDUTF_BASE64_URL_WITH_PADDING = 3,
312 SIMDUTF_BASE64_DEFAULT_ACCEPT_GARBAGE = 4,
313 SIMDUTF_BASE64_URL_ACCEPT_GARBAGE = 5,
314 SIMDUTF_BASE64_DEFAULT_OR_URL = 8,
315 SIMDUTF_BASE64_DEFAULT_OR_URL_ACCEPT_GARBAGE = 12
316} simdutf_base64_options;
317
318typedef enum simdutf_last_chunk_handling_options {
319 SIMDUTF_LAST_CHUNK_LOOSE = 0,
320 SIMDUTF_LAST_CHUNK_STRICT = 1,
321 SIMDUTF_LAST_CHUNK_STOP_BEFORE_PARTIAL = 2,
322 SIMDUTF_LAST_CHUNK_ONLY_FULL_CHUNKS = 3
323} simdutf_last_chunk_handling_options;
324
325/* maximal binary length estimators */
326size_t simdutf_maximal_binary_length_from_base64(const char *input,
327 size_t length);
328size_t simdutf_maximal_binary_length_from_base64_utf16(const char16_t *input,
329 size_t length);
330
331/* base64 decoding/encoding */
332simdutf_result simdutf_base64_to_binary(
333 const char *input, size_t length, char *output,
334 simdutf_base64_options options,
335 simdutf_last_chunk_handling_options last_chunk_options);
336simdutf_result simdutf_base64_to_binary_utf16(
337 const char16_t *input, size_t length, char *output,
338 simdutf_base64_options options,
339 simdutf_last_chunk_handling_options last_chunk_options);
340
341size_t simdutf_base64_length_from_binary(size_t length,
342 simdutf_base64_options options);
343size_t simdutf_base64_length_from_binary_with_lines(
344 size_t length, simdutf_base64_options options, size_t line_length);
345
346size_t simdutf_binary_to_base64(const char *input, size_t length, char *output,
347 simdutf_base64_options options);
348size_t simdutf_binary_to_base64_with_lines(const char *input, size_t length,
349 char *output, size_t line_length,
350 simdutf_base64_options options);
351
352/* safe decoding that provides an in/out outlen parameter */
353simdutf_result simdutf_base64_to_binary_safe(
354 const char *input, size_t length, char *output, size_t *outlen,
355 simdutf_base64_options options,
356 simdutf_last_chunk_handling_options last_chunk_options,
357 bool decode_up_to_bad_char);
358simdutf_result simdutf_base64_to_binary_safe_utf16(
359 const char16_t *input, size_t length, char *output, size_t *outlen,
360 simdutf_base64_options options,
361 simdutf_last_chunk_handling_options last_chunk_options,
362 bool decode_up_to_bad_char);
363
364/* detailed decoding returning input_count and output_count */
365simdutf_full_result simdutf_base64_to_binary_details(
366 const char *input, size_t length, char *output,
367 simdutf_base64_options options,
368 simdutf_last_chunk_handling_options last_chunk_options);
369simdutf_full_result simdutf_base64_to_binary_details_utf16(
370 const char16_t *input, size_t length, char *output,
371 simdutf_base64_options options,
372 simdutf_last_chunk_handling_options last_chunk_options);
373
374/* single-character base64 validation */
375bool simdutf_base64_valid(char input, simdutf_base64_options options);
376bool simdutf_base64_valid_utf16(char16_t input, simdutf_base64_options options);
377
378#ifdef __cplusplus
379} /* extern "C" */
380#endif
381
382#endif /* SIMDUTF_C_H */
simdutf_full_result
Definition simdutf_c.h:52
simdutf_result
Definition simdutf_c.h:47