simdutf Namespace Reference

helpers placed in namespace detail are not a part of the public API More...

Classes
struct	full_result
class	implementation
	An implementation of simdutf for a particular CPU architecture. More...
struct	result

Enumerations
enum	encoding_type {   UTF8 = 1 , UTF16_LE = 2 , UTF16_BE = 4 , UTF32_LE = 8 ,   UTF32_BE = 16 , Latin1 = 32 , unspecified = 0 }
enum	endianness { LITTLE = 0 , BIG = 1 , NATIVE }
enum	error_code {   SUCCESS = 0 , HEADER_BITS , TOO_SHORT , TOO_LONG ,   OVERLONG , TOO_LARGE , SURROGATE , INVALID_BASE64_CHARACTER ,   BASE64_INPUT_REMAINDER , BASE64_EXTRA_BITS , OUTPUT_BUFFER_TOO_SMALL , OTHER }
enum	base64_options : uint64_t {   base64_default = 0 , base64_url = 1 , base64_default_no_padding , base64_url_with_padding ,   base64_default_accept_garbage , base64_url_accept_garbage , base64_default_or_url , base64_default_or_url_accept_garbage }
enum	last_chunk_handling_options : uint64_t { loose = 0 , strict = 1 , stop_before_partial , only_full_chunks }
enum	{ SIMDUTF_VERSION_MAJOR = 9 , SIMDUTF_VERSION_MINOR = 0 , SIMDUTF_VERSION_REVISION = 0 }

Functions
template<typename chartype >
simdutf_warn_unused simdutf_constexpr23 result	slow_base64_to_binary_safe_impl (const chartype *input, size_t length, char *output, size_t &outlen, base64_options options, last_chunk_handling_options last_chunk_options) noexcept
template<typename chartype >
simdutf_warn_unused simdutf_constexpr23 result	base64_to_binary_safe_impl (const chartype *input, size_t length, char *output, size_t &outlen, base64_options options, last_chunk_handling_options last_chunk_handling_options, bool decode_up_to_bad_char) noexcept
simdutf_warn_unused simdutf_really_inline constexpr bool	match_system (endianness e)
simdutf_warn_unused std::string_view	to_string (encoding_type bom)
std::string_view	error_to_string (error_code code) noexcept
simdutf_warn_unused simdutf::encoding_type	autodetect_encoding (const char *input, size_t length) noexcept
	Autodetect the encoding of the input, a single encoding is recommended.
simdutf_really_inline simdutf_warn_unused simdutf::encoding_type	autodetect_encoding (const uint8_t *input, size_t length) noexcept
simdutf_warn_unused int	detect_encodings (const char *input, size_t length) noexcept
	Autodetect the possible encodings of the input in one pass.
simdutf_really_inline simdutf_warn_unused int	detect_encodings (const uint8_t *input, size_t length) noexcept
simdutf_warn_unused bool	validate_utf8 (const char *buf, size_t len) noexcept
	Validate the UTF-8 string.
simdutf_warn_unused result	validate_utf8_with_errors (const char *buf, size_t len) noexcept
	Validate the UTF-8 string and stop on error.
simdutf_warn_unused bool	validate_ascii (const char *buf, size_t len) noexcept
	Validate the ASCII string.
simdutf_warn_unused result	validate_ascii_with_errors (const char *buf, size_t len) noexcept
	Validate the ASCII string and stop on error.
simdutf_warn_unused bool	validate_utf16_as_ascii (const char16_t *buf, size_t len) noexcept
	Validate the ASCII string as a UTF-16 sequence.
simdutf_warn_unused bool	validate_utf16be_as_ascii (const char16_t *buf, size_t len) noexcept
	Validate the ASCII string as a UTF-16BE sequence.
simdutf_warn_unused bool	validate_utf16le_as_ascii (const char16_t *buf, size_t len) noexcept
	Validate the ASCII string as a UTF-16LE sequence.
simdutf_warn_unused bool	validate_utf16 (const char16_t *buf, size_t len) noexcept
	Using native endianness; Validate the UTF-16 string.
simdutf_warn_unused bool	validate_utf16le (const char16_t *buf, size_t len) noexcept
	Validate the UTF-16LE string.
simdutf_warn_unused bool	validate_utf16be (const char16_t *buf, size_t len) noexcept
	Validate the UTF-16BE string.
simdutf_warn_unused result	validate_utf16_with_errors (const char16_t *buf, size_t len) noexcept
	Using native endianness; Validate the UTF-16 string and stop on error.
simdutf_warn_unused result	validate_utf16le_with_errors (const char16_t *buf, size_t len) noexcept
	Validate the UTF-16LE string and stop on error.
simdutf_warn_unused result	validate_utf16be_with_errors (const char16_t *buf, size_t len) noexcept
	Validate the UTF-16BE string and stop on error.
void	to_well_formed_utf16le (const char16_t *input, size_t len, char16_t *output) noexcept
	Fixes an ill-formed UTF-16LE string by replacing mismatched surrogates with the Unicode replacement character U+FFFD.
void	to_well_formed_utf16be (const char16_t *input, size_t len, char16_t *output) noexcept
	Fixes an ill-formed UTF-16BE string by replacing mismatched surrogates with the Unicode replacement character U+FFFD.
void	to_well_formed_utf16 (const char16_t *input, size_t len, char16_t *output) noexcept
	Fixes an ill-formed UTF-16 string by replacing mismatched surrogates with the Unicode replacement character U+FFFD.
simdutf_warn_unused bool	validate_utf32 (const char32_t *buf, size_t len) noexcept
	Validate the UTF-32 string.
simdutf_warn_unused result	validate_utf32_with_errors (const char32_t *buf, size_t len) noexcept
	Validate the UTF-32 string and stop on error.
simdutf_warn_unused size_t	convert_latin1_to_utf8 (const char *input, size_t length, char *utf8_output) noexcept
	Convert Latin1 string into UTF-8 string.
simdutf_warn_unused size_t	convert_latin1_to_utf8_safe (const char *input, size_t length, char *utf8_output, size_t utf8_len) noexcept
	Convert Latin1 string into UTF-8 string with output limit.
simdutf_warn_unused size_t	convert_latin1_to_utf16le (const char *input, size_t length, char16_t *utf16_output) noexcept
	Convert possibly Latin1 string into UTF-16LE string.
simdutf_warn_unused size_t	convert_latin1_to_utf16be (const char *input, size_t length, char16_t *utf16_output) noexcept
	Convert Latin1 string into UTF-16BE string.
simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t	latin1_length_from_utf16 (size_t length) noexcept
	Compute the number of bytes that this UTF-16 string would require in Latin1 format.
simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t	utf16_length_from_latin1 (size_t length) noexcept
	Compute the number of code units that this Latin1 string would require in UTF-16 format.
simdutf_warn_unused size_t	convert_latin1_to_utf32 (const char *input, size_t length, char32_t *utf32_buffer) noexcept
	Convert Latin1 string into UTF-32 string.
simdutf_warn_unused size_t	convert_utf8_to_latin1 (const char *input, size_t length, char *latin1_output) noexcept
	Convert possibly broken UTF-8 string into latin1 string.
simdutf_warn_unused size_t	convert_utf8_to_utf16 (const char *input, size_t length, char16_t *utf16_output) noexcept
	Using native endianness, convert possibly broken UTF-8 string into a UTF-16 string.
simdutf_warn_unused result	utf8_length_from_utf16le_with_replacement (const char16_t *input, size_t length) noexcept
	Compute the number of bytes that this UTF-16LE string would require in UTF-8 format even when the UTF-16LE content contains mismatched surrogates that have to be replaced by the replacement character (0xFFFD).
simdutf_warn_unused result	utf8_length_from_utf16be_with_replacement (const char16_t *input, size_t length) noexcept
	Compute the number of bytes that this UTF-16BE string would require in UTF-8 format even when the UTF-16BE content contains mismatched surrogates that have to be replaced by the replacement character (0xFFFD).
simdutf_warn_unused size_t	convert_latin1_to_utf16 (const char *input, size_t length, char16_t *utf16_output) noexcept
	Using native endianness, convert a Latin1 string into a UTF-16 string.
simdutf_warn_unused size_t	convert_utf8_to_utf16le (const char *input, size_t length, char16_t *utf16_output) noexcept
	Convert possibly broken UTF-8 string into UTF-16LE string.
simdutf_warn_unused size_t	convert_utf8_to_utf16be (const char *input, size_t length, char16_t *utf16_output) noexcept
	Convert possibly broken UTF-8 string into UTF-16BE string.
simdutf_warn_unused result	convert_utf8_to_latin1_with_errors (const char *input, size_t length, char *latin1_output) noexcept
	Convert possibly broken UTF-8 string into latin1 string with errors.
simdutf_warn_unused result	convert_utf8_to_utf16_with_errors (const char *input, size_t length, char16_t *utf16_output) noexcept
	Using native endianness, convert possibly broken UTF-8 string into UTF-16 string and stop on error.
simdutf_warn_unused result	convert_utf8_to_utf16le_with_errors (const char *input, size_t length, char16_t *utf16_output) noexcept
	Convert possibly broken UTF-8 string into UTF-16LE string and stop on error.
simdutf_warn_unused result	convert_utf8_to_utf16be_with_errors (const char *input, size_t length, char16_t *utf16_output) noexcept
	Convert possibly broken UTF-8 string into UTF-16BE string and stop on error.
simdutf_warn_unused size_t	convert_utf8_to_utf32 (const char *input, size_t length, char32_t *utf32_output) noexcept
	Convert possibly broken UTF-8 string into UTF-32 string.
simdutf_warn_unused result	convert_utf8_to_utf32_with_errors (const char *input, size_t length, char32_t *utf32_output) noexcept
	Convert possibly broken UTF-8 string into UTF-32 string and stop on error.
simdutf_warn_unused size_t	convert_valid_utf8_to_latin1 (const char *input, size_t length, char *latin1_output) noexcept
	Convert valid UTF-8 string into latin1 string.
simdutf_warn_unused size_t	convert_valid_utf8_to_utf16 (const char *input, size_t length, char16_t *utf16_buffer) noexcept
	Using native endianness, convert valid UTF-8 string into a UTF-16 string.
simdutf_warn_unused size_t	convert_valid_utf8_to_utf16le (const char *input, size_t length, char16_t *utf16_buffer) noexcept
	Convert valid UTF-8 string into UTF-16LE string.
simdutf_warn_unused size_t	convert_valid_utf8_to_utf16be (const char *input, size_t length, char16_t *utf16_buffer) noexcept
	Convert valid UTF-8 string into UTF-16BE string.
simdutf_warn_unused size_t	convert_valid_utf8_to_utf32 (const char *input, size_t length, char32_t *utf32_buffer) noexcept
	Convert valid UTF-8 string into UTF-32 string.
simdutf_warn_unused size_t	utf8_length_from_latin1 (const char *input, size_t length) noexcept
	Return the number of bytes that this Latin1 string would require in UTF-8 format.
simdutf_warn_unused size_t	latin1_length_from_utf8 (const char *input, size_t length) noexcept
	Compute the number of bytes that this UTF-8 string would require in Latin1 format.
simdutf_warn_unused size_t	utf16_length_from_utf8 (const char *input, size_t length) noexcept
	Compute the number of 2-byte code units that this UTF-8 string would require in UTF-16LE format.
simdutf_warn_unused size_t	utf32_length_from_utf8 (const char *input, size_t length) noexcept
	Compute the number of 4-byte code units that this UTF-8 string would require in UTF-32 format.
simdutf_warn_unused size_t	convert_utf16_to_utf8 (const char16_t *input, size_t length, char *utf8_buffer) noexcept
	Using native endianness, convert possibly broken UTF-16 string into UTF-8 string.
simdutf_warn_unused size_t	convert_utf16_to_utf8_safe (const char16_t *input, size_t length, char *utf8_output, size_t utf8_len) noexcept
	Using native endianness, convert possibly broken UTF-16 string into UTF-8 string with output limit.
simdutf_warn_unused size_t	convert_utf16_to_latin1 (const char16_t *input, size_t length, char *latin1_buffer) noexcept
	Using native endianness, convert possibly broken UTF-16 string into Latin1 string.
simdutf_warn_unused size_t	convert_utf16le_to_latin1 (const char16_t *input, size_t length, char *latin1_buffer) noexcept
	Convert possibly broken UTF-16LE string into Latin1 string.
simdutf_warn_unused size_t	convert_utf16be_to_latin1 (const char16_t *input, size_t length, char *latin1_buffer) noexcept
	Convert possibly broken UTF-16BE string into Latin1 string.
simdutf_warn_unused size_t	convert_utf16le_to_utf8 (const char16_t *input, size_t length, char *utf8_buffer) noexcept
	Convert possibly broken UTF-16LE string into UTF-8 string.
simdutf_warn_unused size_t	convert_utf16be_to_utf8 (const char16_t *input, size_t length, char *utf8_buffer) noexcept
	Convert possibly broken UTF-16BE string into UTF-8 string.
simdutf_warn_unused result	convert_utf16_to_latin1_with_errors (const char16_t *input, size_t length, char *latin1_buffer) noexcept
	Using native endianness, convert possibly broken UTF-16 string into Latin1 string.
simdutf_warn_unused result	convert_utf16le_to_latin1_with_errors (const char16_t *input, size_t length, char *latin1_buffer) noexcept
	Convert possibly broken UTF-16LE string into Latin1 string.
simdutf_warn_unused result	convert_utf16be_to_latin1_with_errors (const char16_t *input, size_t length, char *latin1_buffer) noexcept
	Convert possibly broken UTF-16BE string into Latin1 string.
simdutf_warn_unused result	convert_utf16_to_utf8_with_errors (const char16_t *input, size_t length, char *utf8_buffer) noexcept
	Using native endianness, convert possibly broken UTF-16 string into UTF-8 string and stop on error.
simdutf_warn_unused result	convert_utf16le_to_utf8_with_errors (const char16_t *input, size_t length, char *utf8_buffer) noexcept
	Convert possibly broken UTF-16LE string into UTF-8 string and stop on error.
simdutf_warn_unused result	convert_utf16be_to_utf8_with_errors (const char16_t *input, size_t length, char *utf8_buffer) noexcept
	Convert possibly broken UTF-16BE string into UTF-8 string and stop on error.
simdutf_warn_unused size_t	convert_utf16le_to_utf8_with_replacement (const char16_t *input, size_t length, char *utf8_buffer) noexcept
	Convert possibly broken UTF-16LE string into UTF-8 string, replacing unpaired surrogates with the Unicode replacement character U+FFFD.
simdutf_warn_unused size_t	convert_utf16be_to_utf8_with_replacement (const char16_t *input, size_t length, char *utf8_buffer) noexcept
	Convert possibly broken UTF-16BE string into UTF-8 string, replacing unpaired surrogates with the Unicode replacement character U+FFFD.
simdutf_warn_unused size_t	convert_utf16_to_utf8_with_replacement (const char16_t *input, size_t length, char *utf8_buffer) noexcept
	Convert possibly broken UTF-16 string (native endianness) into UTF-8 string, replacing unpaired surrogates with the Unicode replacement character U+FFFD.
simdutf_warn_unused size_t	convert_valid_utf16_to_utf8 (const char16_t *input, size_t length, char *utf8_buffer) noexcept
	Using native endianness, convert valid UTF-16 string into UTF-8 string.
simdutf_warn_unused size_t	convert_valid_utf16_to_latin1 (const char16_t *input, size_t length, char *latin1_buffer) noexcept
	Using native endianness, convert UTF-16 string into Latin1 string.
simdutf_warn_unused size_t	convert_valid_utf16le_to_latin1 (const char16_t *input, size_t length, char *latin1_buffer) noexcept
	Convert valid UTF-16LE string into Latin1 string.
simdutf_warn_unused size_t	convert_valid_utf16be_to_latin1 (const char16_t *input, size_t length, char *latin1_buffer) noexcept
	Convert valid UTF-16BE string into Latin1 string.
simdutf_warn_unused size_t	convert_valid_utf16le_to_utf8 (const char16_t *input, size_t length, char *utf8_buffer) noexcept
	Convert valid UTF-16LE string into UTF-8 string.
simdutf_warn_unused size_t	convert_valid_utf16be_to_utf8 (const char16_t *input, size_t length, char *utf8_buffer) noexcept
	Convert valid UTF-16BE string into UTF-8 string.
simdutf_warn_unused size_t	convert_utf16_to_utf32 (const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept
	Using native endianness, convert possibly broken UTF-16 string into UTF-32 string.
simdutf_warn_unused size_t	convert_utf16le_to_utf32 (const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept
	Convert possibly broken UTF-16LE string into UTF-32 string.
simdutf_warn_unused size_t	convert_utf16be_to_utf32 (const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept
	Convert possibly broken UTF-16BE string into UTF-32 string.
simdutf_warn_unused result	convert_utf16_to_utf32_with_errors (const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept
	Using native endianness, convert possibly broken UTF-16 string into UTF-32 string and stop on error.
simdutf_warn_unused result	convert_utf16le_to_utf32_with_errors (const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept
	Convert possibly broken UTF-16LE string into UTF-32 string and stop on error.
simdutf_warn_unused result	convert_utf16be_to_utf32_with_errors (const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept
	Convert possibly broken UTF-16BE string into UTF-32 string and stop on error.
simdutf_warn_unused size_t	convert_valid_utf16_to_utf32 (const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept
	Using native endianness, convert valid UTF-16 string into UTF-32 string.
simdutf_warn_unused size_t	convert_valid_utf16le_to_utf32 (const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept
	Convert valid UTF-16LE string into UTF-32 string.
simdutf_warn_unused size_t	convert_valid_utf16be_to_utf32 (const char16_t *input, size_t length, char32_t *utf32_buffer) noexcept
	Convert valid UTF-16BE string into UTF-32 string.
simdutf_warn_unused size_t	utf8_length_from_utf16 (const char16_t *input, size_t length) noexcept
	Using native endianness; Compute the number of bytes that this UTF-16 string would require in UTF-8 format.
simdutf_warn_unused result	utf8_length_from_utf16_with_replacement (const char16_t *input, size_t length) noexcept
	Using native endianness; compute the number of bytes that this UTF-16 string would require in UTF-8 format even when the UTF-16LE content contains mismatched surrogates that have to be replaced by the replacement character (0xFFFD).
simdutf_warn_unused size_t	utf8_length_from_utf16le (const char16_t *input, size_t length) noexcept
	Compute the number of bytes that this UTF-16LE string would require in UTF-8 format.
simdutf_warn_unused size_t	utf8_length_from_utf16be (const char16_t *input, size_t length) noexcept
	Compute the number of bytes that this UTF-16BE string would require in UTF-8 format.
simdutf_warn_unused size_t	convert_utf32_to_utf8 (const char32_t *input, size_t length, char *utf8_buffer) noexcept
	Convert possibly broken UTF-32 string into UTF-8 string.
simdutf_warn_unused result	convert_utf32_to_utf8_with_errors (const char32_t *input, size_t length, char *utf8_buffer) noexcept
	Convert possibly broken UTF-32 string into UTF-8 string and stop on error.
simdutf_warn_unused size_t	convert_valid_utf32_to_utf8 (const char32_t *input, size_t length, char *utf8_buffer) noexcept
	Convert valid UTF-32 string into UTF-8 string.
simdutf_warn_unused size_t	convert_utf32_to_utf16 (const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept
	Using native endianness, convert possibly broken UTF-32 string into a UTF-16 string.
simdutf_warn_unused size_t	convert_utf32_to_utf16le (const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept
	Convert possibly broken UTF-32 string into UTF-16LE string.
simdutf_warn_unused size_t	convert_utf32_to_latin1 (const char32_t *input, size_t length, char *latin1_buffer) noexcept
	Convert possibly broken UTF-32 string into Latin1 string.
simdutf_warn_unused result	convert_utf32_to_latin1_with_errors (const char32_t *input, size_t length, char *latin1_buffer) noexcept
	Convert possibly broken UTF-32 string into Latin1 string and stop on error.
simdutf_warn_unused size_t	convert_valid_utf32_to_latin1 (const char32_t *input, size_t length, char *latin1_buffer) noexcept
	Convert valid UTF-32 string into Latin1 string.
simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 size_t	latin1_length_from_utf32 (size_t length) noexcept
	Compute the number of bytes that this UTF-32 string would require in Latin1 format.
simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 size_t	utf32_length_from_latin1 (size_t length) noexcept
	Compute the number of bytes that this Latin1 string would require in UTF-32 format.
simdutf_warn_unused size_t	convert_utf32_to_utf16be (const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept
	Convert possibly broken UTF-32 string into UTF-16BE string.
simdutf_warn_unused result	convert_utf32_to_utf16_with_errors (const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept
	Using native endianness, convert possibly broken UTF-32 string into UTF-16 string and stop on error.
simdutf_warn_unused result	convert_utf32_to_utf16le_with_errors (const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept
	Convert possibly broken UTF-32 string into UTF-16LE string and stop on error.
simdutf_warn_unused result	convert_utf32_to_utf16be_with_errors (const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept
	Convert possibly broken UTF-32 string into UTF-16BE string and stop on error.
simdutf_warn_unused size_t	convert_valid_utf32_to_utf16 (const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept
	Using native endianness, convert valid UTF-32 string into a UTF-16 string.
simdutf_warn_unused size_t	convert_valid_utf32_to_utf16le (const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept
	Convert valid UTF-32 string into UTF-16LE string.
simdutf_warn_unused size_t	convert_valid_utf32_to_utf16be (const char32_t *input, size_t length, char16_t *utf16_buffer) noexcept
	Convert valid UTF-32 string into UTF-16BE string.
void	change_endianness_utf16 (const char16_t *input, size_t length, char16_t *output) noexcept
	Change the endianness of the input.
simdutf_warn_unused size_t	utf8_length_from_utf32 (const char32_t *input, size_t length) noexcept
	Compute the number of bytes that this UTF-32 string would require in UTF-8 format.
simdutf_warn_unused size_t	utf16_length_from_utf32 (const char32_t *input, size_t length) noexcept
	Compute the number of two-byte code units that this UTF-32 string would require in UTF-16 format.
simdutf_warn_unused size_t	utf32_length_from_utf16 (const char16_t *input, size_t length) noexcept
	Using native endianness; Compute the number of bytes that this UTF-16 string would require in UTF-32 format.
simdutf_warn_unused size_t	utf32_length_from_utf16le (const char16_t *input, size_t length) noexcept
	Compute the number of bytes that this UTF-16LE string would require in UTF-32 format.
simdutf_warn_unused size_t	utf32_length_from_utf16be (const char16_t *input, size_t length) noexcept
	Compute the number of bytes that this UTF-16BE string would require in UTF-32 format.
simdutf_warn_unused size_t	count_utf16 (const char16_t *input, size_t length) noexcept
	Count the number of code points (characters) in the string assuming that it is valid.
simdutf_warn_unused size_t	count_utf16le (const char16_t *input, size_t length) noexcept
	Count the number of code points (characters) in the string assuming that it is valid.
simdutf_warn_unused size_t	count_utf16be (const char16_t *input, size_t length) noexcept
	Count the number of code points (characters) in the string assuming that it is valid.
simdutf_warn_unused size_t	count_utf8 (const char *input, size_t length) noexcept
	Count the number of code points (characters) in the string assuming that it is valid.
simdutf_warn_unused size_t	trim_partial_utf8 (const char *input, size_t length)
	Given a valid UTF-8 string having a possibly truncated last character, this function checks the end of string.
simdutf_warn_unused size_t	trim_partial_utf16be (const char16_t *input, size_t length)
	Given a valid UTF-16BE string having a possibly truncated last character, this function checks the end of string.
simdutf_warn_unused size_t	trim_partial_utf16le (const char16_t *input, size_t length)
	Given a valid UTF-16LE string having a possibly truncated last character, this function checks the end of string.
simdutf_warn_unused size_t	trim_partial_utf16 (const char16_t *input, size_t length)
	Given a valid UTF-16 string having a possibly truncated last character, this function checks the end of string.
simdutf_constexpr23 bool	is_partial (last_chunk_handling_options options)
simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 const char *	find (const char *start, const char *end, char character) noexcept
	Find the first occurrence of a character in a string.
simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 const char16_t *	find (const char16_t *start, const char16_t *end, char16_t character) noexcept
std::string_view	to_string (base64_options options)
std::string_view	to_string (last_chunk_handling_options options)
simdutf_warn_unused size_t	maximal_binary_length_from_base64 (const char *input, size_t length) noexcept
	Provide the maximal binary length in bytes given the base64 input.
simdutf_warn_unused size_t	maximal_binary_length_from_base64 (const char16_t *input, size_t length) noexcept
	Provide the maximal binary length in bytes given the base64 input.
simdutf_warn_unused size_t	binary_length_from_base64 (const char *input, size_t length) noexcept
	Compute the binary length from a base64 input.
simdutf_warn_unused size_t	binary_length_from_base64 (const char16_t *input, size_t length) noexcept
	Compute the binary length from a base64 input.
simdutf_warn_unused result	base64_to_binary (const char *input, size_t length, char *output, base64_options options=base64_default, last_chunk_handling_options last_chunk_options=loose) noexcept
	Convert a base64 input to a binary output.
simdutf_warn_unused simdutf_constexpr23 size_t	base64_length_from_binary (size_t length, base64_options options=base64_default) noexcept
	Provide the base64 length in bytes given the length of a binary input.
simdutf_warn_unused simdutf_constexpr23 size_t	base64_length_from_binary_with_lines (size_t length, base64_options options=base64_default, size_t line_length=default_line_length) noexcept
	Provide the base64 length in bytes given the length of a binary input, taking into account line breaks.
size_t	binary_to_base64 (const char *input, size_t length, char *output, base64_options options=base64_default) noexcept
	Convert a binary input to a base64 output.
size_t	binary_to_base64_with_lines (const char *input, size_t length, char *output, size_t line_length=simdutf::default_line_length, base64_options options=base64_default) noexcept
	Convert a binary input to a base64 output with line breaks.
simdutf_warn_unused result	base64_to_binary (const char16_t *input, size_t length, char *output, base64_options options=base64_default, last_chunk_handling_options last_chunk_options=last_chunk_handling_options::loose) noexcept
	Convert a base64 input to a binary output.
simdutf_warn_unused full_result	base64_to_binary_details (const char *input, size_t length, char *output, base64_options options=base64_default, last_chunk_handling_options last_chunk_options=last_chunk_handling_options::loose) noexcept
	Convert a base64 input to a binary output while returning more details than base64_to_binary.
simdutf_warn_unused full_result	base64_to_binary_details (const char16_t *input, size_t length, char *output, base64_options options=base64_default, last_chunk_handling_options last_chunk_options=last_chunk_handling_options::loose) noexcept
	Convert a base64 input to a binary output while returning more details than base64_to_binary.
simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool	base64_ignorable (char input, base64_options options=base64_default) noexcept
	Check if a character is an ignorable base64 character.
simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool	base64_ignorable (char16_t input, base64_options options=base64_default) noexcept
simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool	base64_valid (char input, base64_options options=base64_default) noexcept
	Check if a character is a valid base64 character.
simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool	base64_valid (char16_t input, base64_options options=base64_default) noexcept
simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool	base64_valid_or_padding (char input, base64_options options=base64_default) noexcept
	Check if a character is a valid base64 character or the padding character ('=').
simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool	base64_valid_or_padding (char16_t input, base64_options options=base64_default) noexcept
simdutf_warn_unused result	base64_to_binary_safe (const char *input, size_t length, char *output, size_t &outlen, base64_options options=base64_default, last_chunk_handling_options last_chunk_options=last_chunk_handling_options::loose, bool decode_up_to_bad_char=false) noexcept
	Convert a base64 input to a binary output.
simdutf_warn_unused result	base64_to_binary_safe (const char16_t *input, size_t length, char *output, size_t &outlen, base64_options options=base64_default, last_chunk_handling_options last_chunk_options=last_chunk_handling_options::loose, bool decode_up_to_bad_char=false) noexcept
SIMDUTF_DLLIMPORTEXPORT const internal::available_implementation_list &	get_available_implementations ()
	The list of available implementations compiled into simdutf.
SIMDUTF_DLLIMPORTEXPORT internal::atomic_ptr< const implementation > &	get_active_implementation ()
	The active implementation.

Variables
constexpr size_t	default_line_length
	default line length for base64 encoding with lines
constexpr uint64_t	base64_reverse_padding

Detailed Description

helpers placed in namespace detail are not a part of the public API

Enumeration Type Documentation

anonymous enum

anonymous enum

Enumerator
SIMDUTF_VERSION_MAJOR	The major version (MAJOR.minor.revision) of simdutf being used.
SIMDUTF_VERSION_MINOR	The minor version (major.MINOR.revision) of simdutf being used.
SIMDUTF_VERSION_REVISION	The revision (major.minor.REVISION) of simdutf being used.

Definition at line 10 of file simdutf_version.h.

base64_options

enum simdutf::base64_options : uint64_t

Definition at line 4100 of file implementation.h.

encoding_type

enum simdutf::encoding_type

Definition at line 15 of file encoding_types.h.

endianness

enum simdutf::endianness

Definition at line 30 of file encoding_types.h.

error_code

enum simdutf::error_code

Definition at line 7 of file error.h.

last_chunk_handling_options

enum simdutf::last_chunk_handling_options : uint64_t

Definition at line 4125 of file implementation.h.

Function Documentation

autodetect_encoding() [1/2]

simdutf_warn_unused simdutf::encoding_type simdutf::autodetect_encoding ( const char *  input, size_t  length  )

noexcept

Autodetect the encoding of the input, a single encoding is recommended.

E.g., the function might return simdutf::encoding_type::UTF8, simdutf::encoding_type::UTF16_LE, simdutf::encoding_type::UTF16_BE, or simdutf::encoding_type::UTF32_LE.

Parameters

input	the string to analyze.
length	the length of the string in bytes.

Returns

the detected encoding type

autodetect_encoding() [2/2]

simdutf_really_inline simdutf_warn_unused simdutf::encoding_type simdutf::autodetect_encoding ( const uint8_t *  input, size_t  length  )

noexcept

Definition at line 204 of file implementation.h.

base64_ignorable() [1/2]

simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool simdutf::base64_ignorable ( char  input, base64_options  options = base64_default  )

noexcept

Check if a character is an ignorable base64 character.

Checking a large input, character by character, is not computationally efficient.

Parameters

input	the character to check
options	the base64 options to use, is base64_default by default.

Returns

true if the character is an ignorable base64 character, false otherwise.

Definition at line 4862 of file implementation.h.

base64_ignorable() [2/2]

simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool simdutf::base64_ignorable ( char16_t  input, base64_options  options = base64_default  )

noexcept

Definition at line 4866 of file implementation.h.

base64_length_from_binary()

simdutf_warn_unused simdutf_constexpr23 size_t simdutf::base64_length_from_binary ( size_t  length, base64_options  options = base64_default  )

inlinenoexcept

Provide the base64 length in bytes given the length of a binary input.

Parameters

length	the length of the input in bytes
options	the base64 options to use (default: base64_default)

Returns

number of base64 bytes

Definition at line 4446 of file implementation.h.

base64_length_from_binary_with_lines()

simdutf_warn_unused simdutf_constexpr23 size_t simdutf::base64_length_from_binary_with_lines ( size_t  length, base64_options  options = base64_default, size_t  line_length = default_line_length  )

inlinenoexcept

Provide the base64 length in bytes given the length of a binary input, taking into account line breaks.

Parameters

length	the length of the input in bytes
options	the base64 options to use (default: base64_default)
line_length	the length of lines, must be at least 4 (otherwise it is interpreted as 4),

Returns

number of base64 bytes

Definition at line 4462 of file implementation.h.

base64_to_binary() [1/2]

simdutf_warn_unused result simdutf::base64_to_binary ( const char *  input, size_t  length, char *  output, base64_options  options = base64_default, last_chunk_handling_options  last_chunk_options = loose  )

noexcept

Convert a base64 input to a binary output.

This function follows the WHATWG forgiving-base64 format, which means that it will ignore any ASCII spaces in the input. You may provide a padded input (with one or two equal signs at the end) or an unpadded input (without any equal signs at the end).

See https://infra.spec.whatwg.org/#forgiving-base64-decode

This function will fail in case of invalid input. When last_chunk_options = loose, there are two possible reasons for failure: the input contains a number of base64 characters that when divided by 4, leaves a single remainder character (BASE64_INPUT_REMAINDER), or the input contains a character that is not a valid base64 character (INVALID_BASE64_CHARACTER).

When the error is INVALID_BASE64_CHARACTER, r.count contains the index in the input where the invalid character was found. When the error is BASE64_INPUT_REMAINDER, then r.count contains the number of bytes decoded.

The default option (simdutf::base64_default) expects the characters + and / as part of its alphabet. The URL option (simdutf::base64_url) expects the characters - and _ as part of its alphabet.

The padding (=) is validated if present. There may be at most two padding characters at the end of the input. If there are any padding characters, the total number of characters (excluding spaces but including padding characters) must be divisible by four.

You should call this function with a buffer that is at least maximal_binary_length_from_base64(input, length) bytes long. If you fail to provide that much space, the function may cause a buffer overflow.

Advanced users may want to tailor how the last chunk is handled. By default, we use a loose (forgiving) approach but we also support a strict approach as well as a stop_before_partial approach, as per the following proposal:

https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64

Parameters

input	the base64 string to process
length	the length of the string in bytes
output	the pointer to a buffer that can hold the conversion result (should be at least maximal_binary_length_from_base64(input, length) bytes long).
options	the base64 options to use, usually base64_default or base64_url, and base64_default by default.
last_chunk_options	the last chunk handling options, last_chunk_handling_options::loose by default but can also be last_chunk_handling_options::strict or last_chunk_handling_options::stop_before_partial.

Returns

a result pair struct (of type simdutf::result containing the two fields error and count) with an error code and either position of the error (in the input in bytes) if any, or the number of bytes written if successful.

base64_to_binary() [2/2]

simdutf_warn_unused result simdutf::base64_to_binary ( const char16_t *  input, size_t  length, char *  output, base64_options  options = base64_default, last_chunk_handling_options  last_chunk_options = last_chunk_handling_options::loose  )

noexcept

Convert a base64 input to a binary output.

This function follows the WHATWG forgiving-base64 format, which means that it will ignore any ASCII spaces in the input. You may provide a padded input (with one or two equal signs at the end) or an unpadded input (without any equal signs at the end).

See https://infra.spec.whatwg.org/#forgiving-base64-decode

This function will fail in case of invalid input. When last_chunk_options = loose, there are two possible reasons for failure: the input contains a number of base64 characters that when divided by 4, leaves a single remainder character (BASE64_INPUT_REMAINDER), or the input contains a character that is not a valid base64 character (INVALID_BASE64_CHARACTER).

When the error is INVALID_BASE64_CHARACTER, r.count contains the index in the input where the invalid character was found. When the error is BASE64_INPUT_REMAINDER, then r.count contains the number of bytes decoded.

The default option (simdutf::base64_default) expects the characters + and / as part of its alphabet. The URL option (simdutf::base64_url) expects the characters - and _ as part of its alphabet.

The padding (=) is validated if present. There may be at most two padding characters at the end of the input. If there are any padding characters, the total number of characters (excluding spaces but including padding characters) must be divisible by four.

You should call this function with a buffer that is at least maximal_binary_length_from_base64(input, length) bytes long. If you fail to provide that much space, the function may cause a buffer overflow.

Advanced users may want to tailor how the last chunk is handled. By default, we use a loose (forgiving) approach but we also support a strict approach as well as a stop_before_partial approach, as per the following proposal:

https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64

Parameters

input	the base64 string to process, in ASCII stored as 16-bit units
length	the length of the string in 16-bit units
output	the pointer to a buffer that can hold the conversion result (should be at least maximal_binary_length_from_base64(input, length) bytes long).
options	the base64 options to use, can be base64_default or base64_url, is base64_default by default.
last_chunk_options	the last chunk handling options, last_chunk_handling_options::loose by default but can also be last_chunk_handling_options::strict or last_chunk_handling_options::stop_before_partial.

Returns

a result pair struct (of type simdutf::result containing the two fields error and count) with an error code and position of the INVALID_BASE64_CHARACTER error (in the input in units) if any, or the number of bytes written if successful.

base64_to_binary_details() [1/2]

simdutf_warn_unused full_result simdutf::base64_to_binary_details ( const char *  input, size_t  length, char *  output, base64_options  options = base64_default, last_chunk_handling_options  last_chunk_options = last_chunk_handling_options::loose  )

noexcept

Convert a base64 input to a binary output while returning more details than base64_to_binary.

This function follows the WHATWG forgiving-base64 format, which means that it will ignore any ASCII spaces in the input. You may provide a padded input (with one or two equal signs at the end) or an unpadded input (without any equal signs at the end).

See https://infra.spec.whatwg.org/#forgiving-base64-decode

Unlike base64_to_binary, this function returns a full_result with both input_count and output_count, so you always know how much input was consumed and how much output was written. There are three cases where the input may not be fully consumed:

1. stop_before_partial: When last_chunk_options is set to stop_before_partial, any incomplete 4-character group at the end of the input is left unconsumed. This is useful for streaming/chunked decoding where you can carry over the unconsumed input to the next chunk.
2. INVALID_BASE64_CHARACTER: The input contains a character that is not a valid base64 character. In this case, input_count indicates where the invalid character was found.
3. BASE64_INPUT_REMAINDER: When last_chunk_options is loose, the input contains a number of base64 characters that, when divided by 4, leaves a single remainder character (which cannot encode any bytes).

You should call this function with a buffer that is at least maximal_binary_length_from_base64(input, length) bytes long. If you fail to provide that much space, the function may cause a buffer overflow.

Parameters

input	the base64 string to process
length	the length of the string in bytes
output	the pointer to a buffer that can hold the conversion result (should be at least maximal_binary_length_from_base64(input, length) bytes long).
options	the base64 options to use, can be base64_default or base64_url, is base64_default by default.
last_chunk_options	the last chunk handling options, last_chunk_handling_options::loose by default but can also be last_chunk_handling_options::strict or last_chunk_handling_options::stop_before_partial.

Returns

a full_result struct (of type simdutf::full_result containing the three fields error, input_count and output_count).

base64_to_binary_details() [2/2]

simdutf_warn_unused full_result simdutf::base64_to_binary_details ( const char16_t *  input, size_t  length, char *  output, base64_options  options = base64_default, last_chunk_handling_options  last_chunk_options = last_chunk_handling_options::loose  )

noexcept

Convert a base64 input to a binary output while returning more details than base64_to_binary.

This function follows the WHATWG forgiving-base64 format, which means that it will ignore any ASCII spaces in the input. You may provide a padded input (with one or two equal signs at the end) or an unpadded input (without any equal signs at the end).

See https://infra.spec.whatwg.org/#forgiving-base64-decode

Unlike base64_to_binary, this function returns a full_result with both input_count and output_count, so you always know how much input was consumed and how much output was written. There are three cases where the input may not be fully consumed:

1. stop_before_partial: When last_chunk_options is set to stop_before_partial, any incomplete 4-character group at the end of the input is left unconsumed. This is useful for streaming/chunked decoding where you can carry over the unconsumed input to the next chunk.
2. INVALID_BASE64_CHARACTER: The input contains a character that is not a valid base64 character. In this case, input_count indicates where the invalid character was found.
3. BASE64_INPUT_REMAINDER: When last_chunk_options is loose, the input contains a number of base64 characters that, when divided by 4, leaves a single remainder character (which cannot encode any bytes).

You should call this function with a buffer that is at least maximal_binary_length_from_base64(input, length) bytes long. If you fail to provide that much space, the function may cause a buffer overflow.

Parameters

input	the base64 string to process, in ASCII stored as 16-bit units
length	the length of the string in 16-bit units
output	the pointer to a buffer that can hold the conversion result (should be at least maximal_binary_length_from_base64(input, length) bytes long).
options	the base64 options to use, can be base64_default or base64_url, is base64_default by default.
last_chunk_options	the last chunk handling options, last_chunk_handling_options::loose by default but can also be last_chunk_handling_options::strict or last_chunk_handling_options::stop_before_partial.

Returns

a full_result struct (of type simdutf::full_result containing the three fields error, input_count and output_count).

base64_to_binary_safe()

simdutf_warn_unused result simdutf::base64_to_binary_safe ( const char *  input, size_t  length, char *  output, size_t &  outlen, base64_options  options = base64_default, last_chunk_handling_options  last_chunk_options = last_chunk_handling_options::loose, bool  decode_up_to_bad_char = false  )

noexcept

Convert a base64 input to a binary output.

This function follows the WHATWG forgiving-base64 format, which means that it will ignore any ASCII spaces in the input. You may provide a padded input (with one or two equal signs at the end) or an unpadded input (without any equal signs at the end).

See https://infra.spec.whatwg.org/#forgiving-base64-decode

This function will fail in case of invalid input. When last_chunk_options = loose, there are three possible reasons for failure: the input contains a number of base64 characters that when divided by 4, leaves a single remainder character (BASE64_INPUT_REMAINDER), the input contains a character that is not a valid base64 character (INVALID_BASE64_CHARACTER), or the output buffer is too small (OUTPUT_BUFFER_TOO_SMALL).

When OUTPUT_BUFFER_TOO_SMALL, we return both the number of bytes written and the number of units processed, see description of the parameters and returned value.

When the error is INVALID_BASE64_CHARACTER, r.count contains the index in the input where the invalid character was found. When the error is BASE64_INPUT_REMAINDER, then r.count contains the number of bytes decoded.

The default option (simdutf::base64_default) expects the characters + and / as part of its alphabet. The URL option (simdutf::base64_url) expects the characters - and _ as part of its alphabet.

The padding (=) is validated if present. There may be at most two padding characters at the end of the input. If there are any padding characters, the total number of characters (excluding spaces but including padding characters) must be divisible by four.

The INVALID_BASE64_CHARACTER cases are considered fatal and you are expected to discard the output unless the parameter decode_up_to_bad_char is set to true. In that case, the function will decode up to the first invalid character. Extra padding characters ('=') are considered invalid characters.

Advanced users may want to tailor how the last chunk is handled. By default, we use a loose (forgiving) approach but we also support a strict approach as well as a stop_before_partial approach, as per the following proposal:

https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64

Parameters

input	the base64 string to process, in ASCII stored as 8-bit or 16-bit units
length	the length of the string in 8-bit or 16-bit units.
output	the pointer to a buffer that can hold the conversion result.
outlen	the number of bytes that can be written in the output buffer. Upon return, it is modified to reflect how many bytes were written.
options	the base64 options to use, can be base64_default or base64_url, is base64_default by default.
last_chunk_options	the last chunk handling options, last_chunk_handling_options::loose by default but can also be last_chunk_handling_options::strict or last_chunk_handling_options::stop_before_partial.
decode_up_to_bad_char	if true, the function will decode up to the first invalid character. By default (false), it is assumed that the output buffer is to be discarded. When there are multiple errors in the input, using decode_up_to_bad_char might trigger a different error.

Returns

a result pair struct (of type simdutf::result containing the two fields error and count) with an error code and position of the INVALID_BASE64_CHARACTER error (in the input in units) if any, or the number of units processed if successful.

base64_to_binary_safe_impl()

template<typename chartype >

simdutf_warn_unused simdutf_constexpr23 result simdutf::base64_to_binary_safe_impl ( const chartype *  input, size_t  length, char *  output, size_t &  outlen, base64_options  options, last_chunk_handling_options  last_chunk_handling_options, bool  decode_up_to_bad_char  )

noexcept

Definition at line 59 of file base64_implementation.h.

base64_valid() [1/2]

simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool simdutf::base64_valid ( char  input, base64_options  options = base64_default  )

noexcept

Check if a character is a valid base64 character.

Checking a large input, character by character, is not computationally efficient. Note that padding characters are not considered valid base64 characters in this context, nor are spaces.

Parameters

input	the character to check
options	the base64 options to use, is base64_default by default.

Returns

true if the character is a base64 character, false otherwise.

Definition at line 4883 of file implementation.h.

base64_valid() [2/2]

simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool simdutf::base64_valid ( char16_t  input, base64_options  options = base64_default  )

noexcept

Definition at line 4887 of file implementation.h.

base64_valid_or_padding() [1/2]

simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool simdutf::base64_valid_or_padding ( char  input, base64_options  options = base64_default  )

noexcept

Check if a character is a valid base64 character or the padding character ('=').

Checking a large input, character by character, is not computationally efficient.

Parameters

input	the character to check
options	the base64 options to use, is base64_default by default.

Returns

true if the character is a base64 character, false otherwise.

Definition at line 4901 of file implementation.h.

base64_valid_or_padding() [2/2]

simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 bool simdutf::base64_valid_or_padding ( char16_t  input, base64_options  options = base64_default  )

noexcept

Definition at line 4906 of file implementation.h.

binary_length_from_base64() [1/2]

simdutf_warn_unused size_t simdutf::binary_length_from_base64 ( const char *  input, size_t  length  )

noexcept

Compute the binary length from a base64 input.

This function is useful for base64 inputs that may contain ASCII whitespaces (such as line breaks). For such inputs, the result is exact, and for any inputs the result can be used to size the output buffer passed to base64_to_binary.

The function ignores whitespace and does not require padding characters ('=').

Parameters

input	the base64 input to process
length	the length of the base64 input in bytes

Returns

number of binary bytes

binary_length_from_base64() [2/2]

simdutf_warn_unused size_t simdutf::binary_length_from_base64 ( const char16_t *  input, size_t  length  )

noexcept

Compute the binary length from a base64 input.

This function is useful for base64 inputs that may contain ASCII whitespaces (such as line breaks). For such inputs, the result is exact, and for any inputs the result can be used to size the output buffer passed to base64_to_binary.

The function ignores whitespace and does not require padding characters ('=').

Parameters

input	the base64 input to process, in ASCII stored as 16-bit units
length	the length of the base64 input in 16-bit units

Returns

number of binary bytes

binary_to_base64()

size_t simdutf::binary_to_base64 ( const char *  input, size_t  length, char *  output, base64_options  options = base64_default  )

noexcept

Convert a binary input to a base64 output.

The default option (simdutf::base64_default) uses the characters + and / as part of its alphabet. Further, it adds padding (=) at the end of the output to ensure that the output length is a multiple of four.

The URL option (simdutf::base64_url) uses the characters - and _ as part of its alphabet. No padding is added at the end of the output.

This function always succeeds.

Parameters

input	the binary to process
length	the length of the input in bytes
output	the pointer to a buffer that can hold the conversion result (should be at least base64_length_from_binary(length) bytes long)
options	the base64 options to use, can be base64_default or base64_url, is base64_default by default.

Returns

number of written bytes, will be equal to base64_length_from_binary(length, options)

binary_to_base64_with_lines()

size_t simdutf::binary_to_base64_with_lines ( const char *  input, size_t  length, char *  output, size_t  line_length = simdutf::default_line_length, base64_options  options = base64_default  )

noexcept

Convert a binary input to a base64 output with line breaks.

The default option (simdutf::base64_default) uses the characters + and / as part of its alphabet. Further, it adds padding (=) at the end of the output to ensure that the output length is a multiple of four.

The URL option (simdutf::base64_url) uses the characters - and _ as part of its alphabet. No padding is added at the end of the output.

This function always succeeds.

Parameters

input	the binary to process
length	the length of the input in bytes
output	the pointer to a buffer that can hold the conversion result (should be at least base64_length_from_binary_with_lines(length, options, line_length) bytes long)
line_length	the length of lines, must be at least 4 (otherwise it is interpreted as 4),
options	the base64 options to use, can be base64_default or base64_url, is base64_default by default.

Returns

number of written bytes, will be equal to base64_length_from_binary_with_lines(length, options)

change_endianness_utf16()

void simdutf::change_endianness_utf16 ( const char16_t *  input, size_t  length, char16_t *  output  )

noexcept

Change the endianness of the input.

Can be used to go from UTF-16LE to UTF-16BE or from UTF-16BE to UTF-16LE.

This function does not validate the input.

This function is not BOM-aware.

Parameters

input	the UTF-16 string to process
length	the length of the string in 2-byte code units (char16_t)
output	the pointer to a buffer that can hold the conversion result

convert_latin1_to_utf16()

simdutf_warn_unused size_t simdutf::convert_latin1_to_utf16 ( const char *  input, size_t  length, char16_t *  utf16_output  )

noexcept

Using native endianness, convert a Latin1 string into a UTF-16 string.

Parameters

input	the Latin1 string to convert
length	the length of the string in bytes
utf16_output	the pointer to buffer that can hold conversion result

Returns

the number of written char16_t.

convert_latin1_to_utf16be()

simdutf_warn_unused size_t simdutf::convert_latin1_to_utf16be ( const char *  input, size_t  length, char16_t *  utf16_output  )

noexcept

Convert Latin1 string into UTF-16BE string.

This function is suitable to work with inputs from untrusted sources.

Parameters

input	the Latin1 string to convert
length	the length of the string in bytes
utf16_output	the pointer to buffer that can hold conversion result

Returns

the number of written char16_t; 0 if conversion is not possible

convert_latin1_to_utf16le()

simdutf_warn_unused size_t simdutf::convert_latin1_to_utf16le ( const char *  input, size_t  length, char16_t *  utf16_output  )

noexcept

Convert possibly Latin1 string into UTF-16LE string.

This function is suitable to work with inputs from untrusted sources.

Parameters

input	the Latin1 string to convert
length	the length of the string in bytes
utf16_output	the pointer to buffer that can hold conversion result

Returns

the number of written char16_t; 0 if conversion is not possible

convert_latin1_to_utf32()

simdutf_warn_unused size_t simdutf::convert_latin1_to_utf32 ( const char *  input, size_t  length, char32_t *  utf32_buffer  )

noexcept

Convert Latin1 string into UTF-32 string.

This function is suitable to work with inputs from untrusted sources.

Parameters

input	the Latin1 string to convert
length	the length of the string in bytes
utf32_buffer	the pointer to buffer that can hold conversion result

Returns

the number of written char32_t; 0 if conversion is not possible

convert_latin1_to_utf8()

simdutf_warn_unused size_t simdutf::convert_latin1_to_utf8 ( const char *  input, size_t  length, char *  utf8_output  )

noexcept

Convert Latin1 string into UTF-8 string.

This function is suitable to work with inputs from untrusted sources.

Parameters

input	the Latin1 string to convert
length	the length of the string in bytes
utf8_output	the pointer to buffer that can hold conversion result

Returns

the number of written char; 0 if conversion is not possible

convert_latin1_to_utf8_safe()

simdutf_warn_unused size_t simdutf::convert_latin1_to_utf8_safe ( const char *  input, size_t  length, char *  utf8_output, size_t  utf8_len  )

noexcept

Convert Latin1 string into UTF-8 string with output limit.

This function is suitable to work with inputs from untrusted sources.

We write as many characters as possible.

Parameters

input	the Latin1 string to convert
length	the length of the string in bytes
utf8_output	the pointer to buffer that can hold conversion result
utf8_len	the maximum output length

Returns

the number of written char; 0 if conversion is not possible

convert_utf16_to_latin1()

simdutf_warn_unused size_t simdutf::convert_utf16_to_latin1 ( const char16_t *  input, size_t  length, char *  latin1_buffer  )

noexcept

Using native endianness, convert possibly broken UTF-16 string into Latin1 string.

During the conversion also validation of the input string is done. This function is suitable to work with inputs from untrusted sources.

This function is not BOM-aware.

Parameters

input	the UTF-16 string to convert
length	the length of the string in 2-byte code units (char16_t)
latin1_buffer	the pointer to buffer that can hold conversion result

Returns

number of written code units; 0 if input is not a valid UTF-16 string or if it cannot be represented as Latin1

convert_utf16_to_latin1_with_errors()

simdutf_warn_unused result simdutf::convert_utf16_to_latin1_with_errors ( const char16_t *  input, size_t  length, char *  latin1_buffer  )

noexcept

Using native endianness, convert possibly broken UTF-16 string into Latin1 string.

During the conversion also validation of the input string is done. This function is suitable to work with inputs from untrusted sources. This function is not BOM-aware.

Parameters

input	the UTF-16 string to convert
length	the length of the string in 2-byte code units (char16_t)
latin1_buffer	the pointer to buffer that can hold conversion result

Returns

a result pair struct (of type simdutf::result containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char written if successful.

convert_utf16_to_utf32()

simdutf_warn_unused size_t simdutf::convert_utf16_to_utf32 ( const char16_t *  input, size_t  length, char32_t *  utf32_buffer  )

noexcept

Using native endianness, convert possibly broken UTF-16 string into UTF-32 string.

During the conversion also validation of the input string is done. This function is suitable to work with inputs from untrusted sources.

This function is not BOM-aware.

Parameters

input	the UTF-16 string to convert
length	the length of the string in 2-byte code units (char16_t)
utf32_buffer	the pointer to buffer that can hold conversion result

Returns

number of written code units; 0 if input is not a valid UTF-16LE string

convert_utf16_to_utf32_with_errors()

simdutf_warn_unused result simdutf::convert_utf16_to_utf32_with_errors ( const char16_t *  input, size_t  length, char32_t *  utf32_buffer  )

noexcept

Using native endianness, convert possibly broken UTF-16 string into UTF-32 string and stop on error.

During the conversion also validation of the input string is done. This function is suitable to work with inputs from untrusted sources.

This function is not BOM-aware.

Parameters

input	the UTF-16 string to convert
length	the length of the string in 2-byte code units (char16_t)
utf32_buffer	the pointer to buffer that can hold conversion result

Returns

a result pair struct (of type simdutf::result containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char32_t written if successful.

convert_utf16_to_utf8()

simdutf_warn_unused size_t simdutf::convert_utf16_to_utf8 ( const char16_t *  input, size_t  length, char *  utf8_buffer  )

noexcept

Using native endianness, convert possibly broken UTF-16 string into UTF-8 string.

During the conversion also validation of the input string is done. This function is suitable to work with inputs from untrusted sources.

This function is not BOM-aware.

Parameters

input	the UTF-16 string to convert
length	the length of the string in 2-byte code units (char16_t)
utf8_buffer	the pointer to buffer that can hold conversion result

Returns

number of written code units; 0 if input is not a valid UTF-16LE string

convert_utf16_to_utf8_safe()

simdutf_warn_unused size_t simdutf::convert_utf16_to_utf8_safe ( const char16_t *  input, size_t  length, char *  utf8_output, size_t  utf8_len  )

noexcept

Using native endianness, convert possibly broken UTF-16 string into UTF-8 string with output limit.

We write as many characters as possible into the output buffer,

During the conversion also validation of the input string is done. This function is suitable to work with inputs from untrusted sources.

This function is not BOM-aware.

Parameters

input	the UTF-16 string to convert
length	the length of the string in 16-bit code units (char16_t)
utf8_output	the pointer to buffer that can hold conversion result
utf8_len	the maximum output length

Returns

the number of written char; 0 if conversion is not possible

convert_utf16_to_utf8_with_errors()

simdutf_warn_unused result simdutf::convert_utf16_to_utf8_with_errors ( const char16_t *  input, size_t  length, char *  utf8_buffer  )

noexcept

Using native endianness, convert possibly broken UTF-16 string into UTF-8 string and stop on error.

During the conversion also validation of the input string is done. This function is suitable to work with inputs from untrusted sources.

This function is not BOM-aware.

Parameters

input	the UTF-16 string to convert
length	the length of the string in 2-byte code units (char16_t)
utf8_buffer	the pointer to buffer that can hold conversion result

Returns

a result pair struct (of type simdutf::result containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char written if successful.

convert_utf16_to_utf8_with_replacement()

simdutf_warn_unused size_t simdutf::convert_utf16_to_utf8_with_replacement ( const char16_t *  input, size_t  length, char *  utf8_buffer  )

noexcept

Convert possibly broken UTF-16 string (native endianness) into UTF-8 string, replacing unpaired surrogates with the Unicode replacement character U+FFFD.

This function always succeeds: unpaired surrogates are replaced with U+FFFD (3 bytes in UTF-8: 0xEF 0xBF 0xBD).

This function is not BOM-aware.

Parameters

input	the UTF-16 string to convert
length	the length of the string in 2-byte code units (char16_t)
utf8_buffer	the pointer to buffer that can hold conversion result

Returns

number of written code units

convert_utf16be_to_latin1()

simdutf_warn_unused size_t simdutf::convert_utf16be_to_latin1 ( const char16_t *  input, size_t  length, char *  latin1_buffer  )

noexcept

Convert possibly broken UTF-16BE string into Latin1 string.

During the conversion also validation of the input string is done. This function is suitable to work with inputs from untrusted sources.

This function is not BOM-aware.

Parameters

input	the UTF-16BE string to convert
length	the length of the string in 2-byte code units (char16_t)
latin1_buffer	the pointer to buffer that can hold conversion result

Returns

number of written code units; 0 if input is not a valid UTF-16BE string or if it cannot be represented as Latin1

convert_utf16be_to_latin1_with_errors()

simdutf_warn_unused result simdutf::convert_utf16be_to_latin1_with_errors ( const char16_t *  input, size_t  length, char *  latin1_buffer  )

noexcept

Convert possibly broken UTF-16BE string into Latin1 string.

If the string cannot be represented as Latin1, an error is returned.

During the conversion also validation of the input string is done. This function is suitable to work with inputs from untrusted sources. This function is not BOM-aware.

Parameters

input	the UTF-16BE string to convert
length	the length of the string in 2-byte code units (char16_t)
latin1_buffer	the pointer to buffer that can hold conversion result

Returns

a result pair struct (of type simdutf::result containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char written if successful.

convert_utf16be_to_utf32()

simdutf_warn_unused size_t simdutf::convert_utf16be_to_utf32 ( const char16_t *  input, size_t  length, char32_t *  utf32_buffer  )

noexcept

Convert possibly broken UTF-16BE string into UTF-32 string.

During the conversion also validation of the input string is done. This function is suitable to work with inputs from untrusted sources.

This function is not BOM-aware.

Parameters

input	the UTF-16BE string to convert
length	the length of the string in 2-byte code units (char16_t)
utf32_buffer	the pointer to buffer that can hold conversion result

Returns

number of written code units; 0 if input is not a valid UTF-16LE string

convert_utf16be_to_utf32_with_errors()

simdutf_warn_unused result simdutf::convert_utf16be_to_utf32_with_errors ( const char16_t *  input, size_t  length, char32_t *  utf32_buffer  )

noexcept

Convert possibly broken UTF-16BE string into UTF-32 string and stop on error.

During the conversion also validation of the input string is done. This function is suitable to work with inputs from untrusted sources.

This function is not BOM-aware.

Parameters

input	the UTF-16BE string to convert
length	the length of the string in 2-byte code units (char16_t)
utf32_buffer	the pointer to buffer that can hold conversion result

Returns

a result pair struct (of type simdutf::result containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char32_t written if successful.

convert_utf16be_to_utf8()

simdutf_warn_unused size_t simdutf::convert_utf16be_to_utf8 ( const char16_t *  input, size_t  length, char *  utf8_buffer  )

noexcept

Convert possibly broken UTF-16BE string into UTF-8 string.

During the conversion also validation of the input string is done. This function is suitable to work with inputs from untrusted sources.

This function is not BOM-aware.

Parameters

input	the UTF-16BE string to convert
length	the length of the string in 2-byte code units (char16_t)
utf8_buffer	the pointer to buffer that can hold conversion result

Returns

number of written code units; 0 if input is not a valid UTF-16LE string

convert_utf16be_to_utf8_with_errors()

simdutf_warn_unused result simdutf::convert_utf16be_to_utf8_with_errors ( const char16_t *  input, size_t  length, char *  utf8_buffer  )

noexcept

Convert possibly broken UTF-16BE string into UTF-8 string and stop on error.

During the conversion also validation of the input string is done. This function is suitable to work with inputs from untrusted sources.

This function is not BOM-aware.

Parameters

input	the UTF-16BE string to convert
length	the length of the string in 2-byte code units (char16_t)
utf8_buffer	the pointer to buffer that can hold conversion result

Returns

a result pair struct (of type simdutf::result containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char written if successful.

convert_utf16be_to_utf8_with_replacement()

simdutf_warn_unused size_t simdutf::convert_utf16be_to_utf8_with_replacement ( const char16_t *  input, size_t  length, char *  utf8_buffer  )

noexcept

Convert possibly broken UTF-16BE string into UTF-8 string, replacing unpaired surrogates with the Unicode replacement character U+FFFD.

This function always succeeds: unpaired surrogates are replaced with U+FFFD (3 bytes in UTF-8: 0xEF 0xBF 0xBD).

This function is not BOM-aware.

Parameters

input	the UTF-16BE string to convert
length	the length of the string in 2-byte code units (char16_t)
utf8_buffer	the pointer to buffer that can hold conversion result

Returns

number of written code units

convert_utf16le_to_latin1()

simdutf_warn_unused size_t simdutf::convert_utf16le_to_latin1 ( const char16_t *  input, size_t  length, char *  latin1_buffer  )

noexcept

Convert possibly broken UTF-16LE string into Latin1 string.

If the string cannot be represented as Latin1, an error is returned.

During the conversion also validation of the input string is done. This function is suitable to work with inputs from untrusted sources.

This function is not BOM-aware.

Parameters

input	the UTF-16LE string to convert
length	the length of the string in 2-byte code units (char16_t)
latin1_buffer	the pointer to buffer that can hold conversion result

Returns

number of written code units; 0 if input is not a valid UTF-16LE string or if it cannot be represented as Latin1

convert_utf16le_to_latin1_with_errors()

simdutf_warn_unused result simdutf::convert_utf16le_to_latin1_with_errors ( const char16_t *  input, size_t  length, char *  latin1_buffer  )

noexcept

Convert possibly broken UTF-16LE string into Latin1 string.

During the conversion also validation of the input string is done. This function is suitable to work with inputs from untrusted sources. This function is not BOM-aware.

Parameters

input	the UTF-16LE string to convert
length	the length of the string in 2-byte code units (char16_t)
latin1_buffer	the pointer to buffer that can hold conversion result

Returns

a result pair struct (of type simdutf::result containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char written if successful.

convert_utf16le_to_utf32()

simdutf_warn_unused size_t simdutf::convert_utf16le_to_utf32 ( const char16_t *  input, size_t  length, char32_t *  utf32_buffer  )

noexcept

Convert possibly broken UTF-16LE string into UTF-32 string.

During the conversion also validation of the input string is done. This function is suitable to work with inputs from untrusted sources.

This function is not BOM-aware.

Parameters

input	the UTF-16LE string to convert
length	the length of the string in 2-byte code units (char16_t)
utf32_buffer	the pointer to buffer that can hold conversion result

Returns

number of written code units; 0 if input is not a valid UTF-16LE string

convert_utf16le_to_utf32_with_errors()

simdutf_warn_unused result simdutf::convert_utf16le_to_utf32_with_errors ( const char16_t *  input, size_t  length, char32_t *  utf32_buffer  )

noexcept

Convert possibly broken UTF-16LE string into UTF-32 string and stop on error.

During the conversion also validation of the input string is done. This function is suitable to work with inputs from untrusted sources.

This function is not BOM-aware.

Parameters

input	the UTF-16LE string to convert
length	the length of the string in 2-byte code units (char16_t)
utf32_buffer	the pointer to buffer that can hold conversion result

Returns

a result pair struct (of type simdutf::result containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char32_t written if successful.

convert_utf16le_to_utf8()

simdutf_warn_unused size_t simdutf::convert_utf16le_to_utf8 ( const char16_t *  input, size_t  length, char *  utf8_buffer  )

noexcept

Convert possibly broken UTF-16LE string into UTF-8 string.

During the conversion also validation of the input string is done. This function is suitable to work with inputs from untrusted sources.

This function is not BOM-aware.

Parameters

input	the UTF-16LE string to convert
length	the length of the string in 2-byte code units (char16_t)
utf8_buffer	the pointer to buffer that can hold conversion result

Returns

number of written code units; 0 if input is not a valid UTF-16LE string

convert_utf16le_to_utf8_with_errors()

simdutf_warn_unused result simdutf::convert_utf16le_to_utf8_with_errors ( const char16_t *  input, size_t  length, char *  utf8_buffer  )

noexcept

Convert possibly broken UTF-16LE string into UTF-8 string and stop on error.

During the conversion also validation of the input string is done. This function is suitable to work with inputs from untrusted sources.

This function is not BOM-aware.

Parameters

input	the UTF-16LE string to convert
length	the length of the string in 2-byte code units (char16_t)
utf8_buffer	the pointer to buffer that can hold conversion result

Returns

a result pair struct (of type simdutf::result containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char written if successful.

convert_utf16le_to_utf8_with_replacement()

simdutf_warn_unused size_t simdutf::convert_utf16le_to_utf8_with_replacement ( const char16_t *  input, size_t  length, char *  utf8_buffer  )

noexcept

Convert possibly broken UTF-16LE string into UTF-8 string, replacing unpaired surrogates with the Unicode replacement character U+FFFD.

This function always succeeds: unpaired surrogates are replaced with U+FFFD (3 bytes in UTF-8: 0xEF 0xBF 0xBD).

This function is not BOM-aware.

Parameters

input	the UTF-16LE string to convert
length	the length of the string in 2-byte code units (char16_t)
utf8_buffer	the pointer to buffer that can hold conversion result

Returns

number of written code units

convert_utf32_to_latin1()

simdutf_warn_unused size_t simdutf::convert_utf32_to_latin1 ( const char32_t *  input, size_t  length, char *  latin1_buffer  )

noexcept

Convert possibly broken UTF-32 string into Latin1 string.

During the conversion also validation of the input string is done. This function is suitable to work with inputs from untrusted sources.

This function is not BOM-aware.

Parameters

input	the UTF-32 string to convert
length	the length of the string in 4-byte code units (char32_t)
latin1_buffer	the pointer to buffer that can hold conversion result

Returns

number of written code units; 0 if input is not a valid UTF-32 string or if it cannot be represented as Latin1

convert_utf32_to_latin1_with_errors()

simdutf_warn_unused result simdutf::convert_utf32_to_latin1_with_errors ( const char32_t *  input, size_t  length, char *  latin1_buffer  )

noexcept

Convert possibly broken UTF-32 string into Latin1 string and stop on error.

If the string cannot be represented as Latin1, an error is returned.

During the conversion also validation of the input string is done. This function is suitable to work with inputs from untrusted sources.

This function is not BOM-aware.

Parameters

input	the UTF-32 string to convert
length	the length of the string in 4-byte code units (char32_t)
latin1_buffer	the pointer to buffer that can hold conversion result

Returns

a result pair struct (of type simdutf::result containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char written if successful.

convert_utf32_to_utf16()

simdutf_warn_unused size_t simdutf::convert_utf32_to_utf16 ( const char32_t *  input, size_t  length, char16_t *  utf16_buffer  )

noexcept

Using native endianness, convert possibly broken UTF-32 string into a UTF-16 string.

During the conversion also validation of the input string is done. This function is suitable to work with inputs from untrusted sources.

This function is not BOM-aware.

Parameters

input	the UTF-32 string to convert
length	the length of the string in 4-byte code units (char32_t)
utf16_buffer	the pointer to buffer that can hold conversion result

Returns

number of written code units; 0 if input is not a valid UTF-32 string

convert_utf32_to_utf16_with_errors()

simdutf_warn_unused result simdutf::convert_utf32_to_utf16_with_errors ( const char32_t *  input, size_t  length, char16_t *  utf16_buffer  )

noexcept

Using native endianness, convert possibly broken UTF-32 string into UTF-16 string and stop on error.

During the conversion also validation of the input string is done. This function is suitable to work with inputs from untrusted sources.

This function is not BOM-aware.

Parameters

input	the UTF-32 string to convert
length	the length of the string in 4-byte code units (char32_t)
utf16_buffer	the pointer to buffer that can hold conversion result

Returns

a result pair struct (of type simdutf::result containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char16_t written if successful.

convert_utf32_to_utf16be()

simdutf_warn_unused size_t simdutf::convert_utf32_to_utf16be ( const char32_t *  input, size_t  length, char16_t *  utf16_buffer  )

noexcept

Convert possibly broken UTF-32 string into UTF-16BE string.

During the conversion also validation of the input string is done. This function is suitable to work with inputs from untrusted sources.

This function is not BOM-aware.

Parameters

input	the UTF-32 string to convert
length	the length of the string in 4-byte code units (char32_t)
utf16_buffer	the pointer to buffer that can hold conversion result

Returns

number of written code units; 0 if input is not a valid UTF-32 string

convert_utf32_to_utf16be_with_errors()

simdutf_warn_unused result simdutf::convert_utf32_to_utf16be_with_errors ( const char32_t *  input, size_t  length, char16_t *  utf16_buffer  )

noexcept

Convert possibly broken UTF-32 string into UTF-16BE string and stop on error.

During the conversion also validation of the input string is done. This function is suitable to work with inputs from untrusted sources.

This function is not BOM-aware.

Parameters

input	the UTF-32 string to convert
length	the length of the string in 4-byte code units (char32_t)
utf16_buffer	the pointer to buffer that can hold conversion result

Returns

a result pair struct (of type simdutf::result containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char16_t written if successful.

convert_utf32_to_utf16le()

simdutf_warn_unused size_t simdutf::convert_utf32_to_utf16le ( const char32_t *  input, size_t  length, char16_t *  utf16_buffer  )

noexcept

Convert possibly broken UTF-32 string into UTF-16LE string.

During the conversion also validation of the input string is done. This function is suitable to work with inputs from untrusted sources.

This function is not BOM-aware.

Parameters

input	the UTF-32 string to convert
length	the length of the string in 4-byte code units (char32_t)
utf16_buffer	the pointer to buffer that can hold conversion result

Returns

number of written code units; 0 if input is not a valid UTF-32 string

convert_utf32_to_utf16le_with_errors()

simdutf_warn_unused result simdutf::convert_utf32_to_utf16le_with_errors ( const char32_t *  input, size_t  length, char16_t *  utf16_buffer  )

noexcept

Convert possibly broken UTF-32 string into UTF-16LE string and stop on error.

During the conversion also validation of the input string is done. This function is suitable to work with inputs from untrusted sources.

This function is not BOM-aware.

Parameters

input	the UTF-32 string to convert
length	the length of the string in 4-byte code units (char32_t)
utf16_buffer	the pointer to buffer that can hold conversion result

Returns

a result pair struct (of type simdutf::result containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char16_t written if successful.

convert_utf32_to_utf8()

simdutf_warn_unused size_t simdutf::convert_utf32_to_utf8 ( const char32_t *  input, size_t  length, char *  utf8_buffer  )

noexcept

Convert possibly broken UTF-32 string into UTF-8 string.

During the conversion also validation of the input string is done. This function is suitable to work with inputs from untrusted sources.

This function is not BOM-aware.

Parameters

input	the UTF-32 string to convert
length	the length of the string in 4-byte code units (char32_t)
utf8_buffer	the pointer to buffer that can hold conversion result

Returns

number of written code units; 0 if input is not a valid UTF-32 string

convert_utf32_to_utf8_with_errors()

simdutf_warn_unused result simdutf::convert_utf32_to_utf8_with_errors ( const char32_t *  input, size_t  length, char *  utf8_buffer  )

noexcept

Convert possibly broken UTF-32 string into UTF-8 string and stop on error.

During the conversion also validation of the input string is done. This function is suitable to work with inputs from untrusted sources.

This function is not BOM-aware.

Parameters

input	the UTF-32 string to convert
length	the length of the string in 4-byte code units (char32_t)
utf8_buffer	the pointer to buffer that can hold conversion result

Returns

a result pair struct (of type simdutf::result containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char written if successful.

convert_utf8_to_latin1()

simdutf_warn_unused size_t simdutf::convert_utf8_to_latin1 ( const char *  input, size_t  length, char *  latin1_output  )

noexcept

Convert possibly broken UTF-8 string into latin1 string.

During the conversion also validation of the input string is done. This function is suitable to work with inputs from untrusted sources.

Parameters

input	the UTF-8 string to convert
length	the length of the string in bytes
latin1_output	the pointer to buffer that can hold conversion result

Returns

the number of written char; 0 if the input was not valid UTF-8 string or if it cannot be represented as Latin1

convert_utf8_to_latin1_with_errors()

simdutf_warn_unused result simdutf::convert_utf8_to_latin1_with_errors ( const char *  input, size_t  length, char *  latin1_output  )

noexcept

Convert possibly broken UTF-8 string into latin1 string with errors.

If the string cannot be represented as Latin1, an error code is returned.

During the conversion also validation of the input string is done. This function is suitable to work with inputs from untrusted sources.

Parameters

input	the UTF-8 string to convert
length	the length of the string in bytes
latin1_output	the pointer to buffer that can hold conversion result

Returns

a result pair struct (of type simdutf::result containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of code units validated if successful.

convert_utf8_to_utf16()

simdutf_warn_unused size_t simdutf::convert_utf8_to_utf16 ( const char *  input, size_t  length, char16_t *  utf16_output  )

noexcept

Using native endianness, convert possibly broken UTF-8 string into a UTF-16 string.

During the conversion also validation of the input string is done. This function is suitable to work with inputs from untrusted sources.

Parameters

input	the UTF-8 string to convert
length	the length of the string in bytes
utf16_output	the pointer to buffer that can hold conversion result

Returns

the number of written char16_t; 0 if the input was not valid UTF-8 string

convert_utf8_to_utf16_with_errors()

simdutf_warn_unused result simdutf::convert_utf8_to_utf16_with_errors ( const char *  input, size_t  length, char16_t *  utf16_output  )

noexcept

Using native endianness, convert possibly broken UTF-8 string into UTF-16 string and stop on error.

During the conversion also validation of the input string is done. This function is suitable to work with inputs from untrusted sources.

Parameters

input	the UTF-8 string to convert
length	the length of the string in bytes
utf16_output	the pointer to buffer that can hold conversion result

Returns

a result pair struct (of type simdutf::result containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char16_t written if successful.

convert_utf8_to_utf16be()

simdutf_warn_unused size_t simdutf::convert_utf8_to_utf16be ( const char *  input, size_t  length, char16_t *  utf16_output  )

noexcept

Convert possibly broken UTF-8 string into UTF-16BE string.

During the conversion also validation of the input string is done. This function is suitable to work with inputs from untrusted sources.

Parameters

input	the UTF-8 string to convert
length	the length of the string in bytes
utf16_output	the pointer to buffer that can hold conversion result

Returns

the number of written char16_t; 0 if the input was not valid UTF-8 string

convert_utf8_to_utf16be_with_errors()

simdutf_warn_unused result simdutf::convert_utf8_to_utf16be_with_errors ( const char *  input, size_t  length, char16_t *  utf16_output  )

noexcept

Convert possibly broken UTF-8 string into UTF-16BE string and stop on error.

During the conversion also validation of the input string is done. This function is suitable to work with inputs from untrusted sources.

Parameters

input	the UTF-8 string to convert
length	the length of the string in bytes
utf16_output	the pointer to buffer that can hold conversion result

Returns

a result pair struct (of type simdutf::result containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char16_t written if successful.

convert_utf8_to_utf16le()

simdutf_warn_unused size_t simdutf::convert_utf8_to_utf16le ( const char *  input, size_t  length, char16_t *  utf16_output  )

noexcept

Convert possibly broken UTF-8 string into UTF-16LE string.

During the conversion also validation of the input string is done. This function is suitable to work with inputs from untrusted sources.

Parameters

input	the UTF-8 string to convert
length	the length of the string in bytes
utf16_output	the pointer to buffer that can hold conversion result

Returns

the number of written char16_t; 0 if the input was not valid UTF-8 string

convert_utf8_to_utf16le_with_errors()

simdutf_warn_unused result simdutf::convert_utf8_to_utf16le_with_errors ( const char *  input, size_t  length, char16_t *  utf16_output  )

noexcept

Convert possibly broken UTF-8 string into UTF-16LE string and stop on error.

During the conversion also validation of the input string is done. This function is suitable to work with inputs from untrusted sources.

Parameters

input	the UTF-8 string to convert
length	the length of the string in bytes
utf16_output	the pointer to buffer that can hold conversion result

Returns

a result pair struct (of type simdutf::result containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char16_t written if successful.

convert_utf8_to_utf32()

simdutf_warn_unused size_t simdutf::convert_utf8_to_utf32 ( const char *  input, size_t  length, char32_t *  utf32_output  )

noexcept

Convert possibly broken UTF-8 string into UTF-32 string.

During the conversion also validation of the input string is done. This function is suitable to work with inputs from untrusted sources.

Parameters

input	the UTF-8 string to convert
length	the length of the string in bytes
utf32_output	the pointer to buffer that can hold conversion result

Returns

the number of written char32_t; 0 if the input was not valid UTF-8 string

convert_utf8_to_utf32_with_errors()

simdutf_warn_unused result simdutf::convert_utf8_to_utf32_with_errors ( const char *  input, size_t  length, char32_t *  utf32_output  )

noexcept

Convert possibly broken UTF-8 string into UTF-32 string and stop on error.

During the conversion also validation of the input string is done. This function is suitable to work with inputs from untrusted sources.

Parameters

input	the UTF-8 string to convert
length	the length of the string in bytes
utf32_output	the pointer to buffer that can hold conversion result

Returns

a result pair struct (of type simdutf::result containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of char32_t written if successful.

convert_valid_utf16_to_latin1()

simdutf_warn_unused size_t simdutf::convert_valid_utf16_to_latin1 ( const char16_t *  input, size_t  length, char *  latin1_buffer  )

noexcept

Using native endianness, convert UTF-16 string into Latin1 string.

This function assumes that the input string is valid UTF-16 and that it can be represented as Latin1. If you violate this assumption, the result is implementation defined and may include system-dependent behavior such as crashes.

This function is for expert users only and not part of our public API. Use convert_utf16_to_latin1 instead. The function may be removed from the library in the future.

This function is not BOM-aware.

Parameters

input	the UTF-16 string to convert
length	the length of the string in 2-byte code units (char16_t)
latin1_buffer	the pointer to buffer that can hold conversion result

Returns

number of written code units; 0 if conversion is not possible

convert_valid_utf16_to_utf32()

simdutf_warn_unused size_t simdutf::convert_valid_utf16_to_utf32 ( const char16_t *  input, size_t  length, char32_t *  utf32_buffer  )

noexcept

Using native endianness, convert valid UTF-16 string into UTF-32 string.

This function assumes that the input string is valid UTF-16 (native endianness).

This function is not BOM-aware.

Parameters

input	the UTF-16 string to convert
length	the length of the string in 2-byte code units (char16_t)
utf32_buffer	the pointer to a buffer that can hold the conversion result

Returns

number of written code units; 0 if conversion is not possible

convert_valid_utf16_to_utf8()

simdutf_warn_unused size_t simdutf::convert_valid_utf16_to_utf8 ( const char16_t *  input, size_t  length, char *  utf8_buffer  )

noexcept

Using native endianness, convert valid UTF-16 string into UTF-8 string.

This function assumes that the input string is valid UTF-16.

This function is not BOM-aware.

Parameters

input	the UTF-16 string to convert
length	the length of the string in 2-byte code units (char16_t)
utf8_buffer	the pointer to a buffer that can hold the conversion result

Returns

number of written code units; 0 if conversion is not possible

convert_valid_utf16be_to_latin1()

simdutf_warn_unused size_t simdutf::convert_valid_utf16be_to_latin1 ( const char16_t *  input, size_t  length, char *  latin1_buffer  )

noexcept

Convert valid UTF-16BE string into Latin1 string.

This function assumes that the input string is valid UTF-16BE and that it can be represented as Latin1. If you violate this assumption, the result is implementation defined and may include system-dependent behavior such as crashes.

This function is for expert users only and not part of our public API. Use convert_utf16be_to_latin1 instead. The function may be removed from the library in the future.

This function is not BOM-aware.

Parameters

input	the UTF-16BE string to convert
length	the length of the string in 2-byte code units (char16_t)
latin1_buffer	the pointer to buffer that can hold conversion result

Returns

number of written code units; 0 if conversion is not possible

convert_valid_utf16be_to_utf32()

simdutf_warn_unused size_t simdutf::convert_valid_utf16be_to_utf32 ( const char16_t *  input, size_t  length, char32_t *  utf32_buffer  )

noexcept

Convert valid UTF-16BE string into UTF-32 string.

This function assumes that the input string is valid UTF-16LE.

This function is not BOM-aware.

Parameters

input	the UTF-16BE string to convert
length	the length of the string in 2-byte code units (char16_t)
utf32_buffer	the pointer to a buffer that can hold the conversion result

Returns

number of written code units; 0 if conversion is not possible

convert_valid_utf16be_to_utf8()

simdutf_warn_unused size_t simdutf::convert_valid_utf16be_to_utf8 ( const char16_t *  input, size_t  length, char *  utf8_buffer  )

noexcept

Convert valid UTF-16BE string into UTF-8 string.

This function assumes that the input string is valid UTF-16BE.

This function is not BOM-aware.

Parameters

input	the UTF-16BE string to convert
length	the length of the string in 2-byte code units (char16_t)
utf8_buffer	the pointer to a buffer that can hold the conversion result

Returns

number of written code units; 0 if conversion is not possible

convert_valid_utf16le_to_latin1()

simdutf_warn_unused size_t simdutf::convert_valid_utf16le_to_latin1 ( const char16_t *  input, size_t  length, char *  latin1_buffer  )

noexcept

Convert valid UTF-16LE string into Latin1 string.

This function assumes that the input string is valid UTF-16LE and that it can be represented as Latin1. If you violate this assumption, the result is implementation defined and may include system-dependent behavior such as crashes.

This function is for expert users only and not part of our public API. Use convert_utf16le_to_latin1 instead. The function may be removed from the library in the future.

This function is not BOM-aware.

Parameters

input	the UTF-16LE string to convert
length	the length of the string in 2-byte code units (char16_t)
latin1_buffer	the pointer to buffer that can hold conversion result

Returns

number of written code units; 0 if conversion is not possible

convert_valid_utf16le_to_utf32()

simdutf_warn_unused size_t simdutf::convert_valid_utf16le_to_utf32 ( const char16_t *  input, size_t  length, char32_t *  utf32_buffer  )

noexcept

Convert valid UTF-16LE string into UTF-32 string.

This function assumes that the input string is valid UTF-16LE.

This function is not BOM-aware.

Parameters

input	the UTF-16LE string to convert
length	the length of the string in 2-byte code units (char16_t)
utf32_buffer	the pointer to a buffer that can hold the conversion result

Returns

number of written code units; 0 if conversion is not possible

convert_valid_utf16le_to_utf8()

simdutf_warn_unused size_t simdutf::convert_valid_utf16le_to_utf8 ( const char16_t *  input, size_t  length, char *  utf8_buffer  )

noexcept

Convert valid UTF-16LE string into UTF-8 string.

This function assumes that the input string is valid UTF-16LE

This function is not BOM-aware.

Parameters

input	the UTF-16LE string to convert
length	the length of the string in 2-byte code units (char16_t)
utf8_buffer	the pointer to a buffer that can hold the conversion result

Returns

number of written code units; 0 if conversion is not possible

convert_valid_utf32_to_latin1()

simdutf_warn_unused size_t simdutf::convert_valid_utf32_to_latin1 ( const char32_t *  input, size_t  length, char *  latin1_buffer  )

noexcept

Convert valid UTF-32 string into Latin1 string.

This function assumes that the input string is valid UTF-32 and that it can be represented as Latin1. If you violate this assumption, the result is implementation defined and may include system-dependent behavior such as crashes.

This function is for expert users only and not part of our public API. Use convert_utf32_to_latin1 instead. The function may be removed from the library in the future.

This function is not BOM-aware.

Parameters

input	the UTF-32 string to convert
length	the length of the string in 4-byte code units (char32_t)
latin1_buffer	the pointer to a buffer that can hold the conversion result

Returns

number of written code units; 0 if conversion is not possible

convert_valid_utf32_to_utf16()

simdutf_warn_unused size_t simdutf::convert_valid_utf32_to_utf16 ( const char32_t *  input, size_t  length, char16_t *  utf16_buffer  )

noexcept

Using native endianness, convert valid UTF-32 string into a UTF-16 string.

This function assumes that the input string is valid UTF-32.

This function is not BOM-aware.

Parameters

input	the UTF-32 string to convert
length	the length of the string in 4-byte code units (char32_t)
utf16_buffer	the pointer to a buffer that can hold the conversion result

Returns

number of written code units; 0 if conversion is not possible

convert_valid_utf32_to_utf16be()

simdutf_warn_unused size_t simdutf::convert_valid_utf32_to_utf16be ( const char32_t *  input, size_t  length, char16_t *  utf16_buffer  )

noexcept

Convert valid UTF-32 string into UTF-16BE string.

This function assumes that the input string is valid UTF-32.

This function is not BOM-aware.

Parameters

input	the UTF-32 string to convert
length	the length of the string in 4-byte code units (char32_t)
utf16_buffer	the pointer to a buffer that can hold the conversion result

Returns

number of written code units; 0 if conversion is not possible

convert_valid_utf32_to_utf16le()

simdutf_warn_unused size_t simdutf::convert_valid_utf32_to_utf16le ( const char32_t *  input, size_t  length, char16_t *  utf16_buffer  )

noexcept

Convert valid UTF-32 string into UTF-16LE string.

This function assumes that the input string is valid UTF-32.

This function is not BOM-aware.

Parameters

input	the UTF-32 string to convert
length	the length of the string in 4-byte code units (char32_t)
utf16_buffer	the pointer to a buffer that can hold the conversion result

Returns

number of written code units; 0 if conversion is not possible

convert_valid_utf32_to_utf8()

simdutf_warn_unused size_t simdutf::convert_valid_utf32_to_utf8 ( const char32_t *  input, size_t  length, char *  utf8_buffer  )

noexcept

Convert valid UTF-32 string into UTF-8 string.

This function assumes that the input string is valid UTF-32.

This function is not BOM-aware.

Parameters

input	the UTF-32 string to convert
length	the length of the string in 4-byte code units (char32_t)
utf8_buffer	the pointer to a buffer that can hold the conversion result

Returns

number of written code units; 0 if conversion is not possible

convert_valid_utf8_to_latin1()

simdutf_warn_unused size_t simdutf::convert_valid_utf8_to_latin1 ( const char *  input, size_t  length, char *  latin1_output  )

noexcept

Convert valid UTF-8 string into latin1 string.

This function assumes that the input string is valid UTF-8 and that it can be represented as Latin1. If you violate this assumption, the result is implementation defined and may include system-dependent behavior such as crashes.

This function is for expert users only and not part of our public API. Use convert_utf8_to_latin1 instead. The function may be removed from the library in the future.

This function is not BOM-aware.

Parameters

input	the UTF-8 string to convert
length	the length of the string in bytes
latin1_output	the pointer to buffer that can hold conversion result

Returns

the number of written char; 0 if the input was not valid UTF-8 string

convert_valid_utf8_to_utf16()

simdutf_warn_unused size_t simdutf::convert_valid_utf8_to_utf16 ( const char *  input, size_t  length, char16_t *  utf16_buffer  )

noexcept

Using native endianness, convert valid UTF-8 string into a UTF-16 string.

This function assumes that the input string is valid UTF-8.

Parameters

input	the UTF-8 string to convert
length	the length of the string in bytes
utf16_buffer	the pointer to buffer that can hold conversion result

Returns

the number of written char16_t

convert_valid_utf8_to_utf16be()

simdutf_warn_unused size_t simdutf::convert_valid_utf8_to_utf16be ( const char *  input, size_t  length, char16_t *  utf16_buffer  )

noexcept

Convert valid UTF-8 string into UTF-16BE string.

This function assumes that the input string is valid UTF-8.

Parameters

input	the UTF-8 string to convert
length	the length of the string in bytes
utf16_buffer	the pointer to buffer that can hold conversion result

Returns

the number of written char16_t

convert_valid_utf8_to_utf16le()

simdutf_warn_unused size_t simdutf::convert_valid_utf8_to_utf16le ( const char *  input, size_t  length, char16_t *  utf16_buffer  )

noexcept

Convert valid UTF-8 string into UTF-16LE string.

This function assumes that the input string is valid UTF-8.

Parameters

input	the UTF-8 string to convert
length	the length of the string in bytes
utf16_buffer	the pointer to buffer that can hold conversion result

Returns

the number of written char16_t

convert_valid_utf8_to_utf32()

simdutf_warn_unused size_t simdutf::convert_valid_utf8_to_utf32 ( const char *  input, size_t  length, char32_t *  utf32_buffer  )

noexcept

Convert valid UTF-8 string into UTF-32 string.

This function assumes that the input string is valid UTF-8.

Parameters

input	the UTF-8 string to convert
length	the length of the string in bytes
utf32_buffer	the pointer to buffer that can hold conversion result

Returns

the number of written char32_t

count_utf16()

simdutf_warn_unused size_t simdutf::count_utf16 ( const char16_t *  input, size_t  length  )

noexcept

Count the number of code points (characters) in the string assuming that it is valid.

This function assumes that the input string is valid UTF-16 (native endianness). It is acceptable to pass invalid UTF-16 strings but in such cases the result is implementation defined.

This function is not BOM-aware.

Parameters

input	the UTF-16 string to process
length	the length of the string in 2-byte code units (char16_t)

Returns

number of code points

count_utf16be()

simdutf_warn_unused size_t simdutf::count_utf16be ( const char16_t *  input, size_t  length  )

noexcept

Count the number of code points (characters) in the string assuming that it is valid.

This function assumes that the input string is valid UTF-16BE. It is acceptable to pass invalid UTF-16 strings but in such cases the result is implementation defined.

This function is not BOM-aware.

Parameters

input	the UTF-16BE string to process
length	the length of the string in 2-byte code units (char16_t)

Returns

number of code points

count_utf16le()

simdutf_warn_unused size_t simdutf::count_utf16le ( const char16_t *  input, size_t  length  )

noexcept

Count the number of code points (characters) in the string assuming that it is valid.

This function assumes that the input string is valid UTF-16LE. It is acceptable to pass invalid UTF-16 strings but in such cases the result is implementation defined.

This function is not BOM-aware.

Parameters

input	the UTF-16LE string to process
length	the length of the string in 2-byte code units (char16_t)

Returns

number of code points

count_utf8()

simdutf_warn_unused size_t simdutf::count_utf8 ( const char *  input, size_t  length  )

noexcept

Count the number of code points (characters) in the string assuming that it is valid.

This function assumes that the input string is valid UTF-8. It is acceptable to pass invalid UTF-8 strings but in such cases the result is implementation defined.

Parameters

input	the UTF-8 string to process
length	the length of the string in bytes

Returns

number of code points

detect_encodings() [1/2]

simdutf_warn_unused int simdutf::detect_encodings ( const char *  input, size_t  length  )

noexcept

Autodetect the possible encodings of the input in one pass.

E.g., if the input might be UTF-16LE or UTF-8, this function returns the value (simdutf::encoding_type::UTF8 | simdutf::encoding_type::UTF16_LE).

Overridden by each implementation.

Parameters

input	the string to analyze.
length	the length of the string in bytes.

Returns

the detected encoding type

detect_encodings() [2/2]

simdutf_really_inline simdutf_warn_unused int simdutf::detect_encodings ( const uint8_t *  input, size_t  length  )

noexcept

Definition at line 241 of file implementation.h.

error_to_string()

std::string_view simdutf::error_to_string ( error_code  code )

inlinenoexcept

Definition at line 47 of file error.h.

find() [1/2]

simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 const char * simdutf::find ( const char *  start, const char *  end, char  character  )

noexcept

Find the first occurrence of a character in a string.

If the character is not found, return a pointer to the end of the string.

Parameters

start	the start of the string
end	the end of the string
character	the character to find

Returns

a pointer to the first occurrence of the character in the string, or a pointer to the end of the string if the character is not found.

Definition at line 4158 of file implementation.h.

find() [2/2]

simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 const char16_t * simdutf::find ( const char16_t *  start, const char16_t *  end, char16_t  character  )

noexcept

Definition at line 4172 of file implementation.h.

get_active_implementation()

SIMDUTF_DLLIMPORTEXPORT internal::atomic_ptr< const implementation > & simdutf::get_active_implementation ( )

extern

The active implementation.

Automatically initialized on first use to the most advanced implementation supported by this hardware.

is_partial()

simdutf_constexpr23 bool simdutf::is_partial ( last_chunk_handling_options  options )

inline

Definition at line 4136 of file implementation.h.

latin1_length_from_utf16()

simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t simdutf::latin1_length_from_utf16 ( size_t  length )

noexcept

Compute the number of bytes that this UTF-16 string would require in Latin1 format.

Parameters

length

the length of the string in Latin1 code units (char)

Returns

the length of the string in Latin1 code units (char) required to encode the UTF-16 string as Latin1

Definition at line 966 of file implementation.h.

latin1_length_from_utf32()

simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 size_t simdutf::latin1_length_from_utf32 ( size_t  length )

noexcept

Compute the number of bytes that this UTF-32 string would require in Latin1 format.

This function does not validate the input. It is acceptable to pass invalid UTF-32 strings but in such cases the result is implementation defined.

This function is not BOM-aware.

Parameters

length

the length of the string in 4-byte code units (char32_t)

Returns

the number of bytes required to encode the UTF-32 string as Latin1

Definition at line 3367 of file implementation.h.

latin1_length_from_utf8()

simdutf_warn_unused size_t simdutf::latin1_length_from_utf8 ( const char *  input, size_t  length  )

noexcept

Compute the number of bytes that this UTF-8 string would require in Latin1 format.

This function does not validate the input. It is acceptable to pass invalid UTF-8 strings but in such cases the result is implementation defined.

This function is not BOM-aware.

Parameters

input	the UTF-8 string to convert
length	the length of the string in byte

Returns

the number of bytes required to encode the UTF-8 string as Latin1

match_system()

simdutf_warn_unused simdutf_really_inline constexpr bool simdutf::match_system ( endianness  e )

constexpr

Definition at line 42 of file encoding_types.h.

maximal_binary_length_from_base64() [1/2]

simdutf_warn_unused size_t simdutf::maximal_binary_length_from_base64 ( const char *  input, size_t  length  )

noexcept

Provide the maximal binary length in bytes given the base64 input.

As long as the input does not contain ignorable characters (e.g., ASCII spaces or linefeed characters), the result is exact. In particular, the function checks for padding characters.

The function is fast (constant time). It checks up to two characters at the end of the string. The input is not otherwise validated or read.

Parameters

input	the base64 input to process
length	the length of the base64 input in bytes

Returns

maximum number of binary bytes

maximal_binary_length_from_base64() [2/2]

simdutf_warn_unused size_t simdutf::maximal_binary_length_from_base64 ( const char16_t *  input, size_t  length  )

noexcept

Provide the maximal binary length in bytes given the base64 input.

As long as the input does not contain ignorable characters (e.g., ASCII spaces or linefeed characters), the result is exact. In particular, the function checks for padding characters.

The function is fast (constant time). It checks up to two characters at the end of the string. The input is not otherwise validated or read.

Parameters

input	the base64 input to process, in ASCII stored as 16-bit units
length	the length of the base64 input in 16-bit units

Returns

maximal number of binary bytes

slow_base64_to_binary_safe_impl()

template<typename chartype >

simdutf_warn_unused simdutf_constexpr23 result simdutf::slow_base64_to_binary_safe_impl ( const chartype *  input, size_t  length, char *  output, size_t &  outlen, base64_options  options, last_chunk_handling_options  last_chunk_options  )

noexcept

Definition at line 11 of file base64_implementation.h.

to_string() [1/2]

std::string_view simdutf::to_string ( base64_options  options )

inline

Definition at line 4194 of file implementation.h.

to_string() [2/2]

std::string_view simdutf::to_string ( last_chunk_handling_options  options )

inline

Definition at line 4216 of file implementation.h.

to_well_formed_utf16()

void simdutf::to_well_formed_utf16 ( const char16_t *  input, size_t  len, char16_t *  output  )

noexcept

Fixes an ill-formed UTF-16 string by replacing mismatched surrogates with the Unicode replacement character U+FFFD.

If input and output points to different memory areas, the procedure copies string, and it's expected that output memory is at least as big as the input. It's also possible to set input equal output, that makes replacements an in-place operation.

Parameters

input	the UTF-16 string to correct.
len	the length of the string in number of 2-byte code units (char16_t).
output	the output buffer.

to_well_formed_utf16be()

void simdutf::to_well_formed_utf16be ( const char16_t *  input, size_t  len, char16_t *  output  )

noexcept

Fixes an ill-formed UTF-16BE string by replacing mismatched surrogates with the Unicode replacement character U+FFFD.

If input and output points to different memory areas, the procedure copies string, and it's expected that output memory is at least as big as the input. It's also possible to set input equal output, that makes replacements an in-place operation.

Parameters

input	the UTF-16BE string to correct.
len	the length of the string in number of 2-byte code units (char16_t).
output	the output buffer.

to_well_formed_utf16le()

void simdutf::to_well_formed_utf16le ( const char16_t *  input, size_t  len, char16_t *  output  )

noexcept

Fixes an ill-formed UTF-16LE string by replacing mismatched surrogates with the Unicode replacement character U+FFFD.

If input and output points to different memory areas, the procedure copies string, and it's expected that output memory is at least as big as the input. It's also possible to set input equal output, that makes replacements an in-place operation.

Parameters

input	the UTF-16LE string to correct.
len	the length of the string in number of 2-byte code units (char16_t).
output	the output buffer.

trim_partial_utf16()

simdutf_warn_unused size_t simdutf::trim_partial_utf16	(	const char16_t *	input,
		size_t	length
	)

Given a valid UTF-16 string having a possibly truncated last character, this function checks the end of string.

If the last character is truncated (or partial), then it returns a shorter length (shorter by 1 unit) so that the short UTF-16 strings only contain complete characters. If there is no truncated character, the original length is returned.

This function assumes that the input string is valid UTF-16, but possibly truncated. We use the native endianness.

Parameters

input	the UTF-16 string to process
length	the length of the string in bytes

Returns

the length of the string in unit, possibly shorter by 1 unit

trim_partial_utf16be()

simdutf_warn_unused size_t simdutf::trim_partial_utf16be	(	const char16_t *	input,
		size_t	length
	)

Given a valid UTF-16BE string having a possibly truncated last character, this function checks the end of string.

If the last character is truncated (or partial), then it returns a shorter length (shorter by 1 unit) so that the short UTF-16BE strings only contain complete characters. If there is no truncated character, the original length is returned.

This function assumes that the input string is valid UTF-16BE, but possibly truncated.

Parameters

input	the UTF-16BE string to process
length	the length of the string in bytes

Returns

the length of the string in bytes, possibly shorter by 1 unit

trim_partial_utf16le()

simdutf_warn_unused size_t simdutf::trim_partial_utf16le	(	const char16_t *	input,
		size_t	length
	)

Given a valid UTF-16LE string having a possibly truncated last character, this function checks the end of string.

If the last character is truncated (or partial), then it returns a shorter length (shorter by 1 unit) so that the short UTF-16LE strings only contain complete characters. If there is no truncated character, the original length is returned.

This function assumes that the input string is valid UTF-16LE, but possibly truncated.

Parameters

input	the UTF-16LE string to process
length	the length of the string in bytes

Returns

the length of the string in unit, possibly shorter by 1 unit

trim_partial_utf8()

simdutf_warn_unused size_t simdutf::trim_partial_utf8	(	const char *	input,
		size_t	length
	)

Given a valid UTF-8 string having a possibly truncated last character, this function checks the end of string.

If the last character is truncated (or partial), then it returns a shorter length (shorter by 1 to 3 bytes) so that the short UTF-8 strings only contain complete characters. If there is no truncated character, the original length is returned.

This function assumes that the input string is valid UTF-8, but possibly truncated.

Parameters

input	the UTF-8 string to process
length	the length of the string in bytes

Returns

the length of the string in bytes, possibly shorter by 1 to 3 bytes

utf16_length_from_latin1()

simdutf_really_inline simdutf_warn_unused simdutf_constexpr23 size_t simdutf::utf16_length_from_latin1 ( size_t  length )

noexcept

Compute the number of code units that this Latin1 string would require in UTF-16 format.

Parameters

length

the length of the string in Latin1 code units (char)

Returns

the length of the string in 2-byte code units (char16_t) required to encode the Latin1 string as UTF-16

Definition at line 979 of file implementation.h.

utf16_length_from_utf32()

simdutf_warn_unused size_t simdutf::utf16_length_from_utf32 ( const char32_t *  input, size_t  length  )

noexcept

Compute the number of two-byte code units that this UTF-32 string would require in UTF-16 format.

This function does not validate the input. It is acceptable to pass invalid UTF-32 strings but in such cases the result is implementation defined.

Parameters

input	the UTF-32 string to convert
length	the length of the string in 4-byte code units (char32_t)

Returns

the number of bytes required to encode the UTF-32 string as UTF-16

utf16_length_from_utf8()

simdutf_warn_unused size_t simdutf::utf16_length_from_utf8 ( const char *  input, size_t  length  )

noexcept

Compute the number of 2-byte code units that this UTF-8 string would require in UTF-16LE format.

This function does not validate the input. It is acceptable to pass invalid UTF-8 strings but in such cases the result is implementation defined.

This function is not BOM-aware.

Parameters

input	the UTF-8 string to process
length	the length of the string in bytes

Returns

the number of char16_t code units required to encode the UTF-8 string as UTF-16LE

utf32_length_from_latin1()

simdutf_warn_unused simdutf_really_inline simdutf_constexpr23 size_t simdutf::utf32_length_from_latin1 ( size_t  length )

noexcept

Compute the number of bytes that this Latin1 string would require in UTF-32 format.

Parameters

length

the length of the string in Latin1 code units (char)

Returns

the length of the string in 4-byte code units (char32_t) required to encode the Latin1 string as UTF-32

Definition at line 3380 of file implementation.h.

utf32_length_from_utf16()

simdutf_warn_unused size_t simdutf::utf32_length_from_utf16 ( const char16_t *  input, size_t  length  )

noexcept

Using native endianness; Compute the number of bytes that this UTF-16 string would require in UTF-32 format.

This function is equivalent to count_utf16.

This function does not validate the input. It is acceptable to pass invalid UTF-16 strings but in such cases the result is implementation defined.

This function is not BOM-aware.

Parameters

input	the UTF-16 string to convert
length	the length of the string in 2-byte code units (char16_t)

Returns

the number of bytes required to encode the UTF-16LE string as UTF-32

utf32_length_from_utf16be()

simdutf_warn_unused size_t simdutf::utf32_length_from_utf16be ( const char16_t *  input, size_t  length  )

noexcept

Compute the number of bytes that this UTF-16BE string would require in UTF-32 format.

This function is equivalent to count_utf16be.

This function does not validate the input. It is acceptable to pass invalid UTF-16 strings but in such cases the result is implementation defined.

This function is not BOM-aware.

Parameters

input	the UTF-16BE string to convert
length	the length of the string in 2-byte code units (char16_t)

Returns

the number of bytes required to encode the UTF-16BE string as UTF-32

utf32_length_from_utf16le()

simdutf_warn_unused size_t simdutf::utf32_length_from_utf16le ( const char16_t *  input, size_t  length  )

noexcept

Compute the number of bytes that this UTF-16LE string would require in UTF-32 format.

This function is equivalent to count_utf16le.

This function does not validate the input. It is acceptable to pass invalid UTF-16 strings but in such cases the result is implementation defined.

This function is not BOM-aware.

Parameters

input	the UTF-16LE string to convert
length	the length of the string in 2-byte code units (char16_t)

Returns

the number of bytes required to encode the UTF-16LE string as UTF-32

utf32_length_from_utf8()

simdutf_warn_unused size_t simdutf::utf32_length_from_utf8 ( const char *  input, size_t  length  )

noexcept

Compute the number of 4-byte code units that this UTF-8 string would require in UTF-32 format.

This function is equivalent to count_utf8

This function does not validate the input. It is acceptable to pass invalid UTF-8 strings but in such cases the result is implementation defined.

This function is not BOM-aware.

Parameters

input	the UTF-8 string to process
length	the length of the string in bytes

Returns

the number of char32_t code units required to encode the UTF-8 string as UTF-32

utf8_length_from_latin1()

simdutf_warn_unused size_t simdutf::utf8_length_from_latin1 ( const char *  input, size_t  length  )

noexcept

Return the number of bytes that this Latin1 string would require in UTF-8 format.

Parameters

input	the Latin1 string to convert
length	the length of the string bytes

Returns

the number of bytes required to encode the Latin1 string as UTF-8

utf8_length_from_utf16()

simdutf_warn_unused size_t simdutf::utf8_length_from_utf16 ( const char16_t *  input, size_t  length  )

noexcept

Using native endianness; Compute the number of bytes that this UTF-16 string would require in UTF-8 format.

This function does not validate the input. It is acceptable to pass invalid UTF-16 strings but in such cases the result is implementation defined.

Parameters

input	the UTF-16 string to convert
length	the length of the string in 2-byte code units (char16_t)

Returns

the number of bytes required to encode the UTF-16LE string as UTF-8

utf8_length_from_utf16_with_replacement()

simdutf_warn_unused result simdutf::utf8_length_from_utf16_with_replacement ( const char16_t *  input, size_t  length  )

noexcept

Using native endianness; compute the number of bytes that this UTF-16 string would require in UTF-8 format even when the UTF-16LE content contains mismatched surrogates that have to be replaced by the replacement character (0xFFFD).

Parameters

input	the UTF-16 string to convert
length	the length of the string in 2-byte code units (char16_t)

Returns

a result pair struct (of type simdutf::result containing the two fields error and count) where the count is the number of bytes required to encode the UTF-16 string as UTF-8, and the error code is either SUCCESS or SURROGATE. The count is correct regardless of the error field. When SURROGATE is returned, it does not indicate an error in the case of this function: it indicates that at least one surrogate has been encountered: the surrogates may be matched or not (thus this function does not validate). If the returned error code is SUCCESS, then the input contains no surrogate, is in the Basic Multilingual Plane, and is necessarily valid.

utf8_length_from_utf16be()

simdutf_warn_unused size_t simdutf::utf8_length_from_utf16be ( const char16_t *  input, size_t  length  )

noexcept

Compute the number of bytes that this UTF-16BE string would require in UTF-8 format.

This function does not validate the input. It is acceptable to pass invalid UTF-16 strings but in such cases the result is implementation defined.

Parameters

input	the UTF-16BE string to convert
length	the length of the string in 2-byte code units (char16_t)

Returns

the number of bytes required to encode the UTF-16BE string as UTF-8

utf8_length_from_utf16be_with_replacement()

simdutf_warn_unused result simdutf::utf8_length_from_utf16be_with_replacement ( const char16_t *  input, size_t  length  )

noexcept

Compute the number of bytes that this UTF-16BE string would require in UTF-8 format even when the UTF-16BE content contains mismatched surrogates that have to be replaced by the replacement character (0xFFFD).

Parameters

input	the UTF-16BE string to convert
length	the length of the string in 2-byte code units (char16_t)

Returns

a result pair struct (of type simdutf::result containing the two fields error and count) where the count is the number of bytes required to encode the UTF-16BE string as UTF-8, and the error code is either SUCCESS or SURROGATE. The count is correct regardless of the error field. When SURROGATE is returned, it does not indicate an error in the case of this function: it indicates that at least one surrogate has been encountered: the surrogates may be matched or not (thus this function does not validate). If the returned error code is SUCCESS, then the input contains no surrogate, is in the Basic Multilingual Plane, and is necessarily valid.

utf8_length_from_utf16le()

simdutf_warn_unused size_t simdutf::utf8_length_from_utf16le ( const char16_t *  input, size_t  length  )

noexcept

Compute the number of bytes that this UTF-16LE string would require in UTF-8 format.

This function does not validate the input. It is acceptable to pass invalid UTF-16 strings but in such cases the result is implementation defined.

Parameters

input	the UTF-16LE string to convert
length	the length of the string in 2-byte code units (char16_t)

Returns

the number of bytes required to encode the UTF-16LE string as UTF-8

utf8_length_from_utf16le_with_replacement()

simdutf_warn_unused result simdutf::utf8_length_from_utf16le_with_replacement ( const char16_t *  input, size_t  length  )

noexcept

Compute the number of bytes that this UTF-16LE string would require in UTF-8 format even when the UTF-16LE content contains mismatched surrogates that have to be replaced by the replacement character (0xFFFD).

Parameters

input	the UTF-16LE string to convert
length	the length of the string in 2-byte code units (char16_t)

Returns

a result pair struct (of type simdutf::result containing the two fields error and count) where the count is the number of bytes required to encode the UTF-16LE string as UTF-8, and the error code is either SUCCESS or SURROGATE. The count is correct regardless of the error field. When SURROGATE is returned, it does not indicate an error in the case of this function: it indicates that at least one surrogate has been encountered: the surrogates may be matched or not (thus this function does not validate). If the returned error code is SUCCESS, then the input contains no surrogate, is in the Basic Multilingual Plane, and is necessarily valid.

utf8_length_from_utf32()

simdutf_warn_unused size_t simdutf::utf8_length_from_utf32 ( const char32_t *  input, size_t  length  )

noexcept

Compute the number of bytes that this UTF-32 string would require in UTF-8 format.

This function does not validate the input. It is acceptable to pass invalid UTF-32 strings but in such cases the result is implementation defined.

Parameters

input	the UTF-32 string to convert
length	the length of the string in 4-byte code units (char32_t)

Returns

the number of bytes required to encode the UTF-32 string as UTF-8

validate_ascii()

simdutf_warn_unused bool simdutf::validate_ascii ( const char *  buf, size_t  len  )

noexcept

Validate the ASCII string.

Overridden by each implementation.

Parameters

buf	the ASCII string to validate.
len	the length of the string in bytes.

Returns

true if and only if the string is valid ASCII.

validate_ascii_with_errors()

simdutf_warn_unused result simdutf::validate_ascii_with_errors ( const char *  buf, size_t  len  )

noexcept

Validate the ASCII string and stop on error.

It might be faster than validate_utf8 when an error is expected to occur early.

Overridden by each implementation.

Parameters

buf	the ASCII string to validate.
len	the length of the string in bytes.

Returns

a result pair struct (of type simdutf::result containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of code units validated if successful.

validate_utf16()

simdutf_warn_unused bool simdutf::validate_utf16 ( const char16_t *  buf, size_t  len  )

noexcept

Using native endianness; Validate the UTF-16 string.

This function may be best when you expect the input to be almost always valid. Otherwise, consider using validate_utf16_with_errors.

Overridden by each implementation.

This function is not BOM-aware.

Parameters

buf	the UTF-16 string to validate.
len	the length of the string in number of 2-byte code units (char16_t).

Returns

true if and only if the string is valid UTF-16.

validate_utf16_as_ascii()

simdutf_warn_unused bool simdutf::validate_utf16_as_ascii ( const char16_t *  buf, size_t  len  )

noexcept

Validate the ASCII string as a UTF-16 sequence.

An UTF-16 sequence is considered an ASCII sequence if it could be converted to an ASCII string losslessly.

Overridden by each implementation.

Parameters

buf	the UTF-16 string to validate.
len	the length of the string in bytes.

Returns

true if and only if the string is valid ASCII.

validate_utf16_with_errors()

simdutf_warn_unused result simdutf::validate_utf16_with_errors ( const char16_t *  buf, size_t  len  )

noexcept

Using native endianness; Validate the UTF-16 string and stop on error.

It might be faster than validate_utf16 when an error is expected to occur early.

Overridden by each implementation.

This function is not BOM-aware.

Parameters

buf	the UTF-16 string to validate.
len	the length of the string in number of 2-byte code units (char16_t).

Returns

a result pair struct (of type simdutf::result containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of code units validated if successful.

validate_utf16be()

simdutf_warn_unused bool simdutf::validate_utf16be ( const char16_t *  buf, size_t  len  )

noexcept

Validate the UTF-16BE string.

This function may be best when you expect the input to be almost always valid. Otherwise, consider using validate_utf16be_with_errors.

Overridden by each implementation.

This function is not BOM-aware.

Parameters

buf	the UTF-16BE string to validate.
len	the length of the string in number of 2-byte code units (char16_t).

Returns

true if and only if the string is valid UTF-16BE.

validate_utf16be_as_ascii()

simdutf_warn_unused bool simdutf::validate_utf16be_as_ascii ( const char16_t *  buf, size_t  len  )

noexcept

Validate the ASCII string as a UTF-16BE sequence.

An UTF-16 sequence is considered an ASCII sequence if it could be converted to an ASCII string losslessly.

Overridden by each implementation.

Parameters

buf	the UTF-16BE string to validate.
len	the length of the string in bytes.

Returns

true if and only if the string is valid ASCII.

validate_utf16be_with_errors()

simdutf_warn_unused result simdutf::validate_utf16be_with_errors ( const char16_t *  buf, size_t  len  )

noexcept

Validate the UTF-16BE string and stop on error.

It might be faster than validate_utf16be when an error is expected to occur early.

Overridden by each implementation.

This function is not BOM-aware.

Parameters

buf	the UTF-16BE string to validate.
len	the length of the string in number of 2-byte code units (char16_t).

Returns

a result pair struct (of type simdutf::result containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of code units validated if successful.

validate_utf16le()

simdutf_warn_unused bool simdutf::validate_utf16le ( const char16_t *  buf, size_t  len  )

noexcept

Validate the UTF-16LE string.

This function may be best when you expect the input to be almost always valid. Otherwise, consider using validate_utf16le_with_errors.

Overridden by each implementation.

This function is not BOM-aware.

Parameters

buf	the UTF-16LE string to validate.
len	the length of the string in number of 2-byte code units (char16_t).

Returns

true if and only if the string is valid UTF-16LE.

validate_utf16le_as_ascii()

simdutf_warn_unused bool simdutf::validate_utf16le_as_ascii ( const char16_t *  buf, size_t  len  )

noexcept

Validate the ASCII string as a UTF-16LE sequence.

An UTF-16 sequence is considered an ASCII sequence if it could be converted to an ASCII string losslessly.

Overridden by each implementation.

Parameters

buf	the UTF-16LE string to validate.
len	the length of the string in bytes.

Returns

true if and only if the string is valid ASCII.

validate_utf16le_with_errors()

simdutf_warn_unused result simdutf::validate_utf16le_with_errors ( const char16_t *  buf, size_t  len  )

noexcept

Validate the UTF-16LE string and stop on error.

It might be faster than validate_utf16le when an error is expected to occur early.

Overridden by each implementation.

This function is not BOM-aware.

Parameters

buf	the UTF-16LE string to validate.
len	the length of the string in number of 2-byte code units (char16_t).

Returns

a result pair struct (of type simdutf::result containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of code units validated if successful.

validate_utf32()

simdutf_warn_unused bool simdutf::validate_utf32 ( const char32_t *  buf, size_t  len  )

noexcept

Validate the UTF-32 string.

This function may be best when you expect the input to be almost always valid. Otherwise, consider using validate_utf32_with_errors.

Overridden by each implementation.

This function is not BOM-aware.

Parameters

buf	the UTF-32 string to validate.
len	the length of the string in number of 4-byte code units (char32_t).

Returns

true if and only if the string is valid UTF-32.

validate_utf32_with_errors()

simdutf_warn_unused result simdutf::validate_utf32_with_errors ( const char32_t *  buf, size_t  len  )

noexcept

Validate the UTF-32 string and stop on error.

It might be faster than validate_utf32 when an error is expected to occur early.

Overridden by each implementation.

This function is not BOM-aware.

Parameters

buf	the UTF-32 string to validate.
len	the length of the string in number of 4-byte code units (char32_t).

Returns

a result pair struct (of type simdutf::result containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of code units validated if successful.

validate_utf8()

simdutf_warn_unused bool simdutf::validate_utf8 ( const char *  buf, size_t  len  )

noexcept

Validate the UTF-8 string.

This function may be best when you expect the input to be almost always valid. Otherwise, consider using validate_utf8_with_errors.

Overridden by each implementation.

Parameters

buf	the UTF-8 string to validate.
len	the length of the string in bytes.

Returns

true if and only if the string is valid UTF-8.

validate_utf8_with_errors()

simdutf_warn_unused result simdutf::validate_utf8_with_errors ( const char *  buf, size_t  len  )

noexcept

Validate the UTF-8 string and stop on error.

Overridden by each implementation.

Parameters

buf	the UTF-8 string to validate.
len	the length of the string in bytes.

Returns

a result pair struct (of type simdutf::result containing the two fields error and count) with an error code and either position of the error (in the input in code units) if any, or the number of code units validated if successful.

Variable Documentation

base64_reverse_padding

constexpr uint64_t simdutf::base64_reverse_padding

constexpr

Initial value:

=
    2

Definition at line 4098 of file implementation.h.

default_line_length

constexpr size_t simdutf::default_line_length

constexpr

Initial value:

=
    76

default line length for base64 encoding with lines

Definition at line 187 of file implementation.h.