From 120273aa1a63eb5b3411ac9fcc4cd09eaabbc9e1 Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sun, 4 Feb 2024 16:14:11 -0800 Subject: [PATCH] pkg/utfcpp --- build.zig | 8 + build.zig.zon | 1 + pkg/utfcpp/build.zig | 62 +++++ pkg/utfcpp/build.zig.zon | 13 + pkg/utfcpp/empty.cc | 2 + pkg/utfcpp/main.zig | 0 src/simd/utf8.h | 46 ---- src/simd/utf8/checked.h | 359 --------------------------- src/simd/utf8/core.h | 492 -------------------------------------- src/simd/utf8/cpp11.h | 70 ------ src/simd/utf8/cpp17.h | 96 -------- src/simd/utf8/cpp20.h | 124 ---------- src/simd/utf8/unchecked.h | 287 ---------------------- src/simd/vt.cpp | 9 +- 14 files changed, 91 insertions(+), 1478 deletions(-) create mode 100644 pkg/utfcpp/build.zig create mode 100644 pkg/utfcpp/build.zig.zon create mode 100644 pkg/utfcpp/empty.cc create mode 100644 pkg/utfcpp/main.zig delete mode 100644 src/simd/utf8.h delete mode 100644 src/simd/utf8/checked.h delete mode 100644 src/simd/utf8/core.h delete mode 100644 src/simd/utf8/cpp11.h delete mode 100644 src/simd/utf8/cpp17.h delete mode 100644 src/simd/utf8/cpp20.h delete mode 100644 src/simd/utf8/unchecked.h diff --git a/build.zig b/build.zig index 1564bb672..c3cbc9223 100644 --- a/build.zig +++ b/build.zig @@ -955,6 +955,10 @@ fn addDeps( .target = target, .optimize = optimize, }); + const utfcpp_dep = b.dependency("utfcpp", .{ + .target = target, + .optimize = optimize, + }); const libpng_dep = b.dependency("libpng", .{ .target = target, .optimize = optimize, @@ -1073,6 +1077,10 @@ fn addDeps( step.linkLibrary(simdutf_dep.artifact("simdutf")); try static_libs.append(simdutf_dep.artifact("simdutf").getEmittedBin()); + // utfcpp + step.linkLibrary(utfcpp_dep.artifact("utfcpp")); + try static_libs.append(utfcpp_dep.artifact("utfcpp").getEmittedBin()); + // Spirv-Cross step.linkLibrary(spirv_cross_dep.artifact("spirv_cross")); try static_libs.append(spirv_cross_dep.artifact("spirv_cross").getEmittedBin()); diff --git a/build.zig.zon b/build.zig.zon index 73a391eec..a694562ea 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -38,6 +38,7 @@ .opengl = .{ .path = "./pkg/opengl" }, .pixman = .{ .path = "./pkg/pixman" }, .simdutf = .{ .path = "./pkg/simdutf" }, + .utfcpp = .{ .path = "./pkg/utfcpp" }, .zlib = .{ .path = "./pkg/zlib" }, // Shader translation diff --git a/pkg/utfcpp/build.zig b/pkg/utfcpp/build.zig new file mode 100644 index 000000000..3776675a6 --- /dev/null +++ b/pkg/utfcpp/build.zig @@ -0,0 +1,62 @@ +const std = @import("std"); + +pub fn build(b: *std.Build) !void { + const target = b.standardTargetOptions(.{}); + const optimize = b.standardOptimizeOption(.{}); + + const upstream = b.dependency("utfcpp", .{}); + + const module = b.addModule("utfcpp", .{ + .root_source_file = .{ .path = "main.zig" }, + .target = target, + .optimize = optimize, + }); + + const lib = b.addStaticLibrary(.{ + .name = "utfcpp", + .target = target, + .optimize = optimize, + }); + lib.linkLibCpp(); + lib.addIncludePath(upstream.path("")); + module.addIncludePath(upstream.path("")); + + if (target.result.isDarwin()) { + const apple_sdk = @import("apple_sdk"); + try apple_sdk.addPaths(b, &lib.root_module); + try apple_sdk.addPaths(b, module); + } + + var flags = std.ArrayList([]const u8).init(b.allocator); + defer flags.deinit(); + try flags.appendSlice(&.{}); + + lib.addCSourceFiles(.{ + .flags = flags.items, + .files = &.{"empty.cc"}, + }); + lib.installHeadersDirectoryOptions(.{ + .source_dir = upstream.path("source"), + .install_dir = .header, + .install_subdir = "", + .include_extensions = &.{".h"}, + }); + + b.installArtifact(lib); + + { + const test_exe = b.addTest(.{ + .name = "test", + .root_source_file = .{ .path = "main.zig" }, + .target = target, + .optimize = optimize, + }); + test_exe.linkLibrary(lib); + + var it = module.import_table.iterator(); + while (it.next()) |entry| test_exe.root_module.addImport(entry.key_ptr.*, entry.value_ptr.*); + const tests_run = b.addRunArtifact(test_exe); + const test_step = b.step("test", "Run tests"); + test_step.dependOn(&tests_run.step); + } +} diff --git a/pkg/utfcpp/build.zig.zon b/pkg/utfcpp/build.zig.zon new file mode 100644 index 000000000..e56d77f19 --- /dev/null +++ b/pkg/utfcpp/build.zig.zon @@ -0,0 +1,13 @@ +.{ + .name = "utfcpp", + .version = "4.0.5", + .paths = .{""}, + .dependencies = .{ + .utfcpp = .{ + .url = "https://github.com/nemtrif/utfcpp/archive/refs/tags/v4.0.5.tar.gz", + .hash = "1220d4d18426ca72fc2b7e56ce47273149815501d0d2395c2a98c726b31ba931e641", + }, + + .apple_sdk = .{ .path = "../apple-sdk" }, + }, +} diff --git a/pkg/utfcpp/empty.cc b/pkg/utfcpp/empty.cc new file mode 100644 index 000000000..f2ef74097 --- /dev/null +++ b/pkg/utfcpp/empty.cc @@ -0,0 +1,2 @@ +// Needed for Zig build to be happy +void ghostty_utfcpp_stub() {} diff --git a/pkg/utfcpp/main.zig b/pkg/utfcpp/main.zig new file mode 100644 index 000000000..e69de29bb diff --git a/src/simd/utf8.h b/src/simd/utf8.h deleted file mode 100644 index b51353093..000000000 --- a/src/simd/utf8.h +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2006 Nemanja Trifunovic - -/* -Permission is hereby granted, free of charge, to any person or organization -obtaining a copy of the software and accompanying documentation covered by -this license (the "Software") to use, reproduce, display, distribute, -execute, and transmit the Software, and to prepare derivative works of the -Software, and to permit third-parties to whom the Software is furnished to -do so, all subject to the following: - -The copyright notices in the Software and this entire statement, including -the above license grant, this restriction and the following disclaimer, -must be included in all copies of the Software, in whole or in part, and -all derivative works of the Software, unless such copies or derivative -works are solely in the form of machine-executable object code generated by -a source language processor. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. -*/ - - -#ifndef UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731 -#define UTF8_FOR_CPP_2675DCD0_9480_4c0c_B92A_CC14C027B731 - -/* -To control the C++ language version used by the library, you can define UTF_CPP_CPLUSPLUS macro -and set it to one of the values used by the __cplusplus predefined macro. - -For instance, - #define UTF_CPP_CPLUSPLUS 199711L -will cause the UTF-8 CPP library to use only types and language features available in the C++ 98 standard. -Some library features will be disabled. - -If you leave UTF_CPP_CPLUSPLUS undefined, it will be internally assigned to __cplusplus. -*/ - -#include "utf8/checked.h" -#include "utf8/unchecked.h" - -#endif // header guard diff --git a/src/simd/utf8/checked.h b/src/simd/utf8/checked.h deleted file mode 100644 index 98949f8bc..000000000 --- a/src/simd/utf8/checked.h +++ /dev/null @@ -1,359 +0,0 @@ -// Copyright 2006-2016 Nemanja Trifunovic - -/* -Permission is hereby granted, free of charge, to any person or organization -obtaining a copy of the software and accompanying documentation covered by -this license (the "Software") to use, reproduce, display, distribute, -execute, and transmit the Software, and to prepare derivative works of the -Software, and to permit third-parties to whom the Software is furnished to -do so, all subject to the following: - -The copyright notices in the Software and this entire statement, including -the above license grant, this restriction and the following disclaimer, -must be included in all copies of the Software, in whole or in part, and -all derivative works of the Software, unless such copies or derivative -works are solely in the form of machine-executable object code generated by -a source language processor. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. -*/ - - -#ifndef UTF8_FOR_CPP_CHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 -#define UTF8_FOR_CPP_CHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 - -#include "core.h" -#include - -namespace utf8 -{ - // Base for the exceptions that may be thrown from the library - class exception : public ::std::exception { - }; - - // Exceptions that may be thrown from the library functions. - class invalid_code_point : public exception { - utfchar32_t cp; - public: - invalid_code_point(utfchar32_t codepoint) : cp(codepoint) {} - virtual const char* what() const UTF_CPP_NOEXCEPT UTF_CPP_OVERRIDE { return "Invalid code point"; } - utfchar32_t code_point() const {return cp;} - }; - - class invalid_utf8 : public exception { - utfchar8_t u8; - public: - invalid_utf8 (utfchar8_t u) : u8(u) {} - invalid_utf8 (char c) : u8(static_cast(c)) {} - virtual const char* what() const UTF_CPP_NOEXCEPT UTF_CPP_OVERRIDE { return "Invalid UTF-8"; } - utfchar8_t utf8_octet() const {return u8;} - }; - - class invalid_utf16 : public exception { - utfchar16_t u16; - public: - invalid_utf16 (utfchar16_t u) : u16(u) {} - virtual const char* what() const UTF_CPP_NOEXCEPT UTF_CPP_OVERRIDE { return "Invalid UTF-16"; } - utfchar16_t utf16_word() const {return u16;} - }; - - class not_enough_room : public exception { - public: - virtual const char* what() const UTF_CPP_NOEXCEPT UTF_CPP_OVERRIDE { return "Not enough space"; } - }; - - /// The library API - functions intended to be called by the users - - template - octet_iterator append(utfchar32_t cp, octet_iterator result) - { - if (!utf8::internal::is_code_point_valid(cp)) - throw invalid_code_point(cp); - - return internal::append(cp, result); - } - - inline void append(utfchar32_t cp, std::string& s) - { - append(cp, std::back_inserter(s)); - } - - template - word_iterator append16(utfchar32_t cp, word_iterator result) - { - if (!utf8::internal::is_code_point_valid(cp)) - throw invalid_code_point(cp); - - return internal::append16(cp, result); - } - - template - output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out, utfchar32_t replacement) - { - while (start != end) { - octet_iterator sequence_start = start; - internal::utf_error err_code = utf8::internal::validate_next(start, end); - switch (err_code) { - case internal::UTF8_OK : - for (octet_iterator it = sequence_start; it != start; ++it) - *out++ = *it; - break; - case internal::NOT_ENOUGH_ROOM: - out = utf8::append (replacement, out); - start = end; - break; - case internal::INVALID_LEAD: - out = utf8::append (replacement, out); - ++start; - break; - case internal::INCOMPLETE_SEQUENCE: - case internal::OVERLONG_SEQUENCE: - case internal::INVALID_CODE_POINT: - out = utf8::append (replacement, out); - ++start; - // just one replacement mark for the sequence - while (start != end && utf8::internal::is_trail(*start)) - ++start; - break; - } - } - return out; - } - - template - inline output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out) - { - static const utfchar32_t replacement_marker = utf8::internal::mask16(0xfffd); - return utf8::replace_invalid(start, end, out, replacement_marker); - } - - inline std::string replace_invalid(const std::string& s, utfchar32_t replacement) - { - std::string result; - replace_invalid(s.begin(), s.end(), std::back_inserter(result), replacement); - return result; - } - - inline std::string replace_invalid(const std::string& s) - { - std::string result; - replace_invalid(s.begin(), s.end(), std::back_inserter(result)); - return result; - } - - template - utfchar32_t next(octet_iterator& it, octet_iterator end) - { - utfchar32_t cp = 0; - internal::utf_error err_code = utf8::internal::validate_next(it, end, cp); - switch (err_code) { - case internal::UTF8_OK : - break; - case internal::NOT_ENOUGH_ROOM : - throw not_enough_room(); - case internal::INVALID_LEAD : - case internal::INCOMPLETE_SEQUENCE : - case internal::OVERLONG_SEQUENCE : - throw invalid_utf8(static_cast(*it)); - case internal::INVALID_CODE_POINT : - throw invalid_code_point(cp); - } - return cp; - } - - template - utfchar32_t next16(word_iterator& it, word_iterator end) - { - utfchar32_t cp = 0; - internal::utf_error err_code = utf8::internal::validate_next16(it, end, cp); - if (err_code == internal::NOT_ENOUGH_ROOM) - throw not_enough_room(); - return cp; - } - - template - utfchar32_t peek_next(octet_iterator it, octet_iterator end) - { - return utf8::next(it, end); - } - - template - utfchar32_t prior(octet_iterator& it, octet_iterator start) - { - // can't do much if it == start - if (it == start) - throw not_enough_room(); - - octet_iterator end = it; - // Go back until we hit either a lead octet or start - while (utf8::internal::is_trail(*(--it))) - if (it == start) - throw invalid_utf8(*it); // error - no lead byte in the sequence - return utf8::peek_next(it, end); - } - - template - void advance (octet_iterator& it, distance_type n, octet_iterator end) - { - const distance_type zero(0); - if (n < zero) { - // backward - for (distance_type i = n; i < zero; ++i) - utf8::prior(it, end); - } else { - // forward - for (distance_type i = zero; i < n; ++i) - utf8::next(it, end); - } - } - - template - typename std::iterator_traits::difference_type - distance (octet_iterator first, octet_iterator last) - { - typename std::iterator_traits::difference_type dist; - for (dist = 0; first < last; ++dist) - utf8::next(first, last); - return dist; - } - - template - octet_iterator utf16to8 (u16bit_iterator start, u16bit_iterator end, octet_iterator result) - { - while (start != end) { - utfchar32_t cp = utf8::internal::mask16(*start++); - // Take care of surrogate pairs first - if (utf8::internal::is_lead_surrogate(cp)) { - if (start != end) { - const utfchar32_t trail_surrogate = utf8::internal::mask16(*start++); - if (utf8::internal::is_trail_surrogate(trail_surrogate)) - cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET; - else - throw invalid_utf16(static_cast(trail_surrogate)); - } - else - throw invalid_utf16(static_cast(cp)); - - } - // Lone trail surrogate - else if (utf8::internal::is_trail_surrogate(cp)) - throw invalid_utf16(static_cast(cp)); - - result = utf8::append(cp, result); - } - return result; - } - - template - u16bit_iterator utf8to16 (octet_iterator start, octet_iterator end, u16bit_iterator result) - { - while (start < end) { - const utfchar32_t cp = utf8::next(start, end); - if (cp > 0xffff) { //make a surrogate pair - *result++ = static_cast((cp >> 10) + internal::LEAD_OFFSET); - *result++ = static_cast((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN); - } - else - *result++ = static_cast(cp); - } - return result; - } - - template - octet_iterator utf32to8 (u32bit_iterator start, u32bit_iterator end, octet_iterator result) - { - while (start != end) - result = utf8::append(*(start++), result); - - return result; - } - - template - u32bit_iterator utf8to32 (octet_iterator start, octet_iterator end, u32bit_iterator result) - { - while (start < end) - (*result++) = utf8::next(start, end); - - return result; - } - - // The iterator class - template - class iterator { - octet_iterator it; - octet_iterator range_start; - octet_iterator range_end; - public: - typedef utfchar32_t value_type; - typedef utfchar32_t* pointer; - typedef utfchar32_t& reference; - typedef std::ptrdiff_t difference_type; - typedef std::bidirectional_iterator_tag iterator_category; - iterator () {} - explicit iterator (const octet_iterator& octet_it, - const octet_iterator& rangestart, - const octet_iterator& rangeend) : - it(octet_it), range_start(rangestart), range_end(rangeend) - { - if (it < range_start || it > range_end) - throw std::out_of_range("Invalid utf-8 iterator position"); - } - // the default "big three" are OK - octet_iterator base () const { return it; } - utfchar32_t operator * () const - { - octet_iterator temp = it; - return utf8::next(temp, range_end); - } - bool operator == (const iterator& rhs) const - { - if (range_start != rhs.range_start || range_end != rhs.range_end) - throw std::logic_error("Comparing utf-8 iterators defined with different ranges"); - return (it == rhs.it); - } - bool operator != (const iterator& rhs) const - { - return !(operator == (rhs)); - } - iterator& operator ++ () - { - utf8::next(it, range_end); - return *this; - } - iterator operator ++ (int) - { - iterator temp = *this; - utf8::next(it, range_end); - return temp; - } - iterator& operator -- () - { - utf8::prior(it, range_start); - return *this; - } - iterator operator -- (int) - { - iterator temp = *this; - utf8::prior(it, range_start); - return temp; - } - }; // class iterator - -} // namespace utf8 - -#if UTF_CPP_CPLUSPLUS >= 202002L // C++ 20 or later -#include "cpp20.h" -#elif UTF_CPP_CPLUSPLUS >= 201703L // C++ 17 or later -#include "cpp17.h" -#elif UTF_CPP_CPLUSPLUS >= 201103L // C++ 11 or later -#include "cpp11.h" -#endif // C++ 11 or later - -#endif //header guard - diff --git a/src/simd/utf8/core.h b/src/simd/utf8/core.h deleted file mode 100644 index 4494c538e..000000000 --- a/src/simd/utf8/core.h +++ /dev/null @@ -1,492 +0,0 @@ -// Copyright 2006 Nemanja Trifunovic - -/* -Permission is hereby granted, free of charge, to any person or organization -obtaining a copy of the software and accompanying documentation covered by -this license (the "Software") to use, reproduce, display, distribute, -execute, and transmit the Software, and to prepare derivative works of the -Software, and to permit third-parties to whom the Software is furnished to -do so, all subject to the following: - -The copyright notices in the Software and this entire statement, including -the above license grant, this restriction and the following disclaimer, -must be included in all copies of the Software, in whole or in part, and -all derivative works of the Software, unless such copies or derivative -works are solely in the form of machine-executable object code generated by -a source language processor. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. -*/ - - -#ifndef UTF8_FOR_CPP_CORE_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 -#define UTF8_FOR_CPP_CORE_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 - -#include -#include -#include - -// Determine the C++ standard version. -// If the user defines UTF_CPP_CPLUSPLUS, use that. -// Otherwise, trust the unreliable predefined macro __cplusplus - -#if !defined UTF_CPP_CPLUSPLUS - #define UTF_CPP_CPLUSPLUS __cplusplus -#endif - -#if UTF_CPP_CPLUSPLUS >= 201103L // C++ 11 or later - #define UTF_CPP_OVERRIDE override - #define UTF_CPP_NOEXCEPT noexcept -#else // C++ 98/03 - #define UTF_CPP_OVERRIDE - #define UTF_CPP_NOEXCEPT throw() -#endif // C++ 11 or later - - -namespace utf8 -{ -// The typedefs for 8-bit, 16-bit and 32-bit code units -#if UTF_CPP_CPLUSPLUS >= 201103L // C++ 11 or later - #if UTF_CPP_CPLUSPLUS >= 202002L // C++ 20 or later - typedef char8_t utfchar8_t; - #else // C++ 11/14/17 - typedef unsigned char utfchar8_t; - #endif - typedef char16_t utfchar16_t; - typedef char32_t utfchar32_t; -#else // C++ 98/03 - typedef unsigned char utfchar8_t; - typedef unsigned short utfchar16_t; - typedef unsigned int utfchar32_t; -#endif // C++ 11 or later - -// Helper code - not intended to be directly called by the library users. May be changed at any time -namespace internal -{ - // Unicode constants - // Leading (high) surrogates: 0xd800 - 0xdbff - // Trailing (low) surrogates: 0xdc00 - 0xdfff - const utfchar16_t LEAD_SURROGATE_MIN = 0xd800u; - const utfchar16_t LEAD_SURROGATE_MAX = 0xdbffu; - const utfchar16_t TRAIL_SURROGATE_MIN = 0xdc00u; - const utfchar16_t TRAIL_SURROGATE_MAX = 0xdfffu; - const utfchar16_t LEAD_OFFSET = 0xd7c0u; // LEAD_SURROGATE_MIN - (0x10000 >> 10) - const utfchar32_t SURROGATE_OFFSET = 0xfca02400u; // 0x10000u - (LEAD_SURROGATE_MIN << 10) - TRAIL_SURROGATE_MIN - - // Maximum valid value for a Unicode code point - const utfchar32_t CODE_POINT_MAX = 0x0010ffffu; - - template - inline utfchar8_t mask8(octet_type oc) - { - return static_cast(0xff & oc); - } - template - inline utfchar16_t mask16(u16_type oc) - { - return static_cast(0xffff & oc); - } - - template - inline bool is_trail(octet_type oc) - { - return ((utf8::internal::mask8(oc) >> 6) == 0x2); - } - - inline bool is_lead_surrogate(utfchar32_t cp) - { - return (cp >= LEAD_SURROGATE_MIN && cp <= LEAD_SURROGATE_MAX); - } - - inline bool is_trail_surrogate(utfchar32_t cp) - { - return (cp >= TRAIL_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX); - } - - inline bool is_surrogate(utfchar32_t cp) - { - return (cp >= LEAD_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX); - } - - inline bool is_code_point_valid(utfchar32_t cp) - { - return (cp <= CODE_POINT_MAX && !utf8::internal::is_surrogate(cp)); - } - - inline bool is_in_bmp(utfchar32_t cp) - { - return cp < utfchar32_t(0x10000); - } - - template - int sequence_length(octet_iterator lead_it) - { - const utfchar8_t lead = utf8::internal::mask8(*lead_it); - if (lead < 0x80) - return 1; - else if ((lead >> 5) == 0x6) - return 2; - else if ((lead >> 4) == 0xe) - return 3; - else if ((lead >> 3) == 0x1e) - return 4; - else - return 0; - } - - inline bool is_overlong_sequence(utfchar32_t cp, int length) - { - if (cp < 0x80) { - if (length != 1) - return true; - } - else if (cp < 0x800) { - if (length != 2) - return true; - } - else if (cp < 0x10000) { - if (length != 3) - return true; - } - return false; - } - - enum utf_error {UTF8_OK, NOT_ENOUGH_ROOM, INVALID_LEAD, INCOMPLETE_SEQUENCE, OVERLONG_SEQUENCE, INVALID_CODE_POINT}; - - /// Helper for get_sequence_x - template - utf_error increase_safely(octet_iterator& it, const octet_iterator end) - { - if (++it == end) - return NOT_ENOUGH_ROOM; - - if (!utf8::internal::is_trail(*it)) - return INCOMPLETE_SEQUENCE; - - return UTF8_OK; - } - - #define UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(IT, END) {utf_error ret = increase_safely(IT, END); if (ret != UTF8_OK) return ret;} - - /// get_sequence_x functions decode utf-8 sequences of the length x - template - utf_error get_sequence_1(octet_iterator& it, octet_iterator end, utfchar32_t& code_point) - { - if (it == end) - return NOT_ENOUGH_ROOM; - - code_point = utf8::internal::mask8(*it); - - return UTF8_OK; - } - - template - utf_error get_sequence_2(octet_iterator& it, octet_iterator end, utfchar32_t& code_point) - { - if (it == end) - return NOT_ENOUGH_ROOM; - - code_point = utf8::internal::mask8(*it); - - UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) - - code_point = ((code_point << 6) & 0x7ff) + ((*it) & 0x3f); - - return UTF8_OK; - } - - template - utf_error get_sequence_3(octet_iterator& it, octet_iterator end, utfchar32_t& code_point) - { - if (it == end) - return NOT_ENOUGH_ROOM; - - code_point = utf8::internal::mask8(*it); - - UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) - - code_point = ((code_point << 12) & 0xffff) + ((utf8::internal::mask8(*it) << 6) & 0xfff); - - UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) - - code_point += (*it) & 0x3f; - - return UTF8_OK; - } - - template - utf_error get_sequence_4(octet_iterator& it, octet_iterator end, utfchar32_t& code_point) - { - if (it == end) - return NOT_ENOUGH_ROOM; - - code_point = utf8::internal::mask8(*it); - - UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) - - code_point = ((code_point << 18) & 0x1fffff) + ((utf8::internal::mask8(*it) << 12) & 0x3ffff); - - UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) - - code_point += (utf8::internal::mask8(*it) << 6) & 0xfff; - - UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end) - - code_point += (*it) & 0x3f; - - return UTF8_OK; - } - - #undef UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR - - template - utf_error validate_next(octet_iterator& it, octet_iterator end, utfchar32_t& code_point) - { - if (it == end) - return NOT_ENOUGH_ROOM; - - // Save the original value of it so we can go back in case of failure - // Of course, it does not make much sense with i.e. stream iterators - octet_iterator original_it = it; - - utfchar32_t cp = 0; - // Determine the sequence length based on the lead octet - const int length = utf8::internal::sequence_length(it); - - // Get trail octets and calculate the code point - utf_error err = UTF8_OK; - switch (length) { - case 0: - return INVALID_LEAD; - case 1: - err = utf8::internal::get_sequence_1(it, end, cp); - break; - case 2: - err = utf8::internal::get_sequence_2(it, end, cp); - break; - case 3: - err = utf8::internal::get_sequence_3(it, end, cp); - break; - case 4: - err = utf8::internal::get_sequence_4(it, end, cp); - break; - } - - if (err == UTF8_OK) { - // Decoding succeeded. Now, security checks... - if (utf8::internal::is_code_point_valid(cp)) { - if (!utf8::internal::is_overlong_sequence(cp, length)){ - // Passed! Return here. - code_point = cp; - ++it; - return UTF8_OK; - } - else - err = OVERLONG_SEQUENCE; - } - else - err = INVALID_CODE_POINT; - } - - // Failure branch - restore the original value of the iterator - it = original_it; - return err; - } - - template - inline utf_error validate_next(octet_iterator& it, octet_iterator end) { - utfchar32_t ignored; - return utf8::internal::validate_next(it, end, ignored); - } - - template - utf_error validate_next16(word_iterator& it, word_iterator end, utfchar32_t& code_point) - { - if (it == end) - return NOT_ENOUGH_ROOM; - // Save the original value of it so we can go back in case of failure - // Of course, it does not make much sense with i.e. stream iterators - word_iterator original_it = it; - - utf_error err = UTF8_OK; - - const utfchar16_t first_word = *it++; - if (!is_surrogate(first_word)) { - code_point = first_word; - return UTF8_OK; - } - else { - if (it == end) - err = NOT_ENOUGH_ROOM; - else if (is_lead_surrogate(first_word)) { - const utfchar16_t second_word = *it++; - if (is_trail_surrogate(second_word)) { - code_point = (first_word << 10) + second_word + SURROGATE_OFFSET; - return UTF8_OK; - } else - err = INCOMPLETE_SEQUENCE; - - } else { - err = INVALID_LEAD; - } - } - // error branch - it = original_it; - return err; - } - - // Internal implementation of both checked and unchecked append() function - // This function will be invoked by the overloads below, as they will know - // the octet_type. - template - octet_iterator append(utfchar32_t cp, octet_iterator result) { - if (cp < 0x80) // one octet - *(result++) = static_cast(cp); - else if (cp < 0x800) { // two octets - *(result++) = static_cast((cp >> 6) | 0xc0); - *(result++) = static_cast((cp & 0x3f) | 0x80); - } - else if (cp < 0x10000) { // three octets - *(result++) = static_cast((cp >> 12) | 0xe0); - *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); - *(result++) = static_cast((cp & 0x3f) | 0x80); - } - else { // four octets - *(result++) = static_cast((cp >> 18) | 0xf0); - *(result++) = static_cast(((cp >> 12) & 0x3f)| 0x80); - *(result++) = static_cast(((cp >> 6) & 0x3f) | 0x80); - *(result++) = static_cast((cp & 0x3f) | 0x80); - } - return result; - } - - // One of the following overloads will be invoked from the API calls - - // A simple (but dangerous) case: the caller appends byte(s) to a char array - inline char* append(utfchar32_t cp, char* result) { - return append(cp, result); - } - - // Hopefully, most common case: the caller uses back_inserter - // i.e. append(cp, std::back_inserter(str)); - template - std::back_insert_iterator append - (utfchar32_t cp, std::back_insert_iterator result) { - return append, - typename container_type::value_type>(cp, result); - } - - // The caller uses some other kind of output operator - not covered above - // Note that in this case we are not able to determine octet_type - // so we assume it's utfchar8_t; that can cause a conversion warning if we are wrong. - template - octet_iterator append(utfchar32_t cp, octet_iterator result) { - return append(cp, result); - } - - // Internal implementation of both checked and unchecked append16() function - // This function will be invoked by the overloads below, as they will know - // the word_type. - template - word_iterator append16(utfchar32_t cp, word_iterator result) { - if (is_in_bmp(cp)) - *(result++) = static_cast(cp); - else { - // Code points from the supplementary planes are encoded via surrogate pairs - *(result++) = static_cast(LEAD_OFFSET + (cp >> 10)); - *(result++) = static_cast(TRAIL_SURROGATE_MIN + (cp & 0x3FF)); - } - return result; - } - - // Hopefully, most common case: the caller uses back_inserter - // i.e. append16(cp, std::back_inserter(str)); - template - std::back_insert_iterator append16 - (utfchar32_t cp, std::back_insert_iterator result) { - return append16, - typename container_type::value_type>(cp, result); - } - - // The caller uses some other kind of output operator - not covered above - // Note that in this case we are not able to determine word_type - // so we assume it's utfchar16_t; that can cause a conversion warning if we are wrong. - template - word_iterator append16(utfchar32_t cp, word_iterator result) { - return append16(cp, result); - } - -} // namespace internal - - /// The library API - functions intended to be called by the users - - // Byte order mark - const utfchar8_t bom[] = {0xef, 0xbb, 0xbf}; - - template - octet_iterator find_invalid(octet_iterator start, octet_iterator end) - { - octet_iterator result = start; - while (result != end) { - utf8::internal::utf_error err_code = utf8::internal::validate_next(result, end); - if (err_code != internal::UTF8_OK) - return result; - } - return result; - } - - inline const char* find_invalid(const char* str) - { - const char* end = str + std::strlen(str); - return find_invalid(str, end); - } - - inline std::size_t find_invalid(const std::string& s) - { - std::string::const_iterator invalid = find_invalid(s.begin(), s.end()); - return (invalid == s.end()) ? std::string::npos : static_cast(invalid - s.begin()); - } - - template - inline bool is_valid(octet_iterator start, octet_iterator end) - { - return (utf8::find_invalid(start, end) == end); - } - - inline bool is_valid(const char* str) - { - return (*(utf8::find_invalid(str)) == '\0'); - } - - inline bool is_valid(const std::string& s) - { - return is_valid(s.begin(), s.end()); - } - - - - template - inline bool starts_with_bom (octet_iterator it, octet_iterator end) - { - return ( - ((it != end) && (utf8::internal::mask8(*it++)) == bom[0]) && - ((it != end) && (utf8::internal::mask8(*it++)) == bom[1]) && - ((it != end) && (utf8::internal::mask8(*it)) == bom[2]) - ); - } - - inline bool starts_with_bom(const std::string& s) - { - return starts_with_bom(s.begin(), s.end()); - } -} // namespace utf8 - -#endif // header guard - - diff --git a/src/simd/utf8/cpp11.h b/src/simd/utf8/cpp11.h deleted file mode 100644 index 691633c84..000000000 --- a/src/simd/utf8/cpp11.h +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2018 Nemanja Trifunovic - -/* -Permission is hereby granted, free of charge, to any person or organization -obtaining a copy of the software and accompanying documentation covered by -this license (the "Software") to use, reproduce, display, distribute, -execute, and transmit the Software, and to prepare derivative works of the -Software, and to permit third-parties to whom the Software is furnished to -do so, all subject to the following: - -The copyright notices in the Software and this entire statement, including -the above license grant, this restriction and the following disclaimer, -must be included in all copies of the Software, in whole or in part, and -all derivative works of the Software, unless such copies or derivative -works are solely in the form of machine-executable object code generated by -a source language processor. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. -*/ - - -#ifndef UTF8_FOR_CPP_a184c22c_d012_11e8_a8d5_f2801f1b9fd1 -#define UTF8_FOR_CPP_a184c22c_d012_11e8_a8d5_f2801f1b9fd1 - -#include "checked.h" - -namespace utf8 -{ - inline void append16(utfchar32_t cp, std::u16string& s) - { - append16(cp, std::back_inserter(s)); - } - - inline std::string utf16to8(const std::u16string& s) - { - std::string result; - utf16to8(s.begin(), s.end(), std::back_inserter(result)); - return result; - } - - inline std::u16string utf8to16(const std::string& s) - { - std::u16string result; - utf8to16(s.begin(), s.end(), std::back_inserter(result)); - return result; - } - - inline std::string utf32to8(const std::u32string& s) - { - std::string result; - utf32to8(s.begin(), s.end(), std::back_inserter(result)); - return result; - } - - inline std::u32string utf8to32(const std::string& s) - { - std::u32string result; - utf8to32(s.begin(), s.end(), std::back_inserter(result)); - return result; - } -} // namespace utf8 - -#endif // header guard - diff --git a/src/simd/utf8/cpp17.h b/src/simd/utf8/cpp17.h deleted file mode 100644 index 6e2fcc238..000000000 --- a/src/simd/utf8/cpp17.h +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2018 Nemanja Trifunovic - -/* -Permission is hereby granted, free of charge, to any person or organization -obtaining a copy of the software and accompanying documentation covered by -this license (the "Software") to use, reproduce, display, distribute, -execute, and transmit the Software, and to prepare derivative works of the -Software, and to permit third-parties to whom the Software is furnished to -do so, all subject to the following: - -The copyright notices in the Software and this entire statement, including -the above license grant, this restriction and the following disclaimer, -must be included in all copies of the Software, in whole or in part, and -all derivative works of the Software, unless such copies or derivative -works are solely in the form of machine-executable object code generated by -a source language processor. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. -*/ - - -#ifndef UTF8_FOR_CPP_7e906c01_03a3_4daf_b420_ea7ea952b3c9 -#define UTF8_FOR_CPP_7e906c01_03a3_4daf_b420_ea7ea952b3c9 - -#include "cpp11.h" - -namespace utf8 -{ - inline std::string utf16to8(std::u16string_view s) - { - std::string result; - utf16to8(s.begin(), s.end(), std::back_inserter(result)); - return result; - } - - inline std::u16string utf8to16(std::string_view s) - { - std::u16string result; - utf8to16(s.begin(), s.end(), std::back_inserter(result)); - return result; - } - - inline std::string utf32to8(std::u32string_view s) - { - std::string result; - utf32to8(s.begin(), s.end(), std::back_inserter(result)); - return result; - } - - inline std::u32string utf8to32(std::string_view s) - { - std::u32string result; - utf8to32(s.begin(), s.end(), std::back_inserter(result)); - return result; - } - - inline std::size_t find_invalid(std::string_view s) - { - std::string_view::const_iterator invalid = find_invalid(s.begin(), s.end()); - return (invalid == s.end()) ? std::string_view::npos : static_cast(invalid - s.begin()); - } - - inline bool is_valid(std::string_view s) - { - return is_valid(s.begin(), s.end()); - } - - inline std::string replace_invalid(std::string_view s, char32_t replacement) - { - std::string result; - replace_invalid(s.begin(), s.end(), std::back_inserter(result), replacement); - return result; - } - - inline std::string replace_invalid(std::string_view s) - { - std::string result; - replace_invalid(s.begin(), s.end(), std::back_inserter(result)); - return result; - } - - inline bool starts_with_bom(std::string_view s) - { - return starts_with_bom(s.begin(), s.end()); - } - -} // namespace utf8 - -#endif // header guard - diff --git a/src/simd/utf8/cpp20.h b/src/simd/utf8/cpp20.h deleted file mode 100644 index 07b61d0fb..000000000 --- a/src/simd/utf8/cpp20.h +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright 2022 Nemanja Trifunovic - -/* -Permission is hereby granted, free of charge, to any person or organization -obtaining a copy of the software and accompanying documentation covered by -this license (the "Software") to use, reproduce, display, distribute, -execute, and transmit the Software, and to prepare derivative works of the -Software, and to permit third-parties to whom the Software is furnished to -do so, all subject to the following: - -The copyright notices in the Software and this entire statement, including -the above license grant, this restriction and the following disclaimer, -must be included in all copies of the Software, in whole or in part, and -all derivative works of the Software, unless such copies or derivative -works are solely in the form of machine-executable object code generated by -a source language processor. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. -*/ - - -#ifndef UTF8_FOR_CPP_207e906c01_03a3_4daf_b420_ea7ea952b3c9 -#define UTF8_FOR_CPP_207e906c01_03a3_4daf_b420_ea7ea952b3c9 - -#include "cpp17.h" - -namespace utf8 -{ - inline std::u8string utf16tou8(const std::u16string& s) - { - std::u8string result; - utf16to8(s.begin(), s.end(), std::back_inserter(result)); - return result; - } - - inline std::u8string utf16tou8(std::u16string_view s) - { - std::u8string result; - utf16to8(s.begin(), s.end(), std::back_inserter(result)); - return result; - } - - inline std::u16string utf8to16(const std::u8string& s) - { - std::u16string result; - utf8to16(s.begin(), s.end(), std::back_inserter(result)); - return result; - } - - inline std::u16string utf8to16(const std::u8string_view& s) - { - std::u16string result; - utf8to16(s.begin(), s.end(), std::back_inserter(result)); - return result; - } - - inline std::u8string utf32tou8(const std::u32string& s) - { - std::u8string result; - utf32to8(s.begin(), s.end(), std::back_inserter(result)); - return result; - } - - inline std::u8string utf32tou8(const std::u32string_view& s) - { - std::u8string result; - utf32to8(s.begin(), s.end(), std::back_inserter(result)); - return result; - } - - inline std::u32string utf8to32(const std::u8string& s) - { - std::u32string result; - utf8to32(s.begin(), s.end(), std::back_inserter(result)); - return result; - } - - inline std::u32string utf8to32(const std::u8string_view& s) - { - std::u32string result; - utf8to32(s.begin(), s.end(), std::back_inserter(result)); - return result; - } - - inline std::size_t find_invalid(const std::u8string& s) - { - std::u8string::const_iterator invalid = find_invalid(s.begin(), s.end()); - return (invalid == s.end()) ? std::string_view::npos : static_cast(invalid - s.begin()); - } - - inline bool is_valid(const std::u8string& s) - { - return is_valid(s.begin(), s.end()); - } - - inline std::u8string replace_invalid(const std::u8string& s, char32_t replacement) - { - std::u8string result; - replace_invalid(s.begin(), s.end(), std::back_inserter(result), replacement); - return result; - } - - inline std::u8string replace_invalid(const std::u8string& s) - { - std::u8string result; - replace_invalid(s.begin(), s.end(), std::back_inserter(result)); - return result; - } - - inline bool starts_with_bom(const std::u8string& s) - { - return starts_with_bom(s.begin(), s.end()); - } - -} // namespace utf8 - -#endif // header guard - diff --git a/src/simd/utf8/unchecked.h b/src/simd/utf8/unchecked.h deleted file mode 100644 index 65d4948f2..000000000 --- a/src/simd/utf8/unchecked.h +++ /dev/null @@ -1,287 +0,0 @@ -// Copyright 2006 Nemanja Trifunovic - -/* -Permission is hereby granted, free of charge, to any person or organization -obtaining a copy of the software and accompanying documentation covered by -this license (the "Software") to use, reproduce, display, distribute, -execute, and transmit the Software, and to prepare derivative works of the -Software, and to permit third-parties to whom the Software is furnished to -do so, all subject to the following: - -The copyright notices in the Software and this entire statement, including -the above license grant, this restriction and the following disclaimer, -must be included in all copies of the Software, in whole or in part, and -all derivative works of the Software, unless such copies or derivative -works are solely in the form of machine-executable object code generated by -a source language processor. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT -SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE -FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. -*/ - - -#ifndef UTF8_FOR_CPP_UNCHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 -#define UTF8_FOR_CPP_UNCHECKED_H_2675DCD0_9480_4c0c_B92A_CC14C027B731 - -#include "core.h" - -namespace utf8 -{ - namespace unchecked - { - template - octet_iterator append(utfchar32_t cp, octet_iterator result) - { - return internal::append(cp, result); - } - - template - word_iterator append16(utfchar32_t cp, word_iterator result) - { - return internal::append16(cp, result); - } - - template - output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out, utfchar32_t replacement) - { - while (start != end) { - octet_iterator sequence_start = start; - internal::utf_error err_code = utf8::internal::validate_next(start, end); - switch (err_code) { - case internal::UTF8_OK : - for (octet_iterator it = sequence_start; it != start; ++it) - *out++ = *it; - break; - case internal::NOT_ENOUGH_ROOM: - out = utf8::unchecked::append(replacement, out); - start = end; - break; - case internal::INVALID_LEAD: - out = utf8::unchecked::append(replacement, out); - ++start; - break; - case internal::INCOMPLETE_SEQUENCE: - case internal::OVERLONG_SEQUENCE: - case internal::INVALID_CODE_POINT: - out = utf8::unchecked::append(replacement, out); - ++start; - // just one replacement mark for the sequence - while (start != end && utf8::internal::is_trail(*start)) - ++start; - break; - } - } - return out; - } - - template - inline output_iterator replace_invalid(octet_iterator start, octet_iterator end, output_iterator out) - { - static const utfchar32_t replacement_marker = utf8::internal::mask16(0xfffd); - return utf8::unchecked::replace_invalid(start, end, out, replacement_marker); - } - - inline std::string replace_invalid(const std::string& s, utfchar32_t replacement) - { - std::string result; - replace_invalid(s.begin(), s.end(), std::back_inserter(result), replacement); - return result; - } - - inline std::string replace_invalid(const std::string& s) - { - std::string result; - replace_invalid(s.begin(), s.end(), std::back_inserter(result)); - return result; - } - - template - utfchar32_t next(octet_iterator& it) - { - utfchar32_t cp = utf8::internal::mask8(*it); - switch (utf8::internal::sequence_length(it)) { - case 1: - break; - case 2: - it++; - cp = ((cp << 6) & 0x7ff) + ((*it) & 0x3f); - break; - case 3: - ++it; - cp = ((cp << 12) & 0xffff) + ((utf8::internal::mask8(*it) << 6) & 0xfff); - ++it; - cp += (*it) & 0x3f; - break; - case 4: - ++it; - cp = ((cp << 18) & 0x1fffff) + ((utf8::internal::mask8(*it) << 12) & 0x3ffff); - ++it; - cp += (utf8::internal::mask8(*it) << 6) & 0xfff; - ++it; - cp += (*it) & 0x3f; - break; - } - ++it; - return cp; - } - - template - utfchar32_t peek_next(octet_iterator it) - { - return utf8::unchecked::next(it); - } - - template - utfchar32_t next16(word_iterator& it) - { - utfchar32_t cp = utf8::internal::mask16(*it++); - if (utf8::internal::is_lead_surrogate(cp)) - return (cp << 10) + *it++ + utf8::internal::SURROGATE_OFFSET; - return cp; - } - - template - utfchar32_t prior(octet_iterator& it) - { - while (utf8::internal::is_trail(*(--it))) ; - octet_iterator temp = it; - return utf8::unchecked::next(temp); - } - - template - void advance(octet_iterator& it, distance_type n) - { - const distance_type zero(0); - if (n < zero) { - // backward - for (distance_type i = n; i < zero; ++i) - utf8::unchecked::prior(it); - } else { - // forward - for (distance_type i = zero; i < n; ++i) - utf8::unchecked::next(it); - } - } - - template - typename std::iterator_traits::difference_type - distance(octet_iterator first, octet_iterator last) - { - typename std::iterator_traits::difference_type dist; - for (dist = 0; first < last; ++dist) - utf8::unchecked::next(first); - return dist; - } - - template - octet_iterator utf16to8(u16bit_iterator start, u16bit_iterator end, octet_iterator result) - { - while (start != end) { - utfchar32_t cp = utf8::internal::mask16(*start++); - // Take care of surrogate pairs first - if (utf8::internal::is_lead_surrogate(cp)) { - if (start == end) - return result; - utfchar32_t trail_surrogate = utf8::internal::mask16(*start++); - cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET; - } - result = utf8::unchecked::append(cp, result); - } - return result; - } - - template - u16bit_iterator utf8to16(octet_iterator start, octet_iterator end, u16bit_iterator result) - { - while (start < end) { - utfchar32_t cp = utf8::unchecked::next(start); - if (cp > 0xffff) { //make a surrogate pair - *result++ = static_cast((cp >> 10) + internal::LEAD_OFFSET); - *result++ = static_cast((cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN); - } - else - *result++ = static_cast(cp); - } - return result; - } - - template - octet_iterator utf32to8(u32bit_iterator start, u32bit_iterator end, octet_iterator result) - { - while (start != end) - result = utf8::unchecked::append(*(start++), result); - - return result; - } - - template - u32bit_iterator utf8to32(octet_iterator start, octet_iterator end, u32bit_iterator result) - { - while (start < end) - (*result++) = utf8::unchecked::next(start); - - return result; - } - - // The iterator class - template - class iterator { - octet_iterator it; - public: - typedef utfchar32_t value_type; - typedef utfchar32_t* pointer; - typedef utfchar32_t& reference; - typedef std::ptrdiff_t difference_type; - typedef std::bidirectional_iterator_tag iterator_category; - iterator () {} - explicit iterator (const octet_iterator& octet_it): it(octet_it) {} - // the default "big three" are OK - octet_iterator base () const { return it; } - utfchar32_t operator * () const - { - octet_iterator temp = it; - return utf8::unchecked::next(temp); - } - bool operator == (const iterator& rhs) const - { - return (it == rhs.it); - } - bool operator != (const iterator& rhs) const - { - return !(operator == (rhs)); - } - iterator& operator ++ () - { - ::std::advance(it, utf8::internal::sequence_length(it)); - return *this; - } - iterator operator ++ (int) - { - iterator temp = *this; - ::std::advance(it, utf8::internal::sequence_length(it)); - return temp; - } - iterator& operator -- () - { - utf8::unchecked::prior(it); - return *this; - } - iterator operator -- (int) - { - iterator temp = *this; - utf8::unchecked::prior(it); - return temp; - } - }; // class iterator - - } // namespace utf8::unchecked -} // namespace utf8 - - -#endif // header guard - diff --git a/src/simd/vt.cpp b/src/simd/vt.cpp index fe4d8a1fa..e6c762bea 100644 --- a/src/simd/vt.cpp +++ b/src/simd/vt.cpp @@ -4,12 +4,13 @@ #include // must come before highway.h #include -#include -#include -#include -#include +#include +#include #include +#include +#include + HWY_BEFORE_NAMESPACE(); namespace ghostty { namespace HWY_NAMESPACE {