123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327 |
- /*
- * Tencent is pleased to support the open source community by making wwsearch
- * available.
- *
- * Copyright (C) 2018-present Tencent. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License. You may obtain a copy of
- * the License at
- *
- * https://opensource.org/licenses/Apache-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OF ANY KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations under the License.
- */
- // Copyright 2006 Nemanja Trifunovic
- /*
- Permission is hereby granted, free of charge, to any person or organization
- obtaining a copy of the software and accompanying documentation covered by
- this license (the "Software") to use, reproduce, display, distribute,
- execute, and transmit the Software, and to prepare derivative works of the
- Software, and to permit third-parties to whom the Software is furnished to
- do so, all subject to the following:
- The copyright notices in the Software and this entire statement, including
- the above license grant, this restriction and the following disclaimer,
- must be included in all copies of the Software, in whole or in part, and
- all derivative works of the Software, unless such copies or derivative
- works are solely in the form of machine-executable object code generated by
- a source language processor.
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
- SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
- FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
- ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
- DEALINGS IN THE SOFTWARE.
- */
- #ifndef UTF8_FOR_CPP_CORE_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
- #define UTF8_FOR_CPP_CORE_H_2675DCD0_9480_4c0c_B92A_CC14C027B731
- #include <iterator>
- namespace utf8 {
- // The typedefs for 8-bit, 16-bit and 32-bit unsigned integers
- // You may need to change them to match your system.
- // These typedefs have the same names as ones from cstdint, or boost/cstdint
- typedef unsigned char uint8_t;
- typedef unsigned short uint16_t;
- typedef unsigned int uint32_t;
- // Helper code - not intended to be directly called by the library users. May be
- // changed at any time
- namespace internal {
- // Unicode constants
- // Leading (high) surrogates: 0xd800 - 0xdbff
- // Trailing (low) surrogates: 0xdc00 - 0xdfff
- const uint16_t LEAD_SURROGATE_MIN = 0xd800u;
- const uint16_t LEAD_SURROGATE_MAX = 0xdbffu;
- const uint16_t TRAIL_SURROGATE_MIN = 0xdc00u;
- const uint16_t TRAIL_SURROGATE_MAX = 0xdfffu;
- const uint16_t LEAD_OFFSET = LEAD_SURROGATE_MIN - (0x10000 >> 10);
- const uint32_t SURROGATE_OFFSET =
- 0x10000u - (LEAD_SURROGATE_MIN << 10) - TRAIL_SURROGATE_MIN;
- // Maximum valid value for a Unicode code point
- const uint32_t CODE_POINT_MAX = 0x0010ffffu;
- template <typename octet_type>
- inline uint8_t mask8(octet_type oc) {
- return static_cast<uint8_t>(0xff & oc);
- }
- template <typename u16_type>
- inline uint16_t mask16(u16_type oc) {
- return static_cast<uint16_t>(0xffff & oc);
- }
- template <typename octet_type>
- inline bool is_trail(octet_type oc) {
- return ((utf8::internal::mask8(oc) >> 6) == 0x2);
- }
- template <typename u16>
- inline bool is_lead_surrogate(u16 cp) {
- return (cp >= LEAD_SURROGATE_MIN && cp <= LEAD_SURROGATE_MAX);
- }
- template <typename u16>
- inline bool is_trail_surrogate(u16 cp) {
- return (cp >= TRAIL_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX);
- }
- template <typename u16>
- inline bool is_surrogate(u16 cp) {
- return (cp >= LEAD_SURROGATE_MIN && cp <= TRAIL_SURROGATE_MAX);
- }
- template <typename u32>
- inline bool is_code_point_valid(u32 cp) {
- return (cp <= CODE_POINT_MAX && !utf8::internal::is_surrogate(cp));
- }
- template <typename octet_iterator>
- inline typename std::iterator_traits<octet_iterator>::difference_type
- sequence_length(octet_iterator lead_it) {
- uint8_t lead = utf8::internal::mask8(*lead_it);
- if (lead < 0x80)
- return 1;
- else if ((lead >> 5) == 0x6)
- return 2;
- else if ((lead >> 4) == 0xe)
- return 3;
- else if ((lead >> 3) == 0x1e)
- return 4;
- else
- return 0;
- }
- template <typename octet_difference_type>
- inline bool is_overlong_sequence(uint32_t cp, octet_difference_type length) {
- if (cp < 0x80) {
- if (length != 1) return true;
- } else if (cp < 0x800) {
- if (length != 2) return true;
- } else if (cp < 0x10000) {
- if (length != 3) return true;
- }
- return false;
- }
- enum utf_error {
- UTF8_OK,
- NOT_ENOUGH_ROOM,
- INVALID_LEAD,
- INCOMPLETE_SEQUENCE,
- OVERLONG_SEQUENCE,
- INVALID_CODE_POINT
- };
- /// Helper for get_sequence_x
- template <typename octet_iterator>
- utf_error increase_safely(octet_iterator& it, octet_iterator end) {
- if (++it == end) return NOT_ENOUGH_ROOM;
- if (!utf8::internal::is_trail(*it)) return INCOMPLETE_SEQUENCE;
- return UTF8_OK;
- }
- #define UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(IT, END) \
- { \
- utf_error ret = increase_safely(IT, END); \
- if (ret != UTF8_OK) return ret; \
- }
- /// get_sequence_x functions decode utf-8 sequences of the length x
- template <typename octet_iterator>
- utf_error get_sequence_1(octet_iterator& it, octet_iterator end,
- uint32_t& code_point) {
- if (it == end) return NOT_ENOUGH_ROOM;
- code_point = utf8::internal::mask8(*it);
- return UTF8_OK;
- }
- template <typename octet_iterator>
- utf_error get_sequence_2(octet_iterator& it, octet_iterator end,
- uint32_t& code_point) {
- if (it == end) return NOT_ENOUGH_ROOM;
- code_point = utf8::internal::mask8(*it);
- UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
- code_point = ((code_point << 6) & 0x7ff) + ((*it) & 0x3f);
- return UTF8_OK;
- }
- template <typename octet_iterator>
- utf_error get_sequence_3(octet_iterator& it, octet_iterator end,
- uint32_t& code_point) {
- if (it == end) return NOT_ENOUGH_ROOM;
- code_point = utf8::internal::mask8(*it);
- UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
- code_point = ((code_point << 12) & 0xffff) +
- ((utf8::internal::mask8(*it) << 6) & 0xfff);
- UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
- code_point += (*it) & 0x3f;
- return UTF8_OK;
- }
- template <typename octet_iterator>
- utf_error get_sequence_4(octet_iterator& it, octet_iterator end,
- uint32_t& code_point) {
- if (it == end) return NOT_ENOUGH_ROOM;
- code_point = utf8::internal::mask8(*it);
- UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
- code_point = ((code_point << 18) & 0x1fffff) +
- ((utf8::internal::mask8(*it) << 12) & 0x3ffff);
- UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
- code_point += (utf8::internal::mask8(*it) << 6) & 0xfff;
- UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)
- code_point += (*it) & 0x3f;
- return UTF8_OK;
- }
- #undef UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR
- template <typename octet_iterator>
- utf_error validate_next(octet_iterator& it, octet_iterator end,
- uint32_t& code_point) {
- // Save the original value of it so we can go back in case of failure
- // Of course, it does not make much sense with i.e. stream iterators
- octet_iterator original_it = it;
- uint32_t cp = 0;
- // Determine the sequence length based on the lead octet
- typedef typename std::iterator_traits<octet_iterator>::difference_type
- octet_difference_type;
- const octet_difference_type length = utf8::internal::sequence_length(it);
- // Get trail octets and calculate the code point
- utf_error err = UTF8_OK;
- switch (length) {
- case 0:
- return INVALID_LEAD;
- case 1:
- err = utf8::internal::get_sequence_1(it, end, cp);
- break;
- case 2:
- err = utf8::internal::get_sequence_2(it, end, cp);
- break;
- case 3:
- err = utf8::internal::get_sequence_3(it, end, cp);
- break;
- case 4:
- err = utf8::internal::get_sequence_4(it, end, cp);
- break;
- }
- if (err == UTF8_OK) {
- // Decoding succeeded. Now, security checks...
- if (utf8::internal::is_code_point_valid(cp)) {
- if (!utf8::internal::is_overlong_sequence(cp, length)) {
- // Passed! Return here.
- code_point = cp;
- ++it;
- return UTF8_OK;
- } else
- err = OVERLONG_SEQUENCE;
- } else
- err = INVALID_CODE_POINT;
- }
- // Failure branch - restore the original value of the iterator
- it = original_it;
- return err;
- }
- template <typename octet_iterator>
- inline utf_error validate_next(octet_iterator& it, octet_iterator end) {
- uint32_t ignored;
- return utf8::internal::validate_next(it, end, ignored);
- }
- } // namespace internal
- /// The library API - functions intended to be called by the users
- // Byte order mark
- const uint8_t bom[] = {0xef, 0xbb, 0xbf};
- template <typename octet_iterator>
- octet_iterator find_invalid(octet_iterator start, octet_iterator end) {
- octet_iterator result = start;
- while (result != end) {
- utf8::internal::utf_error err_code =
- utf8::internal::validate_next(result, end);
- if (err_code != internal::UTF8_OK) return result;
- }
- return result;
- }
- template <typename octet_iterator>
- inline bool is_valid(octet_iterator start, octet_iterator end) {
- return (utf8::find_invalid(start, end) == end);
- }
- template <typename octet_iterator>
- inline bool starts_with_bom(octet_iterator it, octet_iterator end) {
- return (((it != end) && (utf8::internal::mask8(*it++)) == bom[0]) &&
- ((it != end) && (utf8::internal::mask8(*it++)) == bom[1]) &&
- ((it != end) && (utf8::internal::mask8(*it)) == bom[2]));
- }
- // Deprecated in release 2.3
- template <typename octet_iterator>
- inline bool is_bom(octet_iterator it) {
- return ((utf8::internal::mask8(*it++)) == bom[0] &&
- (utf8::internal::mask8(*it++)) == bom[1] &&
- (utf8::internal::mask8(*it)) == bom[2]);
- }
- } // namespace utf8
- #endif // header guard
|