LICENSE.md
6 days ago
README.md
6 days ago
compat-utf8.php
6 days ago
composer.json
6 days ago
utf8-encoder.php
6 days ago
utf8.php
6 days ago
utf8.php
228 lines
| 1 | <?php |
| 2 | |
| 3 | namespace WordPress\Encoding; |
| 4 | |
| 5 | use function WordPress\Encoding\compat\_wp_is_valid_utf8_fallback; |
| 6 | use function WordPress\Encoding\compat\_wp_scrub_utf8_fallback; |
| 7 | use function WordPress\Encoding\compat\_wp_has_noncharacters_fallback; |
| 8 | |
| 9 | if ( extension_loaded( 'mbstring' ) ) : |
| 10 | /** |
| 11 | * Determines if a given byte string represents a valid UTF-8 encoding. |
| 12 | * |
| 13 | * Note that it’s unlikely for non-UTF-8 data to validate as UTF-8, but |
| 14 | * it is still possible. Many texts are simultaneously valid UTF-8, |
| 15 | * valid US-ASCII, and valid ISO-8859-1 (`latin1`). |
| 16 | * |
| 17 | * Example: |
| 18 | * |
| 19 | * true === wp_is_valid_utf8( '' ); |
| 20 | * true === wp_is_valid_utf8( 'just a test' ); |
| 21 | * true === wp_is_valid_utf8( "\xE2\x9C\x8F" ); // Pencil, U+270F. |
| 22 | * true === wp_is_valid_utf8( "\u{270F}" ); // Pencil, U+270F. |
| 23 | * true === wp_is_valid_utf8( '✏' ); // Pencil, U+270F. |
| 24 | * |
| 25 | * false === wp_is_valid_utf8( "just \xC0 test" ); // Invalid bytes. |
| 26 | * false === wp_is_valid_utf8( "\xE2\x9C" ); // Invalid/incomplete sequences. |
| 27 | * false === wp_is_valid_utf8( "\xC1\xBF" ); // Overlong sequences. |
| 28 | * false === wp_is_valid_utf8( "\xED\xB0\x80" ); // Surrogate halves. |
| 29 | * false === wp_is_valid_utf8( "B\xFCch" ); // ISO-8859-1 high-bytes. |
| 30 | * // E.g. The “ü” in ISO-8859-1 is a single byte 0xFC, |
| 31 | * // but in UTF-8 is the two-byte sequence 0xC3 0xBC. |
| 32 | * |
| 33 | * A “valid” string consists of “well-formed UTF-8 code unit sequence[s],” meaning |
| 34 | * that the bytes conform to the UTF-8 encoding scheme, all characters use the minimal |
| 35 | * byte sequence required by UTF-8, and that no sequence encodes a UTF-16 surrogate |
| 36 | * code point or any character above the representable range. |
| 37 | * |
| 38 | * @see https://www.unicode.org/versions/Unicode16.0.0/core-spec/chapter-3/#G32860 |
| 39 | * |
| 40 | * @since 6.9.0 |
| 41 | * |
| 42 | * @param string $bytes String which might contain text encoded as UTF-8. |
| 43 | * @return bool Whether the provided bytes can decode as valid UTF-8. |
| 44 | */ |
| 45 | function wp_is_valid_utf8( string $bytes ): bool { |
| 46 | return mb_check_encoding( $bytes, 'UTF-8' ); |
| 47 | } |
| 48 | else : |
| 49 | /** |
| 50 | * Fallback function for validating UTF-8. |
| 51 | * |
| 52 | * @ignore |
| 53 | * @private |
| 54 | * |
| 55 | * @since 6.9.0 |
| 56 | */ |
| 57 | // phpcs:ignore Universal.NamingConventions.NoReservedKeywordParameterNames.stringFound |
| 58 | function wp_is_valid_utf8( string $string ): bool { |
| 59 | return _wp_is_valid_utf8_fallback( $string ); |
| 60 | } |
| 61 | endif; |
| 62 | |
| 63 | if ( |
| 64 | extension_loaded( 'mbstring' ) && |
| 65 | // Maximal subpart substitution introduced by php/php-src@04e59c916f12b322ac55f22314e31bd0176d01cb. |
| 66 | version_compare( PHP_VERSION, '8.1.6', '>=' ) |
| 67 | ) : |
| 68 | /** |
| 69 | * Replaces ill-formed UTF-8 byte sequences with the Unicode Replacement Character. |
| 70 | * |
| 71 | * Knowing what to do in the presence of text encoding issues can be complicated. |
| 72 | * This function replaces invalid spans of bytes to neutralize any corruption that |
| 73 | * may be there and prevent it from causing further problems downstream. |
| 74 | * |
| 75 | * However, it’s not always ideal to replace those bytes. In some settings it may |
| 76 | * be best to leave the invalid bytes in the string so that downstream code can handle |
| 77 | * them in a specific way. Replacing the bytes too early, like escaping for HTML too |
| 78 | * early, can introduce other forms of corruption and data loss. |
| 79 | * |
| 80 | * When in doubt, use this function to replace spans of invalid bytes. |
| 81 | * |
| 82 | * Replacement follows the “maximal subpart” algorithm for secure and interoperable |
| 83 | * strings. This can lead to sequences of multiple replacement characters in a row. |
| 84 | * |
| 85 | * Example: |
| 86 | * |
| 87 | * // Valid strings come through unchanged. |
| 88 | * 'test' === wp_scrub_utf8( 'test' ); |
| 89 | * |
| 90 | * // Invalid sequences of bytes are replaced. |
| 91 | * $invalid = "the byte \xC0 is never allowed in a UTF-8 string."; |
| 92 | * "the byte \u{FFFD} is never allowed in a UTF-8 string." === wp_scrub_utf8( $invalid, true ); |
| 93 | * 'the byte � is never allowed in a UTF-8 string.' === wp_scrub_utf8( $invalid, true ); |
| 94 | * |
| 95 | * // Maximal subparts are replaced individually. |
| 96 | * '.�.' === wp_scrub_utf8( ".\xC0." ); // C0 is never valid. |
| 97 | * '.�.' === wp_scrub_utf8( ".\xE2\x8C." ); // Missing A3 at end. |
| 98 | * '.��.' === wp_scrub_utf8( ".\xE2\x8C\xE2\x8C." ); // Maximal subparts replaced separately. |
| 99 | * '.��.' === wp_scrub_utf8( ".\xC1\xBF." ); // Overlong sequence. |
| 100 | * '.���.' === wp_scrub_utf8( ".\xED\xA0\x80." ); // Surrogate half. |
| 101 | * |
| 102 | * Note! The Unicode Replacement Character is itself a Unicode character (U+FFFD). |
| 103 | * Once a span of invalid bytes has been replaced by one, it will not be possible |
| 104 | * to know whether the replacement character was originally intended to be there |
| 105 | * or if it is the result of scrubbing bytes. It is ideal to leave replacement for |
| 106 | * display only, but some contexts (e.g. generating XML or passing data into a |
| 107 | * large language model) require valid input strings. |
| 108 | * |
| 109 | * @since 6.9.0 |
| 110 | * |
| 111 | * @see https://www.unicode.org/versions/Unicode16.0.0/core-spec/chapter-5/#G40630 |
| 112 | * |
| 113 | * @param string $text String which is assumed to be UTF-8 but may contain invalid sequences of bytes. |
| 114 | * @return string Input text with invalid sequences of bytes replaced with the Unicode replacement character. |
| 115 | */ |
| 116 | function wp_scrub_utf8( $text ) { |
| 117 | /* |
| 118 | * While it looks like setting the substitute character could fail, |
| 119 | * the internal PHP code will never fail when provided a valid |
| 120 | * code point as a number. In this case, there’s no need to check |
| 121 | * its return value to see if it succeeded. |
| 122 | */ |
| 123 | $prev_replacement_character = mb_substitute_character(); |
| 124 | mb_substitute_character( 0xFFFD ); |
| 125 | $scrubbed = mb_scrub( $text, 'UTF-8' ); |
| 126 | mb_substitute_character( $prev_replacement_character ); |
| 127 | |
| 128 | return $scrubbed; |
| 129 | } |
| 130 | else : |
| 131 | /** |
| 132 | * Fallback function for scrubbing UTF-8. |
| 133 | * |
| 134 | * @ignore |
| 135 | * @private |
| 136 | * |
| 137 | * @since 6.9.0 |
| 138 | */ |
| 139 | function wp_scrub_utf8( $text ) { |
| 140 | return _wp_scrub_utf8_fallback( $text ); |
| 141 | } |
| 142 | endif; |
| 143 | |
| 144 | function _wp_can_use_pcre_u( $set = null ) { |
| 145 | static $utf8_pcre = 'reset'; |
| 146 | |
| 147 | if ( null !== $set ) { |
| 148 | $utf8_pcre = $set; |
| 149 | } |
| 150 | |
| 151 | if ( 'reset' === $utf8_pcre ) { |
| 152 | // phpcs:ignore WordPress.PHP.NoSilencedErrors.Discouraged -- intentional error generated to detect PCRE/u support. |
| 153 | $utf8_pcre = @preg_match( '/^./u', 'a' ); |
| 154 | } |
| 155 | |
| 156 | return $utf8_pcre; |
| 157 | } |
| 158 | |
| 159 | if ( _wp_can_use_pcre_u() ) : |
| 160 | /** |
| 161 | * Returns whether the given string contains Unicode noncharacters. |
| 162 | * |
| 163 | * XML recommends against using noncharacters and HTML forbids their |
| 164 | * use in attribute names. Unicode recommends that they not be used |
| 165 | * in open exchange of data. |
| 166 | * |
| 167 | * Noncharacters are code points within the following ranges: |
| 168 | * - U+FDD0–U+FDEF |
| 169 | * - U+FFFE–U+FFFF |
| 170 | * - U+1FFFE, U+1FFFF, U+2FFFE, U+2FFFF, …, U+10FFFE, U+10FFFF |
| 171 | * |
| 172 | * @see https://www.unicode.org/versions/Unicode17.0.0/core-spec/chapter-23/#G12612 |
| 173 | * @see https://www.w3.org/TR/xml/#charsets |
| 174 | * @see https://html.spec.whatwg.org/#attributes-2 |
| 175 | * |
| 176 | * @since 6.9.0 |
| 177 | * |
| 178 | * @param string $text Are there noncharacters in this string? |
| 179 | * @return bool Whether noncharacters were found in the string. |
| 180 | */ |
| 181 | function wp_has_noncharacters( string $text ): bool { |
| 182 | return 1 === preg_match( |
| 183 | '/[\x{FDD0}-\x{FDEF}\x{FFFE}\x{FFFF}\x{1FFFE}\x{1FFFF}\x{2FFFE}\x{2FFFF}\x{3FFFE}\x{3FFFF}\x{4FFFE}\x{4FFFF}\x{5FFFE}\x{5FFFF}\x{6FFFE}\x{6FFFF}\x{7FFFE}\x{7FFFF}\x{8FFFE}\x{8FFFF}\x{9FFFE}\x{9FFFF}\x{AFFFE}\x{AFFFF}\x{BFFFE}\x{BFFFF}\x{CFFFE}\x{CFFFF}\x{DFFFE}\x{DFFFF}\x{EFFFE}\x{EFFFF}\x{FFFFE}\x{FFFFF}\x{10FFFE}\x{10FFFF}]/u', |
| 184 | $text |
| 185 | ); |
| 186 | } |
| 187 | else : |
| 188 | /** |
| 189 | * Fallback function for detecting noncharacters in a text. |
| 190 | * |
| 191 | * @ignore |
| 192 | * @private |
| 193 | * |
| 194 | * @since 6.9.0 |
| 195 | */ |
| 196 | function wp_has_noncharacters( string $text ): bool { |
| 197 | return _wp_has_noncharacters_fallback( $text ); |
| 198 | } |
| 199 | endif; |
| 200 | |
| 201 | /** |
| 202 | * Convert a UTF-8 byte sequence to its Unicode codepoint. |
| 203 | * |
| 204 | * @param string $character UTF-8 encoded byte sequence representing a single Unicode character. |
| 205 | * |
| 206 | * @return int Unicode codepoint. |
| 207 | */ |
| 208 | function utf8_ord( string $character ): int { |
| 209 | // Convert the byte sequence to its binary representation. |
| 210 | $bytes = unpack( 'C*', $character ); |
| 211 | |
| 212 | // Initialize the codepoint. |
| 213 | $codepoint = 0; |
| 214 | |
| 215 | // Calculate the codepoint based on the number of bytes. |
| 216 | if ( 1 === count( $bytes ) ) { |
| 217 | $codepoint = $bytes[1]; |
| 218 | } elseif ( 2 === count( $bytes ) ) { |
| 219 | $codepoint = ( ( $bytes[1] & 0x1F ) << 6 ) | ( $bytes[2] & 0x3F ); |
| 220 | } elseif ( 3 === count( $bytes ) ) { |
| 221 | $codepoint = ( ( $bytes[1] & 0x0F ) << 12 ) | ( ( $bytes[2] & 0x3F ) << 6 ) | ( $bytes[3] & 0x3F ); |
| 222 | } elseif ( 4 === count( $bytes ) ) { |
| 223 | $codepoint = ( ( $bytes[1] & 0x07 ) << 18 ) | ( ( $bytes[2] & 0x3F ) << 12 ) | ( ( $bytes[3] & 0x3F ) << 6 ) | ( $bytes[4] & 0x3F ); |
| 224 | } |
| 225 | |
| 226 | return $codepoint; |
| 227 | } |
| 228 |