class-phpurlintextprocessor.php
389 lines
| 1 | <?php |
| 2 | |
| 3 | namespace WordPress\DataLiberation\URL; |
| 4 | |
| 5 | use WordPress\DataLiberation\BlockMarkup\URL; |
| 6 | use WP_HTML_Text_Replacement; |
| 7 | |
| 8 | /** |
| 9 | * Finds string fragments that look like URLs and allows replacing them. |
| 10 | * |
| 11 | * This class implements two stages of detection: |
| 12 | * |
| 13 | * 1. **A "thick" sieve** |
| 14 | * 2. **A "fine" sieve** |
| 15 | * |
| 16 | * The thick sieve uses a regular expression to match URL-like substrings. It matches too |
| 17 | * much and may yield false positives. |
| 18 | * |
| 19 | * The fine sieve filters out invalid candidates using a WHATWG-compliant parser so only |
| 20 | * real URLs are returned. |
| 21 | * |
| 22 | * ## URL Detection |
| 23 | * |
| 24 | * The thick sieve looks for URLs: |
| 25 | * |
| 26 | * * Starting with http://, https://, or //, e.g. //wp.org. |
| 27 | * * With no protocol, e.g. www.wp.org or wp.org/path |
| 28 | * |
| 29 | * Here's a list of matching-related rules, limitations, and assumptions: |
| 30 | * |
| 31 | * ### Protocols |
| 32 | * |
| 33 | * As a site migration tool, this processor only considers URLs with HTTP |
| 34 | * and HTTPS protocols. |
| 35 | * |
| 36 | * ### Domain names |
| 37 | * |
| 38 | * UTF-8 characters in the domain names are supported even if they're |
| 39 | * not encoded as punycode. For example, scanning the text: |
| 40 | * |
| 41 | * > Więcej na ł� |
| 42 | ka.pl |
| 43 | * |
| 44 | * Would yield `ł� |
| 45 | ka.pl` |
| 46 | * |
| 47 | * ### Paths |
| 48 | * |
| 49 | * The path is limited to ASCII characters, as per the URL specification. |
| 50 | * For example, scanning the text: |
| 51 | * |
| 52 | * > Visit the WordPress plugins directory https://w.org/plugins?ł� |
| 53 | ka=1 |
| 54 | * |
| 55 | * Would yield `https://w.org/plugins?`, not `https://w.org/plugins?ł� |
| 56 | ka=1`. |
| 57 | * However, scanning this text: |
| 58 | * |
| 59 | * > Visit the WordPress plugins directory https://w.org/plugins?%C5%82%C4%85ka=1 |
| 60 | * |
| 61 | * Would yield `https://w.org/plugins?%C5%82%C4%85ka=1`. |
| 62 | * |
| 63 | * ### Parenthesis treatment |
| 64 | * |
| 65 | * This scanner captures parentheses as a part of the path, query, or fragment, except |
| 66 | * when they're seen as the last character in the URL. For example, scanning the text: |
| 67 | * |
| 68 | * > Visit the WordPress plugins directory (https://w.org/plugins) |
| 69 | * |
| 70 | * Would yield `https://w.org/plugins`, but scanning the text: |
| 71 | * |
| 72 | * > Visit the WordPress plugins directory (https://w.org/plug(in)s |
| 73 | * |
| 74 | * Would yield `https://w.org/plug(in)s`. |
| 75 | * |
| 76 | * ### Rejecting URLs with embedded credentials |
| 77 | * |
| 78 | * `https://user:pass@wp.org` is not matched. Rewriting URLs that presume transferable |
| 79 | * credentials is hazardous and rarely correct for migrations. |
| 80 | * |
| 81 | * ### Reject non-HTTP(S) schemes |
| 82 | * |
| 83 | * Out of scope for site moves; all of these are rejected: |
| 84 | * `gopher://site.com`, `blob:afgh2-48189d`, `ahttp://site.com`, `mailto:user@site.com`, `file://asset.zip`. |
| 85 | * If we need additional schemes later, we can add them intentionally. |
| 86 | * |
| 87 | * ### Reject non‑absolute‑looking references |
| 88 | * |
| 89 | * While we do rely on a base URL, inputs like `::`, `/index.html`, `?query` are still ignored. |
| 90 | * Bare-domain forms like `mysite.org/?query` are still matched. |
| 91 | * |
| 92 | * ### Handle trailing punctuation sensibly |
| 93 | * |
| 94 | * `https://mysite.com/path/..` is interpreted as `https://mysite.com/path/` rather than |
| 95 | * collapsing to the origin. A final period is far more likely sentence punctuation than `../`. |
| 96 | * If a user truly writes `https://mysite.com/path/../`, we parse it as expected. |
| 97 | * |
| 98 | * ### Fuzzy matching for malformed ports |
| 99 | * |
| 100 | * * **WHATWG**: `"http://w.org:100000 plugins are in the directory" → failure`. |
| 101 | * * **Inline detection**: `"http://w.org:100000 plugins are in the directory" → "http://w.org/"` |
| 102 | * (truncate at the invalid port). |
| 103 | * |
| 104 | * This is a best‑effort extraction of the valid prefix rather than an all‑or‑nothing rejection. |
| 105 | * |
| 106 | * ### Whitespace handling |
| 107 | * |
| 108 | * * **WHATWG**: `"http://example\t.\norg" → "http://example.org/"`. |
| 109 | * * **Inline detection**: stops at the first whitespace, yielding `"http://example/"`. |
| 110 | * |
| 111 | * This reflects how URLs actually appear in text blocks where whitespace often terminates a link. |
| 112 | */ |
| 113 | class PHPURLInTextProcessor { |
| 114 | |
| 115 | private $text; |
| 116 | private $url_starts_at; |
| 117 | private $url_length; |
| 118 | private $bytes_already_parsed = 0; |
| 119 | /** |
| 120 | * @var string |
| 121 | */ |
| 122 | private $matched_url; |
| 123 | /** |
| 124 | * @var URL |
| 125 | */ |
| 126 | private $parsed_url; |
| 127 | private $did_prepend_protocol; |
| 128 | /** |
| 129 | * The base URL for the parsing algorithm. |
| 130 | * See https://url.spec.whatwg.org/. |
| 131 | * |
| 132 | * @var mixed|null |
| 133 | */ |
| 134 | private $base_url; |
| 135 | private $base_protocol; |
| 136 | |
| 137 | /** |
| 138 | * The regular expression pattern used for the matchin URL candidates |
| 139 | * from the text. |
| 140 | * |
| 141 | * @var string |
| 142 | */ |
| 143 | private $regex; |
| 144 | |
| 145 | /** |
| 146 | * @see \WP_HTML_Tag_Processor |
| 147 | * @var WP_HTML_Text_Replacement[] |
| 148 | */ |
| 149 | private $lexical_updates = array(); |
| 150 | |
| 151 | /** |
| 152 | * @var bool |
| 153 | * A flag to indicate whether the URL matching should be strict or not. |
| 154 | * If set to true, the matching will be strict, meaning it will only match URLs that strictly adhere to the pattern. |
| 155 | * If set to false, the matching will be more lenient, allowing for potential false positives. |
| 156 | */ |
| 157 | private $strict = false; |
| 158 | public function __construct( $text, $base_url = null ) { |
| 159 | $this->text = $text; |
| 160 | $this->base_url = $base_url; |
| 161 | $this->base_protocol = $base_url ? parse_url( $base_url, PHP_URL_SCHEME ) : null; |
| 162 | |
| 163 | $prefix = $this->strict ? '^' : ''; |
| 164 | $suffix = $this->strict ? '$' : ''; |
| 165 | |
| 166 | // Source: https://github.com/vstelmakh/url-highlight/blob/master/src/Matcher/Matcher.php. |
| 167 | $this->regex = '/' . $prefix . ' |
| 168 | (?: # scheme |
| 169 | (?<scheme>[a-z0-9\+]+?:)? # |
| 170 | (?:\/*) # The protocol may optionally be followed by one or more slashes |
| 171 | )? |
| 172 | (?: # userinfo |
| 173 | (?: |
| 174 | (?<=\/{2}) # prefixed with \/\/ |
| 175 | | # or |
| 176 | (?=[^\p{Sm}\p{Sc}\p{Sk}\p{P}]) # start with not: mathematical, currency, modifier symbol, punctuation |
| 177 | ) |
| 178 | (?<userinfo>[^\s<>@\/]+) # not: whitespace, < > @ \/ |
| 179 | @ # at |
| 180 | )? |
| 181 | (?=%|[^\p{Z}\p{Sm}\p{Sc}\p{Sk}\p{C}\p{P}]) # followed by valid host char |
| 182 | (?| # host |
| 183 | (?<host> # host prefixed by scheme or userinfo (less strict) |
| 184 | (?<=\/\/|@) # prefixed with \/\/ or @ |
| 185 | (?=[^\-]) # label start, not: - |
| 186 | (?:%|[^\p{Z}\p{Sm}\p{Sc}\p{Sk}\p{C}\p{P}]|-){1,63} # label not: whitespace, mathematical, currency, modifier symbol, control point, punctuation | except - |
| 187 | (?<=[^\-]) # label end, not: - |
| 188 | (?: # more label parts |
| 189 | \. |
| 190 | (?=[^\-]) # label start, not: - |
| 191 | (?<tld>(?:[^\p{Z}\p{Sm}\p{Sc}\p{Sk}\p{C}\p{P}]|-){1,63}) # label not: whitespace, mathematical, currency, modifier symbol, control point, punctuation | except - |
| 192 | (?<=[^\-]) # label end, not: - |
| 193 | )* |
| 194 | ) |
| 195 | | # or |
| 196 | (?<host> # host with tld (no scheme or userinfo) |
| 197 | (?=[^\-]) # label start, not: - |
| 198 | (?:%|[^\p{Z}\p{Sm}\p{Sc}\p{Sk}\p{C}\p{P}]|-){1,63} # label not: whitespace, mathematical, currency, modifier symbol, control point, punctuation | except - |
| 199 | (?<=[^\-]) # label end, not: - |
| 200 | (?: # more label parts |
| 201 | \. |
| 202 | (?=[^\-]) # label start, not: - |
| 203 | (?:%|[^\p{Z}\p{Sm}\p{Sc}\p{Sk}\p{C}\p{P}]|-){1,63} # label not: whitespace, mathematical, currency, modifier symbol, control point, punctuation | except - |
| 204 | (?<=[^\-]) # label end, not: - |
| 205 | )* |
| 206 | \.(?<tld>\w{2,63}) # tld |
| 207 | ) |
| 208 | ) |
| 209 | (?:\:(?<port>\d{1,5}(?!\d)))? # port |
| 210 | (?<path> # path, query, fragment |
| 211 | [\/?#] # prefixed with \/ or ? or # |
| 212 | [^\s<>]* # any chars except whitespace and <> |
| 213 | (?<=[^\s<>({\[`!;:\'".,?«»“”‘’]) # end with not a space or some punctuation chars |
| 214 | )? |
| 215 | ' . $suffix . '/ixuJ'; |
| 216 | } |
| 217 | |
| 218 | /** |
| 219 | * @return string |
| 220 | */ |
| 221 | public function next_url() { |
| 222 | while ( true ) { |
| 223 | $this->matched_url = null; |
| 224 | $this->parsed_url = null; |
| 225 | $this->url_starts_at = null; |
| 226 | $this->url_length = null; |
| 227 | $this->did_prepend_protocol = false; |
| 228 | |
| 229 | /** |
| 230 | * Thick sieve – eagerly match things that look like URLs but turn out to not be URLs in the end. |
| 231 | */ |
| 232 | $matches = array(); |
| 233 | $found = preg_match( $this->regex, $this->text, $matches, PREG_OFFSET_CAPTURE, $this->bytes_already_parsed ); |
| 234 | if ( 1 !== $found ) { |
| 235 | return false; |
| 236 | } |
| 237 | |
| 238 | $this->matched_url = $matches[0][0]; |
| 239 | // Do not consider just :: as a URL. |
| 240 | if ( '::' === $this->matched_url ) { |
| 241 | continue; |
| 242 | } |
| 243 | if ( |
| 244 | ')' === $this->matched_url[ strlen( $this->matched_url ) - 1 ] || |
| 245 | '.' === $this->matched_url[ strlen( $this->matched_url ) - 1 ] |
| 246 | ) { |
| 247 | $this->matched_url = substr( $this->matched_url, 0, - 1 ); |
| 248 | } |
| 249 | $url_starts_at = $matches[0][1]; |
| 250 | $this->bytes_already_parsed = $url_starts_at + strlen( $this->matched_url ); |
| 251 | |
| 252 | $had_protocol = WPURL::has_http_https_protocol( $this->matched_url ); |
| 253 | |
| 254 | $preprocessed_url = $this->matched_url; |
| 255 | if ( $this->base_url && $this->base_protocol && ! $had_protocol ) { |
| 256 | $preprocessed_url = WPURL::ensure_protocol( $preprocessed_url, $this->base_protocol ); |
| 257 | $this->did_prepend_protocol = true; |
| 258 | } |
| 259 | |
| 260 | /* |
| 261 | * Extra fine sieve – parse the candidates using a WHATWG-compliant parser to rule out false positives. |
| 262 | */ |
| 263 | $parsed_url = WPURL::parse( $preprocessed_url, $this->base_url ); |
| 264 | if ( false === $parsed_url ) { |
| 265 | continue; |
| 266 | } |
| 267 | |
| 268 | // Only consider HTTP and HTTPS URLs. |
| 269 | if ( $parsed_url->protocol && ! in_array( $parsed_url->protocol, array( 'http:', 'https:' ), true ) ) { |
| 270 | continue; |
| 271 | } |
| 272 | |
| 273 | // Disregard URLs with auth details. |
| 274 | if ( $parsed_url->username || $parsed_url->password ) { |
| 275 | continue; |
| 276 | } |
| 277 | |
| 278 | // Additional rigor for URLs that are not explicitly preceded by a double slash. |
| 279 | if ( ! $had_protocol ) { |
| 280 | /* |
| 281 | * Skip TLDs that are not in the public suffix. |
| 282 | * This reduces false positives like `index.html` or `plugins.php`. |
| 283 | * |
| 284 | * See https://publicsuffix.org/. |
| 285 | */ |
| 286 | $last_dot_position = strrpos( $parsed_url->hostname, '.' ); |
| 287 | if ( false === $last_dot_position ) { |
| 288 | /* |
| 289 | * Oh, there was no dot in the hostname AND no double slash at |
| 290 | * the beginning! Let's assume this isn't a valid URL and move on. |
| 291 | * @TODO: Explore updating the regular expression above to avoid matching |
| 292 | * URLs without a dot in the hostname when they're not preceeded |
| 293 | * by a protocol. |
| 294 | */ |
| 295 | continue; |
| 296 | } |
| 297 | |
| 298 | $tld = substr( $parsed_url->hostname, $last_dot_position + 1 ); |
| 299 | if ( ! WPURL::is_known_public_domain( $tld ) ) { |
| 300 | // This TLD is not in the public suffix list. It's not a valid domain name. |
| 301 | continue; |
| 302 | } |
| 303 | } |
| 304 | |
| 305 | $this->parsed_url = $parsed_url; |
| 306 | $this->url_starts_at = $url_starts_at; |
| 307 | $this->url_length = strlen( $matches[0][0] ); |
| 308 | |
| 309 | return true; |
| 310 | } |
| 311 | } |
| 312 | |
| 313 | public function get_raw_url() { |
| 314 | return $this->matched_url ?? false; |
| 315 | } |
| 316 | |
| 317 | public function get_parsed_url() { |
| 318 | if ( null === $this->parsed_url ) { |
| 319 | return false; |
| 320 | } |
| 321 | |
| 322 | return $this->parsed_url; |
| 323 | } |
| 324 | |
| 325 | public function set_raw_url( $new_url ) { |
| 326 | if ( null === $this->matched_url ) { |
| 327 | return false; |
| 328 | } |
| 329 | if ( $this->did_prepend_protocol ) { |
| 330 | $new_url = substr( $new_url, strpos( $new_url, '://' ) + 3 ); |
| 331 | } |
| 332 | $this->matched_url = $new_url; |
| 333 | $this->lexical_updates[ $this->url_starts_at ] = new WP_HTML_Text_Replacement( |
| 334 | $this->url_starts_at, |
| 335 | $this->url_length, |
| 336 | $new_url |
| 337 | ); |
| 338 | |
| 339 | return true; |
| 340 | } |
| 341 | |
| 342 | private function apply_lexical_updates() { |
| 343 | if ( ! count( $this->lexical_updates ) ) { |
| 344 | return 0; |
| 345 | } |
| 346 | |
| 347 | /* |
| 348 | * Attribute updates can be enqueued in any order but updates |
| 349 | * to the document must occur in lexical order; that is, each |
| 350 | * replacement must be made before all others which follow it |
| 351 | * at later string indices in the input document. |
| 352 | * |
| 353 | * Sorting avoid making out-of-order replacements which |
| 354 | * can lead to mangled output, partially-duplicated |
| 355 | * attributes, and overwritten attributes. |
| 356 | */ |
| 357 | |
| 358 | ksort( $this->lexical_updates ); |
| 359 | |
| 360 | $bytes_already_copied = 0; |
| 361 | $output_buffer = ''; |
| 362 | foreach ( $this->lexical_updates as $diff ) { |
| 363 | $shift = strlen( $diff->text ) - $diff->length; |
| 364 | |
| 365 | // Adjust the cursor position by however much an update affects it. |
| 366 | if ( $diff->start < $this->bytes_already_parsed ) { |
| 367 | $this->bytes_already_parsed += $shift; |
| 368 | } |
| 369 | |
| 370 | $output_buffer .= substr( $this->text, $bytes_already_copied, $diff->start - $bytes_already_copied ); |
| 371 | if ( $diff->start === $this->url_starts_at ) { |
| 372 | $this->url_starts_at = strlen( $output_buffer ); |
| 373 | $this->url_length = strlen( $diff->text ); |
| 374 | } |
| 375 | $output_buffer .= $diff->text; |
| 376 | $bytes_already_copied = $diff->start + $diff->length; |
| 377 | } |
| 378 | |
| 379 | $this->text = $output_buffer . substr( $this->text, $bytes_already_copied ); |
| 380 | $this->lexical_updates = array(); |
| 381 | } |
| 382 | |
| 383 | public function get_updated_text() { |
| 384 | $this->apply_lexical_updates(); |
| 385 | |
| 386 | return $this->text; |
| 387 | } |
| 388 | } |
| 389 |