class-blockswithmetadataentityreader.php
6 days ago
class-databasecontententityreader.php
6 days ago
class-databaserowsentityreader.php
6 days ago
class-entityreaderiterator.php
6 days ago
class-epubentityreader.php
6 days ago
class-filesystementityreader.php
6 days ago
class-htmlentityreader.php
6 days ago
class-wxrentityreader.php
6 days ago
interface-entity-reader.php
6 days ago
class-wxrentityreader.php
974 lines
| 1 | <?php |
| 2 | |
| 3 | namespace WordPress\DataLiberation\EntityReader; |
| 4 | |
| 5 | use WordPress\ByteStream\ReadStream\ByteReadStream; |
| 6 | use WordPress\DataLiberation\ImportEntity; |
| 7 | use WordPress\XML\XMLProcessor; |
| 8 | use WordPress\XML\XMLUnsupportedException; |
| 9 | |
| 10 | /** |
| 11 | * Data Liberation API: WP_WXR_Entity_Reader class |
| 12 | * |
| 13 | * Reads WordPress eXtended RSS (WXR) files and emits entities like posts, |
| 14 | * comments, users, and terms. Enables efficient processing of large WXR |
| 15 | * files without loading everything into memory. |
| 16 | * |
| 17 | * Note this is just a reader. It doesn't import any data into WordPress. It |
| 18 | * only reads meaningful entities from the WXR file. |
| 19 | * |
| 20 | * ## Design goals |
| 21 | * |
| 22 | * WP_WXR_Entity_Reader is built with the following characteristics in mind: |
| 23 | * |
| 24 | * * Speed – it should be as fast as possible |
| 25 | * * No PHP extensions required – it can run on any PHP installation |
| 26 | * * Reliability – no random crashes when encountering malformed XML or UTF-8 sequences |
| 27 | * * Low, predictable memory footprint to support 1000GB+ WXR files |
| 28 | * * Ability to pause, finish execution, and resume later, e.g. after a fatal error |
| 29 | * |
| 30 | * ## Implementation |
| 31 | * |
| 32 | * `WP_WXR_Entity_Reader` uses the `WP_XML_Processor` to find XML tags representing meaningful |
| 33 | * WordPress entities. The reader knows the WXR schema and only looks for relevant elements. |
| 34 | * For example, it knows that posts are stored in `rss > channel > item` and comments are |
| 35 | * stored in `rss > channel > item > `wp:comment`. |
| 36 | * |
| 37 | * The `$wxr->next_entity()` method stream-parses the next entity from the WXR document and |
| 38 | * exposes it to the API consumer via `$wxr->get_entity_type()` and `$wxr->get_entity_date()`. |
| 39 | * The next call to `$wxr->next_entity()` remembers where the parsing has stopped and parses |
| 40 | * the next entity after that point. |
| 41 | * |
| 42 | * Example: |
| 43 | * |
| 44 | * $reader = WP_WXR_Entity_Reader::create_for_streaming(); |
| 45 | * |
| 46 | * // Add data as it becomes available |
| 47 | * $reader->append_bytes( fread( $file_handle, 65536 ) ); |
| 48 | * |
| 49 | * // Process entities |
| 50 | * while ( $reader->next_entity() ) { |
| 51 | * switch ( $wxr_reader->get_entity_type() ) { |
| 52 | * case 'post': |
| 53 | * // ... process post ... |
| 54 | * break; |
| 55 | * |
| 56 | * case 'comment': |
| 57 | * // ... process comment ... |
| 58 | * break; |
| 59 | * |
| 60 | * case 'site_option': |
| 61 | * // ... process site option ... |
| 62 | * break; |
| 63 | * |
| 64 | * // ... process other entity types ... |
| 65 | * } |
| 66 | * } |
| 67 | * |
| 68 | * // Check if we need more input |
| 69 | * if ( $reader->is_paused_at_incomplete_input() ) { |
| 70 | * // Add more data and continue processing |
| 71 | * $reader->append_bytes( fread( $file_handle, 65536 ) ); |
| 72 | * } |
| 73 | * |
| 74 | * The next_entity() -> fread -> break usage pattern may seem a bit tedious. This is expected. Even |
| 75 | * if the WXR parsing part of the WP_WXR_Entity_Reader offers a high-level API, working with byte streams |
| 76 | * requires reasoning on a much lower level. The StreamChain class shipped in this repository will |
| 77 | * make the API consumption easier with its transformation–oriented API for chaining data processors. |
| 78 | * |
| 79 | * Similarly to `WP_XML_Processor`, the `WP_WXR_Entity_Reader` enters a paused state when it doesn't |
| 80 | * have enough XML bytes to parse the entire entity. |
| 81 | * |
| 82 | * ## Caveats |
| 83 | * |
| 84 | * ### Extensibility |
| 85 | * |
| 86 | * `WP_WXR_Entity_Reader` ignores any XML elements it doesn't recognize. The WXR format is extensible |
| 87 | * so in the future the reader may start supporting registration of custom handlers for unknown |
| 88 | * tags in the future. |
| 89 | * |
| 90 | * ### Nested entities intertwined with data |
| 91 | * |
| 92 | * `WP_WXR_Entity_Reader` flushes the current entity whenever another entity starts. The upside is |
| 93 | * simplicity and a tiny memory footprint. The downside is that it's possible to craft a WXR |
| 94 | * document where some information would be lost. For example: |
| 95 | * |
| 96 | * ```xml |
| 97 | * <rss> |
| 98 | * <channel> |
| 99 | * <item> |
| 100 | * <title>Page with comments</title> |
| 101 | * <link>http://wpthemetestdata.wordpress.com/about/page-with-comments/</link> |
| 102 | * <wp:postmeta> |
| 103 | * <wp:meta_key>_wp_page_template</wp:meta_key> |
| 104 | * <wp:meta_value><![CDATA[default]]></wp:meta_value> |
| 105 | * </wp:postmeta> |
| 106 | * <wp:post_id>146</wp:post_id> |
| 107 | * </item> |
| 108 | * </channel> |
| 109 | * </rss> |
| 110 | * ``` |
| 111 | * |
| 112 | * `WP_WXR_Entity_Reader` would accumulate post data until the `wp:post_meta` tag. Then it would emit a |
| 113 | * `post` entity and accumulate the meta information until the `</wp:postmeta>` closer. Then it |
| 114 | * would advance to `<wp:post_id>` and **ignore it**. |
| 115 | * |
| 116 | * This is not a problem in all the `.wxr` files I saw. Still, it is important to note this limitation. |
| 117 | * It is possible there is a `.wxr` generator somewhere out there that intertwines post fields with post |
| 118 | * meta and comments. If this ever comes up, we could: |
| 119 | * |
| 120 | * * Emit the `post` entity first, then all the nested entities, and then emit a special `post_update` entity. |
| 121 | * * Do multiple passes over the WXR file – one for each level of nesting, e.g. 1. Insert posts, 2. Insert Comments, 3. Insert comment meta |
| 122 | * |
| 123 | * Buffering all the post meta and comments seems like a bad idea – there might be gigabytes of data. |
| 124 | * |
| 125 | * ## Remaining work |
| 126 | * |
| 127 | * @TODO: |
| 128 | * |
| 129 | * - Revisit the need to implement the Iterator interface. |
| 130 | * |
| 131 | * @since WP_VERSION |
| 132 | */ |
| 133 | class WXREntityReader implements EntityReader { |
| 134 | |
| 135 | /** |
| 136 | * The XML processor used to parse the WXR file. |
| 137 | * |
| 138 | * @since WP_VERSION |
| 139 | * @var WP_XML_Processor |
| 140 | */ |
| 141 | private $xml; |
| 142 | |
| 143 | /** |
| 144 | * The name of the XML tag containing information about the WordPress entity |
| 145 | * currently being extracted from the WXR file. |
| 146 | * |
| 147 | * @since WP_VERSION |
| 148 | * @var string|null |
| 149 | */ |
| 150 | private $entity_tag; |
| 151 | |
| 152 | /** |
| 153 | * The name of the current WordPress entity, such as 'post' or 'comment'. |
| 154 | * |
| 155 | * @since WP_VERSION |
| 156 | * @var string|null |
| 157 | */ |
| 158 | private $entity_type; |
| 159 | |
| 160 | /** |
| 161 | * The data accumulated for the current entity. |
| 162 | * |
| 163 | * @since WP_VERSION |
| 164 | * @var array |
| 165 | */ |
| 166 | private $entity_data; |
| 167 | |
| 168 | /** |
| 169 | * The byte offset of the current entity in the original input stream. |
| 170 | * |
| 171 | * @since WP_VERSION |
| 172 | * @var int |
| 173 | */ |
| 174 | private $entity_opener_byte_offset; |
| 175 | |
| 176 | /** |
| 177 | * Whether the current entity has been emitted. |
| 178 | * |
| 179 | * @since WP_VERSION |
| 180 | * @var bool |
| 181 | */ |
| 182 | private $entity_finished = false; |
| 183 | |
| 184 | /** |
| 185 | * The number of entities read so far. |
| 186 | * |
| 187 | * @since WP_VERSION |
| 188 | * @var int |
| 189 | */ |
| 190 | private $entities_read_so_far = 0; |
| 191 | |
| 192 | /** |
| 193 | * The attributes from the last opening tag. |
| 194 | * |
| 195 | * @since WP_VERSION |
| 196 | * @var array |
| 197 | */ |
| 198 | private $last_opener_attributes = array(); |
| 199 | |
| 200 | /** |
| 201 | * The ID of the last processed post. |
| 202 | * |
| 203 | * @since WP_VERSION |
| 204 | * @var int|null |
| 205 | */ |
| 206 | private $last_post_id = null; |
| 207 | |
| 208 | /** |
| 209 | * The ID of the last processed comment. |
| 210 | * |
| 211 | * @since WP_VERSION |
| 212 | * @var int|null |
| 213 | */ |
| 214 | private $last_comment_id = null; |
| 215 | |
| 216 | /** |
| 217 | * Buffer for accumulating text content between tags. |
| 218 | * |
| 219 | * @since WP_VERSION |
| 220 | * @var string |
| 221 | */ |
| 222 | private $text_buffer = ''; |
| 223 | |
| 224 | /** |
| 225 | * Stream to pull bytes from when the input bytes are exhausted. |
| 226 | * |
| 227 | * @var WP_Byte_Producer |
| 228 | */ |
| 229 | private $upstream; |
| 230 | |
| 231 | /** |
| 232 | * Whether the reader has finished processing the input stream. |
| 233 | * |
| 234 | * @var bool |
| 235 | */ |
| 236 | private $is_finished = false; |
| 237 | |
| 238 | /** |
| 239 | * Mapping of WXR tags representing site options to their WordPress options names. |
| 240 | * These tags are only matched if they are children of the <channel> element. |
| 241 | * |
| 242 | * @since WP_VERSION |
| 243 | * @var array |
| 244 | */ |
| 245 | private $known_site_options = array(); |
| 246 | |
| 247 | /** |
| 248 | * Mapping of WXR tags to their corresponding entity types and field mappings. |
| 249 | * |
| 250 | * @since WP_VERSION |
| 251 | * @var array |
| 252 | */ |
| 253 | private $known_entities = array(); |
| 254 | |
| 255 | public static function create( ?ByteReadStream $upstream = null, $cursor = null, $options = array() ) { |
| 256 | $xml_cursor = null; |
| 257 | if ( null !== $cursor ) { |
| 258 | $cursor = json_decode( $cursor, true ); |
| 259 | if ( false === $cursor ) { |
| 260 | _doing_it_wrong( |
| 261 | __METHOD__, |
| 262 | 'Invalid cursor provided for WP_WXR_Entity_Reader::create().', |
| 263 | null |
| 264 | ); |
| 265 | |
| 266 | return false; |
| 267 | } |
| 268 | $xml_cursor = $cursor['xml']; |
| 269 | } |
| 270 | |
| 271 | $xml = XMLProcessor::create_for_streaming( '', $xml_cursor ); |
| 272 | $reader = new WXREntityReader( $xml, $options ); |
| 273 | if ( null !== $cursor ) { |
| 274 | $reader->last_post_id = $cursor['last_post_id']; |
| 275 | $reader->last_comment_id = $cursor['last_comment_id']; |
| 276 | } |
| 277 | if ( null !== $upstream ) { |
| 278 | $reader->connect_upstream( $upstream ); |
| 279 | if ( null !== $cursor ) { |
| 280 | if ( ! isset( $cursor['upstream'] ) ) { |
| 281 | // No upstream cursor means we've processed the |
| 282 | // entire input stream. |
| 283 | $xml->input_finished(); |
| 284 | $xml->next_token(); |
| 285 | } else { |
| 286 | $upstream->seek( $cursor['upstream'] ); |
| 287 | } |
| 288 | } |
| 289 | } |
| 290 | |
| 291 | return $reader; |
| 292 | } |
| 293 | |
| 294 | /** |
| 295 | * Constructor. |
| 296 | * |
| 297 | * @param XMLProcessor $xml The XML processor to use. |
| 298 | * |
| 299 | * @since WP_VERSION |
| 300 | */ |
| 301 | protected function __construct( XMLProcessor $xml, $options = array() ) { |
| 302 | $this->xml = $xml; |
| 303 | |
| 304 | if ( isset( $options['known_site_options'] ) || isset( $options['known_entities'] ) ) { |
| 305 | $this->known_site_options = isset( $options['known_site_options'] ) ? $options['known_site_options'] : array(); |
| 306 | $this->known_entities = isset( $options['known_entities'] ) ? $options['known_entities'] : array(); |
| 307 | return; |
| 308 | } |
| 309 | |
| 310 | // Every XML element is a combination of a long-form namespace and a |
| 311 | // local element name, e.g. a syntax <wp:post_id> could actually refer |
| 312 | // to a (https://wordpress.org/export/1.0/, post_id) element. |
| 313 | // |
| 314 | // Namespaces are paramount for parsing XML and cannot be ignored. Elements |
| 315 | // element must be matched based on both their namespace and local name. |
| 316 | // |
| 317 | // Unfortunately, different WXR files defined the `wp` namespace in a different way. |
| 318 | // Folks use a mixture of HTTP vs HTTPS protocols and version numbers. We must |
| 319 | // account for all possible options to parse these documents correctly. |
| 320 | $wxr_namespaces = array( |
| 321 | 'http://wordpress.org/export/1.0/', |
| 322 | 'https://wordpress.org/export/1.0/', |
| 323 | 'http://wordpress.org/export/1.1/', |
| 324 | 'https://wordpress.org/export/1.1/', |
| 325 | 'http://wordpress.org/export/1.2/', |
| 326 | 'https://wordpress.org/export/1.2/', |
| 327 | ); |
| 328 | $this->known_entities = array( |
| 329 | 'item' => array( |
| 330 | 'type' => 'post', |
| 331 | 'fields' => array( |
| 332 | 'title' => 'post_title', |
| 333 | 'link' => 'link', |
| 334 | 'guid' => 'guid', |
| 335 | 'description' => 'post_excerpt', |
| 336 | 'pubDate' => 'post_published_at', |
| 337 | '{http://purl.org/dc/elements/1.1/}creator' => 'post_author', |
| 338 | '{http://purl.org/rss/1.0/modules/content/}encoded' => 'post_content', |
| 339 | '{http://wordpress.org/export/1.0/excerpt/}encoded' => 'post_excerpt', |
| 340 | '{http://wordpress.org/export/1.1/excerpt/}encoded' => 'post_excerpt', |
| 341 | '{http://wordpress.org/export/1.2/excerpt/}encoded' => 'post_excerpt', |
| 342 | ), |
| 343 | ), |
| 344 | ); |
| 345 | foreach ( $wxr_namespaces as $wxr_namespace ) { |
| 346 | $this->known_site_options = array_merge( |
| 347 | $this->known_site_options, |
| 348 | array( |
| 349 | '{' . $wxr_namespace . '}base_blog_url' => 'home', |
| 350 | '{' . $wxr_namespace . '}base_site_url' => 'siteurl', |
| 351 | 'title' => 'blogname', |
| 352 | ) |
| 353 | ); |
| 354 | $this->known_entities['item']['fields'] = array_merge( |
| 355 | $this->known_entities['item']['fields'], |
| 356 | array( |
| 357 | '{' . $wxr_namespace . '}post_id' => 'post_id', |
| 358 | '{' . $wxr_namespace . '}status' => 'post_status', |
| 359 | '{' . $wxr_namespace . '}post_date' => 'post_date', |
| 360 | '{' . $wxr_namespace . '}post_date_gmt' => 'post_date_gmt', |
| 361 | '{' . $wxr_namespace . '}post_modified' => 'post_modified', |
| 362 | '{' . $wxr_namespace . '}post_modified_gmt' => 'post_modified_gmt', |
| 363 | '{' . $wxr_namespace . '}comment_status' => 'comment_status', |
| 364 | '{' . $wxr_namespace . '}ping_status' => 'ping_status', |
| 365 | '{' . $wxr_namespace . '}post_name' => 'post_name', |
| 366 | '{' . $wxr_namespace . '}post_parent' => 'post_parent', |
| 367 | '{' . $wxr_namespace . '}menu_order' => 'menu_order', |
| 368 | '{' . $wxr_namespace . '}post_type' => 'post_type', |
| 369 | '{' . $wxr_namespace . '}post_password' => 'post_password', |
| 370 | '{' . $wxr_namespace . '}is_sticky' => 'is_sticky', |
| 371 | '{' . $wxr_namespace . '}attachment_url' => 'attachment_url', |
| 372 | ) |
| 373 | ); |
| 374 | $this->known_entities = array_merge( |
| 375 | $this->known_entities, |
| 376 | array( |
| 377 | '{' . $wxr_namespace . '}comment' => array( |
| 378 | 'type' => 'comment', |
| 379 | 'fields' => array( |
| 380 | '{' . $wxr_namespace . '}comment_id' => 'comment_id', |
| 381 | '{' . $wxr_namespace . '}comment_author' => 'comment_author', |
| 382 | '{' . $wxr_namespace . '}comment_author_email' => 'comment_author_email', |
| 383 | '{' . $wxr_namespace . '}comment_author_url' => 'comment_author_url', |
| 384 | '{' . $wxr_namespace . '}comment_author_IP' => 'comment_author_IP', |
| 385 | '{' . $wxr_namespace . '}comment_date' => 'comment_date', |
| 386 | '{' . $wxr_namespace . '}comment_date_gmt' => 'comment_date_gmt', |
| 387 | '{' . $wxr_namespace . '}comment_content' => 'comment_content', |
| 388 | '{' . $wxr_namespace . '}comment_approved' => 'comment_approved', |
| 389 | '{' . $wxr_namespace . '}comment_type' => 'comment_type', |
| 390 | '{' . $wxr_namespace . '}comment_parent' => 'comment_parent', |
| 391 | '{' . $wxr_namespace . '}comment_user_id' => 'comment_user_id', |
| 392 | ), |
| 393 | ), |
| 394 | '{' . $wxr_namespace . '}commentmeta' => array( |
| 395 | 'type' => 'comment_meta', |
| 396 | 'fields' => array( |
| 397 | '{' . $wxr_namespace . '}meta_key' => 'meta_key', |
| 398 | '{' . $wxr_namespace . '}meta_value' => 'meta_value', |
| 399 | ), |
| 400 | ), |
| 401 | '{' . $wxr_namespace . '}author' => array( |
| 402 | 'type' => 'user', |
| 403 | 'fields' => array( |
| 404 | '{' . $wxr_namespace . '}author_id' => 'ID', |
| 405 | '{' . $wxr_namespace . '}author_login' => 'user_login', |
| 406 | '{' . $wxr_namespace . '}author_email' => 'user_email', |
| 407 | '{' . $wxr_namespace . '}author_display_name' => 'display_name', |
| 408 | '{' . $wxr_namespace . '}author_first_name' => 'first_name', |
| 409 | '{' . $wxr_namespace . '}author_last_name' => 'last_name', |
| 410 | ), |
| 411 | ), |
| 412 | '{' . $wxr_namespace . '}postmeta' => array( |
| 413 | 'type' => 'post_meta', |
| 414 | 'fields' => array( |
| 415 | '{' . $wxr_namespace . '}meta_key' => 'meta_key', |
| 416 | '{' . $wxr_namespace . '}meta_value' => 'meta_value', |
| 417 | ), |
| 418 | ), |
| 419 | '{' . $wxr_namespace . '}term' => array( |
| 420 | 'type' => 'term', |
| 421 | 'fields' => array( |
| 422 | '{' . $wxr_namespace . '}term_id' => 'term_id', |
| 423 | '{' . $wxr_namespace . '}term_taxonomy' => 'taxonomy', |
| 424 | '{' . $wxr_namespace . '}term_slug' => 'slug', |
| 425 | '{' . $wxr_namespace . '}term_parent' => 'parent', |
| 426 | '{' . $wxr_namespace . '}term_name' => 'name', |
| 427 | ), |
| 428 | ), |
| 429 | '{' . $wxr_namespace . '}tag' => array( |
| 430 | 'type' => 'tag', |
| 431 | 'fields' => array( |
| 432 | '{' . $wxr_namespace . '}term_id' => 'term_id', |
| 433 | '{' . $wxr_namespace . '}tag_slug' => 'slug', |
| 434 | '{' . $wxr_namespace . '}tag_name' => 'name', |
| 435 | '{' . $wxr_namespace . '}tag_description' => 'description', |
| 436 | ), |
| 437 | ), |
| 438 | '{' . $wxr_namespace . '}category' => array( |
| 439 | 'type' => 'category', |
| 440 | 'fields' => array( |
| 441 | '{' . $wxr_namespace . '}category_nicename' => 'slug', |
| 442 | '{' . $wxr_namespace . '}category_parent' => 'parent', |
| 443 | '{' . $wxr_namespace . '}cat_name' => 'name', |
| 444 | '{' . $wxr_namespace . '}category_description' => 'description', |
| 445 | ), |
| 446 | ), |
| 447 | ) |
| 448 | ); |
| 449 | } |
| 450 | } |
| 451 | |
| 452 | public function get_reentrancy_cursor() { |
| 453 | /** |
| 454 | * @TODO: Instead of adjusting the XML cursor internals, adjust the get_reentrancy_cursor() |
| 455 | * call to support $bookmark_name, e.g. $this->xml->get_reentrancy_cursor( 'last_entity' ); |
| 456 | * If the cursor internal data was a part of every bookmark, this would have worked |
| 457 | * even after evicting the actual bytes where $last_entity is stored. |
| 458 | */ |
| 459 | $xml_cursor = $this->xml->get_reentrancy_cursor(); |
| 460 | $xml_cursor = json_decode( base64_decode( $xml_cursor ), true ); // phpcs:ignore WordPress.PHP.DiscouragedPHPFunctions.obfuscation_base64_decode |
| 461 | $xml_cursor['upstream_bytes_forgotten'] = $this->entity_opener_byte_offset; |
| 462 | $xml_cursor = base64_encode( json_encode( $xml_cursor ) ); // phpcs:ignore WordPress.PHP.DiscouragedPHPFunctions.obfuscation_base64_encode |
| 463 | |
| 464 | return json_encode( |
| 465 | array( |
| 466 | 'xml' => $xml_cursor, |
| 467 | 'upstream' => $this->entity_opener_byte_offset, |
| 468 | 'last_post_id' => $this->last_post_id, |
| 469 | 'last_comment_id' => $this->last_comment_id, |
| 470 | ) |
| 471 | ); |
| 472 | } |
| 473 | |
| 474 | /** |
| 475 | * Gets the data for the current entity. |
| 476 | * |
| 477 | * @return ImportEntity The entity. |
| 478 | * @since WP_VERSION |
| 479 | */ |
| 480 | public function get_entity() { |
| 481 | if ( ! $this->get_entity_type() ) { |
| 482 | return false; |
| 483 | } |
| 484 | |
| 485 | return new ImportEntity( |
| 486 | $this->get_entity_type(), |
| 487 | $this->entity_data |
| 488 | ); |
| 489 | } |
| 490 | |
| 491 | /** |
| 492 | * Gets the type of the current entity. |
| 493 | * |
| 494 | * @return string|false The entity type, or false if no entity is being processed. |
| 495 | * @since WP_VERSION |
| 496 | */ |
| 497 | private function get_entity_type() { |
| 498 | if ( null !== $this->entity_type ) { |
| 499 | return $this->entity_type; |
| 500 | } |
| 501 | if ( null === $this->entity_tag ) { |
| 502 | return false; |
| 503 | } |
| 504 | if ( ! array_key_exists( $this->entity_tag, $this->known_entities ) ) { |
| 505 | return false; |
| 506 | } |
| 507 | |
| 508 | return $this->known_entities[ $this->entity_tag ]['type']; |
| 509 | } |
| 510 | |
| 511 | /** |
| 512 | * Gets the ID of the last processed post. |
| 513 | * |
| 514 | * @return int|null The post ID, or null if no posts have been processed. |
| 515 | * @since WP_VERSION |
| 516 | */ |
| 517 | public function get_last_post_id() { |
| 518 | return $this->last_post_id; |
| 519 | } |
| 520 | |
| 521 | /** |
| 522 | * Gets the ID of the last processed comment. |
| 523 | * |
| 524 | * @return int|null The comment ID, or null if no comments have been processed. |
| 525 | * @since WP_VERSION |
| 526 | */ |
| 527 | public function get_last_comment_id() { |
| 528 | return $this->last_comment_id; |
| 529 | } |
| 530 | |
| 531 | /** |
| 532 | * Appends bytes to the input stream. |
| 533 | * |
| 534 | * @param string $bytes The bytes to append. |
| 535 | * |
| 536 | * @since WP_VERSION |
| 537 | */ |
| 538 | public function append_bytes( string $bytes ): void { |
| 539 | $this->xml->append_bytes( $bytes ); |
| 540 | } |
| 541 | |
| 542 | /** |
| 543 | * Marks the input as finished. |
| 544 | * |
| 545 | * @since WP_VERSION |
| 546 | */ |
| 547 | public function input_finished(): void { |
| 548 | $this->xml->input_finished(); |
| 549 | } |
| 550 | |
| 551 | /** |
| 552 | * Checks if processing is finished. |
| 553 | * |
| 554 | * @return bool Whether processing is finished. |
| 555 | * @since WP_VERSION |
| 556 | */ |
| 557 | public function is_finished(): bool { |
| 558 | return $this->is_finished; |
| 559 | } |
| 560 | |
| 561 | /** |
| 562 | * Checks if processing is paused waiting for more input. |
| 563 | * |
| 564 | * @return bool Whether processing is paused. |
| 565 | * @since WP_VERSION |
| 566 | */ |
| 567 | public function is_paused_at_incomplete_input(): bool { |
| 568 | return $this->xml->is_paused_at_incomplete_input(); |
| 569 | } |
| 570 | |
| 571 | /** |
| 572 | * Gets the last error that occurred. |
| 573 | * |
| 574 | * @return string|null The error message, or null if no error occurred. |
| 575 | * @since WP_VERSION |
| 576 | */ |
| 577 | public function get_last_error(): ?string { |
| 578 | return $this->xml->get_last_error(); |
| 579 | } |
| 580 | |
| 581 | public function get_xml_exception(): ?XMLUnsupportedException { |
| 582 | return $this->xml->get_exception(); |
| 583 | } |
| 584 | |
| 585 | /** |
| 586 | * Advances to the next entity in the WXR file. |
| 587 | * |
| 588 | * @return bool Whether another entity was found. |
| 589 | * @since WP_VERSION |
| 590 | */ |
| 591 | public function next_entity() { |
| 592 | if ( $this->is_finished ) { |
| 593 | return false; |
| 594 | } |
| 595 | while ( true ) { |
| 596 | if ( $this->read_next_entity() ) { |
| 597 | return true; |
| 598 | } |
| 599 | // If the read failed because of incomplete input data, |
| 600 | // try pulling more bytes from upstream before giving up. |
| 601 | if ( $this->is_paused_at_incomplete_input() ) { |
| 602 | if ( $this->pull_upstream_bytes() ) { |
| 603 | continue; |
| 604 | } else { |
| 605 | break; |
| 606 | } |
| 607 | } |
| 608 | $this->is_finished = true; |
| 609 | break; |
| 610 | } |
| 611 | |
| 612 | return false; |
| 613 | } |
| 614 | |
| 615 | /** |
| 616 | * Advances to the next entity in the WXR file. |
| 617 | * |
| 618 | * @return bool Whether another entity was found. |
| 619 | * @since WP_VERSION |
| 620 | */ |
| 621 | private function read_next_entity() { |
| 622 | if ( $this->xml->is_finished() ) { |
| 623 | $this->after_entity(); |
| 624 | |
| 625 | return false; |
| 626 | } |
| 627 | |
| 628 | if ( $this->xml->is_paused_at_incomplete_input() ) { |
| 629 | return false; |
| 630 | } |
| 631 | |
| 632 | /** |
| 633 | * This is the first call after emitting an entity. |
| 634 | * Remove the previous entity details from the internal state |
| 635 | * and prepare for the next entity. |
| 636 | */ |
| 637 | if ( $this->entity_type && $this->entity_finished ) { |
| 638 | $this->after_entity(); |
| 639 | // If we finished processing the entity on a closing tag, advance the XML processor to. |
| 640 | // the next token. Otherwise the array_key_exists( $tag, static::known_entities ) branch. |
| 641 | // below will cause an infinite loop. |
| 642 | if ( $this->xml->is_tag_closer() ) { |
| 643 | if ( false === $this->xml->next_token() ) { |
| 644 | return false; |
| 645 | } |
| 646 | } |
| 647 | } |
| 648 | |
| 649 | /** |
| 650 | * Main parsing loop. It advances the XML parser state until a full entity |
| 651 | * is available. |
| 652 | */ |
| 653 | do { |
| 654 | $breadcrumbs = $this->xml->get_breadcrumbs(); |
| 655 | // Don't process anything outside the <rss> <channel> hierarchy. |
| 656 | if ( |
| 657 | count( $breadcrumbs ) < 2 || |
| 658 | array( '', 'rss' ) !== $breadcrumbs[0] || |
| 659 | array( '', 'channel' ) !== $breadcrumbs[1] |
| 660 | ) { |
| 661 | continue; |
| 662 | } |
| 663 | |
| 664 | /* |
| 665 | * Buffer text and CDATA sections until we find the next tag. |
| 666 | * Each tag may contain multiple text or CDATA sections so we can't |
| 667 | * just assume that a single `get_modifiable_text()` call would get |
| 668 | * the entire text content of an element. |
| 669 | */ |
| 670 | if ( |
| 671 | '#text' === $this->xml->get_token_type() || |
| 672 | '#cdata-section' === $this->xml->get_token_type() |
| 673 | ) { |
| 674 | $this->text_buffer .= $this->xml->get_modifiable_text(); |
| 675 | continue; |
| 676 | } |
| 677 | |
| 678 | // We're only interested in tags after this point. |
| 679 | if ( '#tag' !== $this->xml->get_token_type() ) { |
| 680 | continue; |
| 681 | } |
| 682 | |
| 683 | if ( count( $breadcrumbs ) <= 2 && $this->xml->is_tag_opener() ) { |
| 684 | $this->entity_opener_byte_offset = $this->xml->get_token_byte_offset_in_the_input_stream(); |
| 685 | } |
| 686 | |
| 687 | $tag_with_namespace = $this->xml->get_tag_namespace_and_local_name(); |
| 688 | |
| 689 | /** |
| 690 | * Custom adjustment: the Accessibility WXR file uses a non-standard |
| 691 | * wp:wp_author tag. |
| 692 | * |
| 693 | * @TODO: Should WP_WXR_Entity_Reader care about such non-standard tags when |
| 694 | * the regular WXR importer would ignore them? Perhaps a warning |
| 695 | * and an upstream PR would be a better solution. |
| 696 | */ |
| 697 | if ( '{http://wordpress.org/export/1.2/}wp_author' === $tag_with_namespace ) { |
| 698 | $tag_with_namespace = '{http://wordpress.org/export/1.2/}author'; |
| 699 | } |
| 700 | |
| 701 | /** |
| 702 | * If the tag is a known entity root, assume the previous entity is |
| 703 | * finished, emit it, and start processing the new entity the next |
| 704 | * time this function is called. |
| 705 | */ |
| 706 | if ( array_key_exists( $tag_with_namespace, $this->known_entities ) ) { |
| 707 | if ( $this->entity_type && ! $this->entity_finished ) { |
| 708 | $this->emit_entity(); |
| 709 | |
| 710 | return true; |
| 711 | } |
| 712 | $this->after_entity(); |
| 713 | // Only tag openers indicate a new entity. Closers just mean |
| 714 | // the previous entity is finished. |
| 715 | if ( $this->xml->is_tag_opener() ) { |
| 716 | $this->set_entity_tag( $tag_with_namespace ); |
| 717 | $this->entity_opener_byte_offset = $this->xml->get_token_byte_offset_in_the_input_stream(); |
| 718 | } |
| 719 | continue; |
| 720 | } |
| 721 | |
| 722 | /** |
| 723 | * We're inside of an entity tag at this point. |
| 724 | * |
| 725 | * The following code assumes that we'll only see three types of tags: |
| 726 | * |
| 727 | * * Empty elements – such as <wp:comment_content />, that we'll ignore |
| 728 | * * XML element openers with only text nodes inside them. |
| 729 | * * XML element closers. |
| 730 | * |
| 731 | * Specifically, we don't expect to see any nested XML elements such as: |
| 732 | * |
| 733 | * <wp:comment_content> |
| 734 | * <title>Pygmalion</title> |
| 735 | * Long time ago... |
| 736 | * </wp:comment_content> |
| 737 | * |
| 738 | * The semantics of such a structure is not clear. The WP_WXR_Entity_Reader will |
| 739 | * enter an error state when it encounters such a structure. |
| 740 | * |
| 741 | * Such nesting wasn't found in any WXR files analyzed when building |
| 742 | * this class. If it actually is a part of the WXR standard, every |
| 743 | * supported nested element will need a custom handler. |
| 744 | */ |
| 745 | |
| 746 | /** |
| 747 | * Buffer the XML tag opener attributes for later use. |
| 748 | * |
| 749 | * In WXR files, entity attributes come from two sources: |
| 750 | * * XML attributes on the tag itself |
| 751 | * * Text content between the opening and closing tags |
| 752 | * |
| 753 | * We store the XML attributes when encountering an opening tag, |
| 754 | * but wait until the closing tag to process the entity attributes. |
| 755 | * Why? Because only at that point we have both the attributes |
| 756 | * and all the related text nodes. |
| 757 | */ |
| 758 | if ( $this->xml->is_tag_opener() ) { |
| 759 | $this->last_opener_attributes = array(); |
| 760 | // Get non-namespaced attributes. |
| 761 | $names = $this->xml->get_attribute_names_with_prefix( '', '' ); |
| 762 | foreach ( $names as list($namespace, $name) ) { |
| 763 | $this->last_opener_attributes[ $name ] = $this->xml->get_attribute( $namespace, $name ); |
| 764 | } |
| 765 | $this->text_buffer = ''; |
| 766 | |
| 767 | $is_site_option_opener = ( |
| 768 | 3 === count( $this->xml->get_breadcrumbs() ) && |
| 769 | $this->xml->matches_breadcrumbs( array( 'rss', 'channel', '*' ) ) && |
| 770 | array_key_exists( $this->xml->get_tag_namespace_and_local_name(), $this->known_site_options ) |
| 771 | ); |
| 772 | if ( $is_site_option_opener ) { |
| 773 | $this->entity_opener_byte_offset = $this->xml->get_token_byte_offset_in_the_input_stream(); |
| 774 | } |
| 775 | |
| 776 | continue; |
| 777 | } |
| 778 | |
| 779 | /** |
| 780 | * At this point we're looking for the nearest tag closer so we can |
| 781 | * turn the buffered data into an entity attribute. |
| 782 | */ |
| 783 | if ( ! $this->xml->is_tag_closer() ) { |
| 784 | continue; |
| 785 | } |
| 786 | |
| 787 | if ( |
| 788 | ! $this->entity_finished && |
| 789 | array( array( '', 'rss' ), array( '', 'channel' ) ) === $this->xml->get_breadcrumbs() |
| 790 | ) { |
| 791 | // Look for site options in children of the <channel> tag. |
| 792 | if ( $this->parse_site_option() ) { |
| 793 | return true; |
| 794 | } else { |
| 795 | // Keep looking for an entity if none was found in the current tag. |
| 796 | continue; |
| 797 | } |
| 798 | } |
| 799 | |
| 800 | /** |
| 801 | * Special handling to accumulate categories stored inside the <category> |
| 802 | * tag found inside <item> tags. |
| 803 | * |
| 804 | * For example, we want to convert this: |
| 805 | * |
| 806 | * <category><![CDATA[Uncategorized]]></category> |
| 807 | * <category domain="category" nicename="wordpress"> |
| 808 | * <![CDATA[WordPress]]> |
| 809 | * </category> |
| 810 | * |
| 811 | * Into this: |
| 812 | * |
| 813 | * 'terms' => [ |
| 814 | * [ 'taxonomy' => 'category', 'slug' => '', 'description' => 'Uncategorized' ], |
| 815 | * [ 'taxonomy' => 'category', 'slug' => 'WordPress', 'description' => 'WordPress' ], |
| 816 | * ] |
| 817 | */ |
| 818 | if ( |
| 819 | 'post' === $this->entity_type && |
| 820 | 'category' === $this->xml->get_tag_local_name() && |
| 821 | array_key_exists( 'domain', $this->last_opener_attributes ) && |
| 822 | array_key_exists( 'nicename', $this->last_opener_attributes ) |
| 823 | ) { |
| 824 | $this->entity_data['terms'][] = array( |
| 825 | 'taxonomy' => $this->last_opener_attributes['domain'], |
| 826 | 'slug' => $this->last_opener_attributes['nicename'], |
| 827 | 'description' => $this->text_buffer, |
| 828 | ); |
| 829 | $this->text_buffer = ''; |
| 830 | continue; |
| 831 | } |
| 832 | |
| 833 | /** |
| 834 | * Store the text content of known tags as the value of the corresponding |
| 835 | * entity attribute as defined by the $known_entities mapping. |
| 836 | * |
| 837 | * Ignores tags unlisted in the $known_entities mapping. |
| 838 | * |
| 839 | * The WXR format is extensible so this reader could potentially |
| 840 | * support registering custom handlers for unknown tags in the future. |
| 841 | */ |
| 842 | if ( ! isset( $this->known_entities[ $this->entity_tag ]['fields'][ $tag_with_namespace ] ) ) { |
| 843 | continue; |
| 844 | } |
| 845 | |
| 846 | $key = $this->known_entities[ $this->entity_tag ]['fields'][ $tag_with_namespace ]; |
| 847 | $this->entity_data[ $key ] = $this->text_buffer; |
| 848 | $this->text_buffer = ''; |
| 849 | } while ( $this->xml->next_token() ); |
| 850 | |
| 851 | if ( $this->is_paused_at_incomplete_input() ) { |
| 852 | return false; |
| 853 | } |
| 854 | |
| 855 | /** |
| 856 | * Emit the last unemitted entity after parsing all the data. |
| 857 | */ |
| 858 | if ( |
| 859 | $this->is_finished() && |
| 860 | $this->entity_type && |
| 861 | ! $this->entity_finished |
| 862 | ) { |
| 863 | $this->emit_entity(); |
| 864 | |
| 865 | return true; |
| 866 | } |
| 867 | |
| 868 | return false; |
| 869 | } |
| 870 | |
| 871 | /** |
| 872 | * Emits a site option entity from known children of the <channel> |
| 873 | * tag, e.g. <wp:base_blog_url> or <title>. |
| 874 | * |
| 875 | * @return bool Whether a site_option entity was emitted. |
| 876 | */ |
| 877 | private function parse_site_option() { |
| 878 | if ( ! array_key_exists( $this->xml->get_tag_namespace_and_local_name(), $this->known_site_options ) ) { |
| 879 | return false; |
| 880 | } |
| 881 | |
| 882 | $this->entity_type = 'site_option'; |
| 883 | $this->entity_data = array( |
| 884 | 'option_name' => $this->known_site_options[ $this->xml->get_tag_namespace_and_local_name() ], |
| 885 | 'option_value' => $this->text_buffer, |
| 886 | ); |
| 887 | $this->emit_entity(); |
| 888 | |
| 889 | return true; |
| 890 | } |
| 891 | |
| 892 | /** |
| 893 | * Connects a byte stream to automatically pull bytes from once |
| 894 | * the last input chunk have been processed. |
| 895 | * |
| 896 | * @param ByteReadStream $stream The upstream stream. |
| 897 | */ |
| 898 | public function connect_upstream( ByteReadStream $stream ) { |
| 899 | $this->upstream = $stream; |
| 900 | } |
| 901 | |
| 902 | /** |
| 903 | * Appends another chunk of bytes from upstream if available. |
| 904 | */ |
| 905 | private function pull_upstream_bytes() { |
| 906 | if ( ! $this->upstream ) { |
| 907 | return false; |
| 908 | } |
| 909 | if ( $this->upstream->reached_end_of_data() ) { |
| 910 | $this->input_finished(); |
| 911 | |
| 912 | return false; |
| 913 | } |
| 914 | |
| 915 | $available_bytes = $this->upstream->pull( 65536 ); |
| 916 | $this->append_bytes( $this->upstream->consume( $available_bytes ) ); |
| 917 | |
| 918 | return true; |
| 919 | } |
| 920 | |
| 921 | /** |
| 922 | * Marks the current entity as emitted and updates tracking variables. |
| 923 | * |
| 924 | * @since WP_VERSION |
| 925 | */ |
| 926 | private function emit_entity() { |
| 927 | if ( 'post' === $this->entity_type ) { |
| 928 | // Not all posts have a `<wp:post_id>` tag. |
| 929 | $this->last_post_id = isset( $this->entity_data['post_id'] ) ? $this->entity_data['post_id'] : null; |
| 930 | } elseif ( 'post_meta' === $this->entity_type ) { |
| 931 | $this->entity_data['post_id'] = $this->last_post_id; |
| 932 | } elseif ( 'comment' === $this->entity_type ) { |
| 933 | $this->last_comment_id = $this->entity_data['comment_id']; |
| 934 | $this->entity_data['post_id'] = $this->last_post_id; |
| 935 | } elseif ( 'comment_meta' === $this->entity_type ) { |
| 936 | $this->entity_data['comment_id'] = $this->last_comment_id; |
| 937 | } elseif ( 'tag' === $this->entity_type ) { |
| 938 | $this->entity_data['taxonomy'] = 'post_tag'; |
| 939 | } elseif ( 'category' === $this->entity_type ) { |
| 940 | $this->entity_data['taxonomy'] = 'category'; |
| 941 | } |
| 942 | $this->entity_finished = true; |
| 943 | ++$this->entities_read_so_far; |
| 944 | } |
| 945 | |
| 946 | /** |
| 947 | * Sets the current entity tag and type. |
| 948 | * |
| 949 | * @param string $tag_with_namespace The entity tag name. |
| 950 | * |
| 951 | * @since WP_VERSION |
| 952 | */ |
| 953 | private function set_entity_tag( string $tag_with_namespace ) { |
| 954 | $this->entity_tag = $tag_with_namespace; |
| 955 | if ( array_key_exists( $tag_with_namespace, $this->known_entities ) ) { |
| 956 | $this->entity_type = $this->known_entities[ $tag_with_namespace ]['type']; |
| 957 | } |
| 958 | } |
| 959 | |
| 960 | /** |
| 961 | * Resets the state after processing an entity. |
| 962 | * |
| 963 | * @since WP_VERSION |
| 964 | */ |
| 965 | private function after_entity() { |
| 966 | $this->entity_tag = null; |
| 967 | $this->entity_type = null; |
| 968 | $this->entity_data = array(); |
| 969 | $this->entity_finished = false; |
| 970 | $this->text_buffer = ''; |
| 971 | $this->last_opener_attributes = array(); |
| 972 | } |
| 973 | } |
| 974 |