| @@ -2,29 +2,65 @@ | ||
| 2 | 2 | |
| 3 | 3 | namespace ImportWP\Common\Importer\File; |
| 4 | 4 | |
| 5 | 5 | use ImportWP\Common\Importer\FileInterface; |
| 6 | +use ImportWP\Common\Importer\State\ImporterState; | |
| 6 | 7 | |
| 7 | 8 | class XMLFile extends AbstractIndexedFile implements FileInterface |
| 8 | 9 | { |
| 9 | 10 | |
| 10 | - private $base_path; | |
| 11 | - private $base_path_segments = array(); | |
| 12 | - private $record_offset = 0; | |
| 13 | - private $chunk_size = 8192; | |
| 14 | - private $depth = 0; | |
| 15 | - private $open_nodes = array(); | |
| 16 | - private $node_list = array(); | |
| 17 | - private $chunk; | |
| 18 | - private $chunk_offset = 0; | |
| 11 | + /** | |
| 12 | + * Base path | |
| 13 | + * | |
| 14 | + * Name of the files base path node, added onto the end of base_path_segments. | |
| 15 | + * e.g. if record path is 'records/record', base_path would be 'record'. | |
| 16 | + * | |
| 17 | + * @var string | |
| 18 | + */ | |
| 19 | + protected $base_path; | |
| 19 | 20 | |
| 20 | - private $record_counter = 0; | |
| 21 | - private $record_start_index = 0; | |
| 22 | - private $record_opened = false; | |
| 23 | - private $last_record_incomplete = false; | |
| 24 | - private $record_patch_cache_key = ''; | |
| 21 | + /** | |
| 22 | + * Base path segments | |
| 23 | + * | |
| 24 | + * Stores a list of nodes up to and not including the final base_path | |
| 25 | + * e.g. if base_path is 'records/record' , base_path_node would be ['records'] | |
| 26 | + * | |
| 27 | + * @var string[] | |
| 28 | + */ | |
| 29 | + protected $base_path_segments = array(); | |
| 25 | 30 | |
| 26 | 31 | /** |
| 32 | + * Open nodes | |
| 33 | + * | |
| 34 | + * List of nodes that have not been opened so far while parsing the file. | |
| 35 | + * | |
| 36 | + * @var string[] | |
| 37 | + */ | |
| 38 | + protected $open_nodes = array(); | |
| 39 | + | |
| 40 | + protected $record_offset = 0; | |
| 41 | + protected $chunk_size = 8192; | |
| 42 | + protected $depth = 0; | |
| 43 | + protected $node_list = array(); | |
| 44 | + protected $namespace_list = null; | |
| 45 | + protected $chunk; | |
| 46 | + protected $chunk_offset = 0; | |
| 47 | + | |
| 48 | + protected $record_counter = 0; | |
| 49 | + protected $record_start_index = 0; | |
| 50 | + | |
| 51 | + /** | |
| 52 | + * Record Opened | |
| 53 | + * | |
| 54 | + * Flag to say we are parsing data within the chosen record path. | |
| 55 | + * | |
| 56 | + * @var boolean | |
| 57 | + */ | |
| 58 | + protected $record_opened = false; | |
| 59 | + protected $last_record_incomplete = false; | |
| 60 | + protected $record_patch_cache_key = ''; | |
| 61 | + | |
| 62 | + /** | |
| 27 | 63 | * Set base path for records |
| 28 | 64 | * |
| 29 | 65 | * @param string $base_path |
| 30 | 66 | */ |
| @@ -73,9 +109,9 @@ | ||
| 73 | 109 | * @param $record |
| 74 | 110 | * |
| 75 | 111 | * @return bool|string |
| 76 | 112 | */ |
| 77 | - private function fixRecord($record) | |
| 113 | + public function fixRecord($record) | |
| 78 | 114 | { |
| 79 | 115 | $open = "/<[^\/][^>]+[^\/]>/"; |
| 80 | 116 | $close = "/<\/[^>]+[^\/]>/"; |
| 81 | 117 | $depth = 0; |
| @@ -129,9 +165,9 @@ | ||
| 129 | 165 | * @param $node |
| 130 | 166 | * |
| 131 | 167 | * @return bool|string |
| 132 | 168 | */ |
| 133 | - private function getNodeName($node) | |
| 169 | + public function getNodeName($node) | |
| 134 | 170 | { |
| 135 | 171 | |
| 136 | 172 | preg_match("/<\/?([a-zA-Z0-9\-_\.:]+)[ ]*[^>]*>/", $node, $matches); |
| 137 | 173 | if (isset($matches[0]) && isset($matches[1])) { |
| @@ -149,25 +185,51 @@ | ||
| 149 | 185 | * Read through entire xml file generating record start and end positions |
| 150 | 186 | * |
| 151 | 187 | * @return mixed |
| 152 | 188 | */ |
| 153 | - protected function generateIndex() | |
| 189 | + public function generateIndex() | |
| 154 | 190 | { |
| 155 | 191 | $this->record_counter = 0; |
| 156 | 192 | $this->record_start_index = 0; |
| 157 | 193 | |
| 194 | + $last_percent = 0; | |
| 195 | + $size = intval($this->get_file_size($this->getFileHandle())); | |
| 196 | + $this->config->set('process', 0); | |
| 197 | + $this->namespace_list = []; | |
| 198 | + | |
| 158 | 199 | rewind($this->getFileHandle()); |
| 200 | + | |
| 159 | 201 | while (!feof($this->getFileHandle())) { |
| 160 | 202 | |
| 161 | 203 | $this->chunk .= $this->getChunk(); |
| 162 | - $this->readChunkXmlNodes(); | |
| 204 | + $this->chunk = $this->read_chunk_xml_nodes($this->chunk); | |
| 163 | 205 | |
| 164 | - // only read the first 1mb of file | |
| 165 | - if ($this->is_processing && ($this->record_counter > 0 || ftell($this->getFileHandle()) > $this->process_max_size)) { | |
| 206 | + // only read the first 1mb of file when sampling | |
| 207 | + if ($this->is_processing && ftell($this->getFileHandle()) > $this->process_max_size) { | |
| 166 | 208 | break; |
| 167 | 209 | } |
| 210 | + | |
| 211 | + $current_pos = intval(ftell($this->getFileHandle())); | |
| 212 | + if ($size > 0 && $current_pos > 0) { | |
| 213 | + $percent = round($current_pos / $size, 2); | |
| 214 | + if ($percent > $last_percent) { | |
| 215 | + | |
| 216 | + do_action('iwp/importer/process', $last_percent * 100); | |
| 217 | + $this->config->set('process', $last_percent * 100); | |
| 218 | + $last_percent = $percent; | |
| 219 | + | |
| 220 | + if (!is_null(iwp()->importer)) { | |
| 221 | + $state = ImporterState::get_state(iwp()->importer->getId()); | |
| 222 | + if (isset($state['status']) && $state['status'] === 'cancelled') { | |
| 223 | + return; | |
| 224 | + } | |
| 225 | + } | |
| 226 | + } | |
| 227 | + } | |
| 168 | 228 | } |
| 169 | 229 | |
| 230 | + $this->config->set('namespace_list', $this->namespace_list); | |
| 231 | + | |
| 170 | 232 | // if file is incomplete do we try and fix the record, or skip it? |
| 171 | 233 | if (!$this->is_processing && true === $this->record_opened) { |
| 172 | 234 | $this->record_opened = false; |
| 173 | 235 | $this->last_record_incomplete = true; |
| @@ -183,9 +245,9 @@ | ||
| 183 | 245 | * Read chunk from file |
| 184 | 246 | * |
| 185 | 247 | * @return bool|string |
| 186 | 248 | */ |
| 187 | - private function getChunk() | |
| 249 | + public function getChunk() | |
| 188 | 250 | { |
| 189 | 251 | return fread($this->getFileHandle(), $this->chunk_size); |
| 190 | 252 | } |
| 191 | 253 | |
| @@ -191,9 +253,9 @@ | ||
| 191 | 253 | |
| 192 | 254 | /** |
| 193 | 255 | * Read all xml nodes in chunk |
| 194 | 256 | */ |
| 195 | - private function readChunkXmlNodes() | |
| 257 | + public function readChunkXmlNodes() | |
| 196 | 258 | { |
| 197 | 259 | $tags = [ |
| 198 | 260 | ["<?", "?>", true, 0], |
| 199 | 261 | ["<!--", "-->", true, 0], |
| @@ -241,58 +303,94 @@ | ||
| 241 | 303 | $string_offset = $offset + strlen($captured); |
| 242 | 304 | $this->chunk_offset += $string_offset; |
| 243 | 305 | $this->chunk = substr($this->chunk, $string_offset); |
| 244 | 306 | } |
| 245 | - return; | |
| 307 | + } | |
| 246 | 308 | |
| 247 | - // TODO: Match xmlns: <[^>]+xmlns:(.*?)="(.*?)"[^>]+> | |
| 248 | - /*$open = "/<[^\/][^>]*[^\/]>/s"; | |
| 249 | - $close = "/<\/[^>]*[^\/]>/"; | |
| 250 | - $openclose = "/<[^\/][^>]+\/>/s"; | |
| 309 | + /** | |
| 310 | + * Optimised version of readChunkXMLNodes | |
| 311 | + * | |
| 312 | + * @param string $chunk | |
| 313 | + * @return string | |
| 314 | + */ | |
| 315 | + public function read_chunk_xml_nodes($chunk) | |
| 316 | + { | |
| 317 | + $tags = [ | |
| 318 | + "<?" => ["<?", "?>", true, 0], | |
| 319 | + "<!--" => ["<!--", "-->", true, 0], | |
| 320 | + "<![CDATA[" => ["<![CDATA[", "]]>", true, 0], | |
| 321 | + "<!" => ["<!", ">", true, 0], | |
| 322 | + "</" => ["</", ">", false, -1], | |
| 323 | + "<" => ["<", "/>", false, 0], | |
| 324 | + "<" => ["<", ">", false, 1], | |
| 325 | + ]; | |
| 251 | 326 | |
| 252 | - while (preg_match("/<[^?|!\-\-][^>]*>/s", $this->chunk, $matches, | |
| 253 | - PREG_OFFSET_CAPTURE) !== 0) { | |
| 254 | - if (isset($matches[0]) && isset($matches[0][0])) { | |
| 255 | - list($captured, $offset) = $matches[0]; | |
| 327 | + // 1.Loop through the file, each node at a time | |
| 328 | + $chunk_offset = 0; | |
| 256 | 329 | |
| 257 | - $this->record_offset = $offset; | |
| 330 | + while (preg_match("/<[^>]+>/", $chunk, $matches, PREG_OFFSET_CAPTURE, $chunk_offset) !== 0) { | |
| 331 | + list($captured, $offset) = $matches[0]; | |
| 258 | 332 | |
| 259 | - if (preg_match($close, $captured) === 1) { | |
| 260 | - $this->depth--; | |
| 333 | + // length of content between last and current tag | |
| 334 | + $content_length = ($offset - $chunk_offset); | |
| 335 | + $chunk_offset += $content_length + strlen($captured); | |
| 261 | 336 | |
| 262 | - $node = array_pop($this->open_nodes); | |
| 337 | + if (preg_match('/^(<\?|<!--|<!\[CDATA\[|<!|<\/|<)/', $captured, $tag_matches) !== 1) { | |
| 338 | + continue; | |
| 339 | + } | |
| 263 | 340 | |
| 264 | - if ($this->isRecordEnd($node)) { | |
| 265 | - $this->record_opened = false; | |
| 266 | - $temp = $this->chunk_offset + $this->record_offset + strlen($captured); | |
| 267 | - $this->setIndex($this->record_counter, | |
| 268 | - $this->record_start_index, $temp); | |
| 269 | - $this->record_counter++; | |
| 270 | - } | |
| 271 | - }elseif (preg_match($open, $captured) === 1) { | |
| 272 | - $this->depth++; | |
| 341 | + $tags_test = []; | |
| 342 | + if ($tag_matches[0] === '<') { | |
| 343 | + $tags_test[] = ["<", "/>", false, 0]; | |
| 344 | + $tags_test[] = ["<", ">", false, 1]; | |
| 345 | + } else { | |
| 346 | + $tags_test[] = $tags[$tag_matches[0]]; | |
| 347 | + } | |
| 273 | 348 | |
| 274 | - $node_name = $this->getNodeName($captured); | |
| 349 | + // print_r($tag_matches); | |
| 275 | 350 | |
| 276 | - if ($this->isRecordStart($node_name)) { | |
| 277 | - $this->record_opened = true; | |
| 278 | - $temp = $this->chunk_offset + $this->record_offset; | |
| 279 | - $this->record_start_index = $temp; | |
| 280 | - } | |
| 351 | + foreach ($tags_test as $tag) { | |
| 352 | + list($open_tag, $close_tag, $search_for_close, $depth_delta) = $tag; | |
| 281 | 353 | |
| 282 | - $this->open_nodes[] = $node_name; | |
| 283 | - $this->record_open_nodes(); | |
| 354 | + // if (strpos($captured, $open_tag) !== 0) { | |
| 355 | + // continue; | |
| 356 | + // } | |
| 284 | 357 | |
| 358 | + if (strlen($captured) - strlen($close_tag) === strpos($captured, $close_tag)) { | |
| 359 | + // Tag Found | |
| 360 | + if ($search_for_close === false) { | |
| 361 | + $this->record_node($captured, $depth_delta, $this->chunk_offset + $offset); | |
| 362 | + } | |
| 363 | + break; | |
| 364 | + } elseif ($search_for_close === true) { | |
| 365 | + $close_tag_position = strpos($chunk, $close_tag, $offset); | |
| 366 | + if ($close_tag_position !== false) { | |
| 367 | + // Close tag is found in current chunk | |
| 368 | + $captured = substr($chunk, $offset, $close_tag_position + strlen($close_tag) - $offset); | |
| 369 | + $chunk_offset = $close_tag_position + strlen($close_tag); | |
| 370 | + if ($search_for_close === false) { | |
| 371 | + $this->record_node($captured, $depth_delta, $this->chunk_offset + $offset); | |
| 372 | + } | |
| 373 | + } else { | |
| 374 | + // We need to load next chunk to find | |
| 375 | + $this->chunk_offset += $offset; | |
| 376 | + return substr($chunk, $offset); | |
| 377 | + } | |
| 378 | + break; | |
| 285 | 379 | } |
| 286 | - $string_offset = $offset + strlen($captured); | |
| 287 | - $this->chunk_offset += $string_offset; | |
| 288 | - $this->chunk = substr($this->chunk, $string_offset); | |
| 289 | 380 | } |
| 290 | - }*/ | |
| 381 | + } | |
| 382 | + | |
| 383 | + $this->chunk_offset += $chunk_offset; | |
| 384 | + | |
| 385 | + return substr($chunk, $chunk_offset); | |
| 291 | 386 | } |
| 292 | 387 | |
| 293 | - private function record_node($node, $depth_delta) | |
| 388 | + public function record_node($node, $depth_delta, $offset = null) | |
| 294 | 389 | { |
| 390 | + if (is_null($offset)) { | |
| 391 | + $offset = $this->chunk_offset + $this->record_offset; | |
| 392 | + } | |
| 295 | 393 | |
| 296 | 394 | $this->depth += $depth_delta; |
| 297 | 395 | $node_name = $this->getNodeName($node); |
| 298 | 396 | |
| @@ -299,13 +397,18 @@ | ||
| 299 | 397 | if ($depth_delta > 0) { |
| 300 | 398 | |
| 301 | 399 | if ($this->isRecordStart($node_name)) { |
| 302 | 400 | $this->record_opened = true; |
| 303 | - $this->record_start_index = $this->chunk_offset + $this->record_offset; | |
| 401 | + $this->record_start_index = $offset; | |
| 304 | 402 | } |
| 305 | 403 | |
| 306 | 404 | $this->open_nodes[] = $node_name; |
| 307 | 405 | $this->record_open_nodes(); |
| 406 | + | |
| 407 | + // find namespaces | |
| 408 | + if (preg_match_all('/xmlns:([a-zA-Z0-9_-]+)="([^"]+)"/', $node, $matches) > 0) { | |
| 409 | + $this->namespace_list = array_merge($this->namespace_list, array_combine($matches[1], $matches[2])); | |
| 410 | + } | |
| 308 | 411 | } elseif ($depth_delta < 0) { |
| 309 | 412 | |
| 310 | 413 | array_pop($this->open_nodes); |
| 311 | 414 | if ($this->isRecordEnd($node_name)) { |
| @@ -313,15 +416,13 @@ | ||
| 313 | 416 | |
| 314 | 417 | $this->setIndex( |
| 315 | 418 | $this->record_counter, |
| 316 | 419 | $this->record_start_index, |
| 317 | - $this->chunk_offset + $this->record_offset + strlen($node) | |
| 420 | + $offset + strlen($node) | |
| 318 | 421 | ); |
| 319 | 422 | $this->record_counter++; |
| 320 | 423 | } |
| 321 | 424 | } |
| 322 | - | |
| 323 | - // echo sprintf("%s %s - %d\n", str_repeat("\t", $this->depth), $node_name, $depth_delta); | |
| 324 | 425 | } |
| 325 | 426 | |
| 326 | 427 | /** |
| 327 | 428 | * Check for record start node |
| @@ -329,9 +430,9 @@ | ||
| 329 | 430 | * @param $node_name |
| 330 | 431 | * |
| 331 | 432 | * @return bool |
| 332 | 433 | */ |
| 333 | - private function isRecordStart($node_name) | |
| 434 | + public function isRecordStart($node_name) | |
| 334 | 435 | { |
| 335 | 436 | |
| 336 | 437 | if ($this->base_path_segments === $this->open_nodes && $node_name === $this->base_path) { |
| 337 | 438 | return true; |
| @@ -346,24 +447,23 @@ | ||
| 346 | 447 | * @param $node_name |
| 347 | 448 | * |
| 348 | 449 | * @return bool |
| 349 | 450 | */ |
| 350 | - private function isRecordEnd($node_name) | |
| 451 | + public function isRecordEnd($node_name) | |
| 351 | 452 | { |
| 352 | - if ($this->record_opened === true && $node_name === $this->base_path) { | |
| 453 | + if ($this->record_opened === false) { | |
| 454 | + return false; | |
| 455 | + } | |
| 353 | 456 | |
| 354 | - // check to see if we are a nested duplicate node name /Events/Event/Instance and we are closing Event | |
| 355 | - if (in_array($node_name, $this->open_nodes)) { | |
| 356 | - $array_counts = array_count_values($this->open_nodes); | |
| 357 | - if (isset($array_counts[$node_name]) && $array_counts[$node_name] > 0) { | |
| 358 | - return false; | |
| 359 | - } | |
| 360 | - } | |
| 457 | + if ($this->base_path_segments !== $this->open_nodes) { | |
| 458 | + return false; | |
| 459 | + } | |
| 361 | 460 | |
| 362 | - return true; | |
| 461 | + if ($node_name !== $this->base_path) { | |
| 462 | + return false; | |
| 363 | 463 | } |
| 364 | 464 | |
| 365 | - return false; | |
| 465 | + return true; | |
| 366 | 466 | } |
| 367 | 467 | |
| 368 | 468 | public function wrapWithRecordTag($record) |
| 369 | 469 | { |
| @@ -372,9 +472,10 @@ | ||
| 372 | 472 | |
| 373 | 473 | // clear sub namespaces |
| 374 | 474 | $record = preg_replace('/xmlns[^=]*="[^"]*"/i', '', $record); |
| 375 | 475 | |
| 376 | - $temp = $record; | |
| 476 | + // No need to check within cdata tags | |
| 477 | + $temp = preg_replace('/<!\[CDATA\[((?:[^]]|\](?!\]>))*)\]\]>/', '', $record); | |
| 377 | 478 | |
| 378 | 479 | // $new_regex = '/<([^:!\/]+):[^ ]+[^>]*>/'; |
| 379 | 480 | // $new_regex = '/<([^!\/>]+):[^>]+>/'; // breaking due to attr="http://", would match the : |
| 380 | 481 | $new_regex = '/<([^!\/">]+):[^>]+>/'; |
| @@ -389,9 +490,22 @@ | ||
| 389 | 490 | $namespace = $namespace_parts[count($namespace_parts) - 1]; |
| 390 | 491 | |
| 391 | 492 | if (!in_array($namespace, $unique_matches)) { |
| 392 | 493 | $unique_matches[] = $namespace; |
| 393 | - $namespace_attrs .= ' xmlns:' . $namespace . '="false"'; | |
| 494 | + | |
| 495 | + // load namespace list if needed | |
| 496 | + if (is_null($this->namespace_list) && !is_null($this->config)) { | |
| 497 | + $this->namespace_list = $this->config->get('namespace_list'); | |
| 498 | + if (!is_array($this->namespace_list)) { | |
| 499 | + $this->namespace_list = array(); | |
| 500 | + } | |
| 501 | + } | |
| 502 | + | |
| 503 | + if (isset($this->namespace_list[$namespace])) { | |
| 504 | + $namespace_attrs .= ' xmlns:' . $namespace . '="' . $this->namespace_list[$namespace] . '"'; | |
| 505 | + } else { | |
| 506 | + $namespace_attrs .= ' xmlns:' . $namespace . '="false"'; | |
| 507 | + } | |
| 394 | 508 | } |
| 395 | 509 | |
| 396 | 510 | $temp = substr($temp, $offset + strlen($match)); |
| 397 | 511 | } |
| @@ -401,9 +515,9 @@ | ||
| 401 | 515 | $close_tag = '</record>'; |
| 402 | 516 | return $open_tag . $record . $close_tag; |
| 403 | 517 | } |
| 404 | 518 | |
| 405 | - private function record_open_nodes() | |
| 519 | + public function record_open_nodes() | |
| 406 | 520 | { |
| 407 | 521 | $key = '/' . implode('/', $this->open_nodes); |
| 408 | 522 | if (!in_array($key, $this->node_list, true)) { |
| 409 | 523 | array_push($this->node_list, $key); |
| @@ -428,9 +542,9 @@ | ||
| 428 | 542 | |
| 429 | 543 | return $this->node_list; |
| 430 | 544 | } |
| 431 | 545 | |
| 432 | - protected final function getFileIndexKey() | |
| 546 | + public final function getFileIndexKey() | |
| 433 | 547 | { |
| 434 | 548 | $key = sprintf('file_index-%s', $this->record_patch_cache_key); |
| 435 | 549 | |
| 436 | 550 | return $key; |
