| @@ -4,16 +4,22 @@ | ||
| 4 | 4 | |
| 5 | 5 | use ImportWP\Common\Filesystem\Filesystem; |
| 6 | 6 | use ImportWP\Common\Importer\Config\Config; |
| 7 | 7 | use ImportWP\Common\Importer\File\CSVFile; |
| 8 | +use ImportWP\Common\Importer\File\JSONFile; | |
| 8 | 9 | use ImportWP\Common\Importer\File\XMLFile; |
| 10 | +use ImportWP\Common\Importer\Mapper\AttachmentMapper; | |
| 11 | +use ImportWP\Common\Importer\Mapper\CommentMapper; | |
| 9 | 12 | use ImportWP\Common\Importer\Mapper\PostMapper; |
| 10 | 13 | use ImportWP\Common\Importer\Mapper\TermMapper; |
| 11 | 14 | use ImportWP\Common\Importer\Mapper\UserMapper; |
| 12 | 15 | use ImportWP\Common\Importer\Parser\CSVParser; |
| 16 | +use ImportWP\Common\Importer\Parser\JSONParser; | |
| 13 | 17 | use ImportWP\Common\Importer\Parser\XMLParser; |
| 14 | 18 | use ImportWP\Common\Importer\Permission\Permission; |
| 15 | 19 | use ImportWP\Common\Importer\State\ImporterState; |
| 20 | +use ImportWP\Common\Importer\Template\AttachmentTemplate; | |
| 21 | +use ImportWP\Common\Importer\Template\CommentTemplate; | |
| 16 | 22 | use ImportWP\Common\Importer\Template\CustomPostTypeTemplate; |
| 17 | 23 | use ImportWP\Common\Importer\Template\PageTemplate; |
| 18 | 24 | use ImportWP\Common\Importer\Template\PostTemplate; |
| 19 | 25 | use ImportWP\Common\Importer\Template\Template; |
| @@ -19,10 +25,12 @@ | ||
| 19 | 25 | use ImportWP\Common\Importer\Template\Template; |
| 20 | 26 | use ImportWP\Common\Importer\Template\TemplateManager; |
| 21 | 27 | use ImportWP\Common\Importer\Template\TermTemplate; |
| 22 | 28 | use ImportWP\Common\Importer\Template\UserTemplate; |
| 29 | +use ImportWP\Common\Migration\Migrations; | |
| 23 | 30 | use ImportWP\Common\Model\ImporterModel; |
| 24 | 31 | use ImportWP\Common\Properties\Properties; |
| 32 | +use ImportWP\Common\Runner\ImporterRunnerState; | |
| 25 | 33 | use ImportWP\Common\Util\Logger; |
| 26 | 34 | use ImportWP\Common\Util\Util; |
| 27 | 35 | use ImportWP\Container; |
| 28 | 36 | use ImportWP\EventHandler; |
| @@ -49,11 +57,57 @@ | ||
| 49 | 57 | { |
| 50 | 58 | $this->filesystem = $filesystem; |
| 51 | 59 | $this->template_manager = $template_manager; |
| 52 | 60 | $this->event_handler = $event_handler; |
| 61 | + | |
| 62 | + add_action('admin_init', [$this, 'download_debug_log']); | |
| 53 | 63 | } |
| 54 | 64 | |
| 55 | 65 | /** |
| 66 | + * Stream an importer debug log through an authenticated admin request. | |
| 67 | + * | |
| 68 | + * Direct public URLs under uploads/importwp are blocked by .htaccess (CVE-2025-12894). | |
| 69 | + * | |
| 70 | + * @return void | |
| 71 | + */ | |
| 72 | + public function download_debug_log() | |
| 73 | + { | |
| 74 | + if (!isset($_GET['page'], $_GET['import'], $_GET['download_debug']) || $_GET['page'] !== 'importwp') { | |
| 75 | + return; | |
| 76 | + } | |
| 77 | + | |
| 78 | + if (!is_user_logged_in() || !current_user_can('manage_options')) { | |
| 79 | + wp_die(esc_html__('You do not have permission to download this file.', 'jc-importer'), '', array('response' => 403)); | |
| 80 | + } | |
| 81 | + | |
| 82 | + if (!isset($_GET['_wpnonce']) || !wp_verify_nonce(sanitize_key(wp_unslash($_GET['_wpnonce'])), 'iwp_debug_log_download')) { | |
| 83 | + wp_die(esc_html__('Invalid download request.', 'jc-importer'), '', array('response' => 403)); | |
| 84 | + } | |
| 85 | + | |
| 86 | + $importer_id = intval($_GET['import']); | |
| 87 | + $importer_data = $this->get_importer($importer_id); | |
| 88 | + if (!$importer_data) { | |
| 89 | + wp_die(esc_html__('Invalid download request.', 'jc-importer'), '', array('response' => 403)); | |
| 90 | + } | |
| 91 | + | |
| 92 | + if (!$this->is_debug()) { | |
| 93 | + wp_die(esc_html__('Debug mode is not enabled.', 'jc-importer'), '', array('response' => 403)); | |
| 94 | + } | |
| 95 | + | |
| 96 | + $file_path = Logger::getLogFile($importer_id); | |
| 97 | + if (!is_string($file_path) || !file_exists($file_path)) { | |
| 98 | + wp_die(esc_html__('Debug log file not found.', 'jc-importer'), '', array('response' => 404)); | |
| 99 | + } | |
| 100 | + | |
| 101 | + nocache_headers(); | |
| 102 | + header('Content-Type: text/plain; charset=utf-8'); | |
| 103 | + header('Content-Disposition: attachment; filename="' . basename($file_path) . '"'); | |
| 104 | + header('Content-Length: ' . (string) filesize($file_path)); | |
| 105 | + readfile($file_path); | |
| 106 | + exit; | |
| 107 | + } | |
| 108 | + | |
| 109 | + /** | |
| 56 | 110 | * Get Importers |
| 57 | 111 | * |
| 58 | 112 | * @return ImporterModel[] |
| 59 | 113 | */ |
| @@ -141,8 +195,10 @@ | ||
| 141 | 195 | if ('xml' === $parser) { |
| 142 | 196 | return $this->get_xml_file($importer, $config); |
| 143 | 197 | } elseif ('csv' === $parser) { |
| 144 | 198 | return $this->get_csv_file($importer, $config); |
| 199 | + } elseif ('json' === $parser) { | |
| 200 | + return $this->get_json_file($importer, $config); | |
| 145 | 201 | } |
| 146 | 202 | |
| 147 | 203 | return false; |
| 148 | 204 | } |
| @@ -152,8 +208,9 @@ | ||
| 152 | 208 | $importer = $this->get_importer($id); |
| 153 | 209 | $file = new CSVFile($importer->getFile(), $config); |
| 154 | 210 | $file->setDelimiter($importer->getFileSetting('delimiter')); |
| 155 | 211 | $file->setEnclosure($importer->getFileSetting('enclosure')); |
| 212 | + $file->setEscape($importer->getFileSetting('escape', "\\")); | |
| 156 | 213 | return $file; |
| 157 | 214 | } |
| 158 | 215 | |
| 159 | 216 | public function get_xml_file($id, $config) |
| @@ -163,8 +220,16 @@ | ||
| 163 | 220 | $file->setRecordPath($importer->getFileSetting('base_path')); |
| 164 | 221 | return $file; |
| 165 | 222 | } |
| 166 | 223 | |
| 224 | + public function get_json_file($id, $config) | |
| 225 | + { | |
| 226 | + $importer = $this->get_importer($id); | |
| 227 | + $file = new JSONFile($importer->getFile(), $config); | |
| 228 | + $file->setRecordPath($importer->getFileSetting('base_path')); | |
| 229 | + return $file; | |
| 230 | + } | |
| 231 | + | |
| 167 | 232 | public function preview_csv_file($id, $fields = [], $row = 0) |
| 168 | 233 | { |
| 169 | 234 | $importer = $this->get_importer($id); |
| 170 | 235 | $config = $this->get_config($importer, true); |
| @@ -188,8 +253,20 @@ | ||
| 188 | 253 | $record = $parser->getRecord($row); |
| 189 | 254 | return $record->queryGroup(['fields' => $fields]); |
| 190 | 255 | } |
| 191 | 256 | |
| 257 | + public function preview_json_file($id, $fields = [], $row = 0) | |
| 258 | + { | |
| 259 | + $importer = $this->get_importer($id); | |
| 260 | + $config = $this->get_config($importer, true); | |
| 261 | + | |
| 262 | + $file = $this->get_json_file($importer, $config); | |
| 263 | + $parser = new JSONParser($file); | |
| 264 | + | |
| 265 | + $record = $parser->getRecord($row); | |
| 266 | + return $record->queryGroup(['fields' => $fields]); | |
| 267 | + } | |
| 268 | + | |
| 192 | 269 | public function process_csv_file($id, $delimiter, $enclosure, $tmp = false) |
| 193 | 270 | { |
| 194 | 271 | $importer = $this->get_importer($id); |
| 195 | 272 | $config = $this->get_config($importer->getId(), $tmp); |
| @@ -198,36 +275,8 @@ | ||
| 198 | 275 | $file->setDelimiter($delimiter); |
| 199 | 276 | $file->setEnclosure($enclosure); |
| 200 | 277 | $file->processing(true); |
| 201 | 278 | |
| 202 | - if (empty($importer->getMap())) { | |
| 203 | - | |
| 204 | - // we are on first file import | |
| 205 | - $headings = str_getcsv($file->getRecord(0), $file->getDelimiter(), $file->getEnclosure()); | |
| 206 | - $headings = array_map('trim', $headings); | |
| 207 | - | |
| 208 | - $template = $this->get_importer_template($id); | |
| 209 | - $field_map = $template->generate_field_map($headings, $importer); | |
| 210 | - $field_map = apply_filters('iwp/importer/generate_field_map', $field_map, $headings, $importer); | |
| 211 | - | |
| 212 | - $map = $field_map['map']; | |
| 213 | - foreach ($map as $key => $value) { | |
| 214 | - $importer->setMap($key, $value); | |
| 215 | - } | |
| 216 | - | |
| 217 | - $enabled = $field_map['enabled']; | |
| 218 | - foreach ($enabled as $enabled_field) { | |
| 219 | - $importer->setEnabled($enabled_field); | |
| 220 | - } | |
| 221 | - | |
| 222 | - // Disabled due to testing: | |
| 223 | - // https://localdev/wp-admin/tools.php?page=importwp&edit=29&step=1 | |
| 224 | - // | |
| 225 | - // a:9:{s:8:"template";s:19:"woocommerce-product";s:13:"template_type";s:0:"";s:4:"file";a:2:{s:2:"id";i:9;s:8:"settings";a:6:{s:9:"enclosure";s:1:""";s:9:"delimiter";s:1:",";s:13:"show_headings";b:1;s:5:"setup";b:0;s:5:"count";i:2;s:9:"processed";b:1;}}s:10:"datasource";a:2:{s:4:"type";s:5:"local";s:8:"settings";a:2:{s:10:"remote_url";s:0:"";s:9:"local_url";s:48:"/var/www/html/wp-content/uploads/exportwp/30.csv";}}s:6:"parser";s:3:"csv";s:3:"map";a:0:{}s:7:"enabled";a:0:{}s:11:"permissions";a:0:{}s:8:"settings";a:4:{s:9:"post_type";a:2:{i:0;s:7:"product";i:1;s:17:"product_variation";}s:12:"unique_field";a:3:{i:0;s:2:"ID";i:1;s:4:"_sku";i:2;s:9:"post_name";}s:9:"start_row";N;s:7:"max_row";N;}} | |
| 226 | - // | |
| 227 | - $importer->save(); | |
| 228 | - } | |
| 229 | - | |
| 230 | 279 | return $file->getRecordCount(); |
| 231 | 280 | } |
| 232 | 281 | |
| 233 | 282 | public function process_xml_file($id, $tmp = false) |
| @@ -252,8 +301,19 @@ | ||
| 252 | 301 | |
| 253 | 302 | return $results; |
| 254 | 303 | } |
| 255 | 304 | |
| 305 | + public function process_json_file($id, $tmp = false) | |
| 306 | + { | |
| 307 | + $importer = $this->get_importer($id); | |
| 308 | + $config = $this->get_config($importer->getId(), $tmp); | |
| 309 | + | |
| 310 | + $file = new JSONFile($importer->getFile(), $config); | |
| 311 | + $file->processing(true); | |
| 312 | + | |
| 313 | + return $file->get_path_list(); | |
| 314 | + } | |
| 315 | + | |
| 256 | 316 | /** |
| 257 | 317 | * Link import file to importer via post meta |
| 258 | 318 | * |
| 259 | 319 | * @param ImporterModel $id |
| @@ -269,8 +329,15 @@ | ||
| 269 | 329 | } |
| 270 | 330 | |
| 271 | 331 | $index++; |
| 272 | 332 | |
| 333 | + // Store uploads-relative paths so site moves / open_basedir changes do not break lookups. | |
| 334 | + $file_path = wp_normalize_path($file_path); | |
| 335 | + $relative = Filesystem::to_uploads_relative_path($file_path); | |
| 336 | + if ($relative !== '' && $relative !== $file_path) { | |
| 337 | + $file_path = $relative; | |
| 338 | + } | |
| 339 | + | |
| 273 | 340 | update_post_meta($importer->getId(), '_importer_files', $index); |
| 274 | 341 | update_post_meta($importer->getId(), '_importer_file_' . $index, $file_path); |
| 275 | 342 | return $index; |
| 276 | 343 | } |
| @@ -284,11 +351,50 @@ | ||
| 284 | 351 | $file_index = 1; |
| 285 | 352 | } |
| 286 | 353 | $file_index++; |
| 287 | 354 | |
| 288 | - return $importer->getId() . '-' . intval($file_index) . '-'; | |
| 355 | + return $importer->getId() . '-' . intval($file_index) . '-' . wp_generate_password(12, false, false) . '-'; | |
| 289 | 356 | } |
| 290 | 357 | |
| 358 | + public function set_custom_upload_path($dir) | |
| 359 | + { | |
| 360 | + remove_filter('upload_dir', [$this, 'set_custom_upload_path']); | |
| 361 | + | |
| 362 | + $path = $this->filesystem->get_temp_directory(); | |
| 363 | + $url = $this->filesystem->get_temp_directory(true); | |
| 364 | + | |
| 365 | + add_filter('upload_dir', [$this, 'set_custom_upload_path']); | |
| 366 | + | |
| 367 | + $path .= '/uploads'; | |
| 368 | + $url .= '/uploads'; | |
| 369 | + | |
| 370 | + if (!is_dir($path)) { | |
| 371 | + wp_mkdir_p($path); | |
| 372 | + } | |
| 373 | + | |
| 374 | + if (!file_exists($path . '/.htaccess')) { | |
| 375 | + file_put_contents($path . '/.htaccess', "# Apache 2.4+ | |
| 376 | +<IfModule mod_authz_core.c> | |
| 377 | + Require all denied | |
| 378 | +</IfModule> | |
| 379 | + | |
| 380 | +# Apache 2.2 and older (or when mod_authz_core isn't available) | |
| 381 | +<IfModule !mod_authz_core.c> | |
| 382 | + Deny from all | |
| 383 | +</IfModule>"); | |
| 384 | + } | |
| 385 | + | |
| 386 | + if (!file_exists($path . '/index.html')) { | |
| 387 | + touch($path . '/index.html'); | |
| 388 | + } | |
| 389 | + | |
| 390 | + return array( | |
| 391 | + 'path' => $path, | |
| 392 | + 'url' => $url, | |
| 393 | + 'subdir' => '/importwp/uploads', | |
| 394 | + ) + $dir; | |
| 395 | + } | |
| 396 | + | |
| 291 | 397 | public function upload_file($id, $file) |
| 292 | 398 | { |
| 293 | 399 | $importer = $this->get_importer($id); |
| 294 | 400 | Logger::setId($importer->getId()); |
| @@ -293,10 +399,22 @@ | ||
| 293 | 399 | $importer = $this->get_importer($id); |
| 294 | 400 | Logger::setId($importer->getId()); |
| 295 | 401 | |
| 296 | 402 | $allowed_file_types = $this->event_handler->run('importer.allowed_file_types', [$importer->getAllowedFileTypes()]); |
| 403 | + | |
| 404 | + $prefix = $this->get_importer_file_prefix($importer); | |
| 405 | + | |
| 406 | + $prefix_upload = function ($file) use ($prefix) { | |
| 407 | + $file['name'] = $prefix . $file['name']; | |
| 408 | + return $file; | |
| 409 | + }; | |
| 410 | + | |
| 411 | + add_filter('wp_handle_upload_prefilter', $prefix_upload); | |
| 412 | + | |
| 297 | 413 | $result = $this->filesystem->upload_file($file, $allowed_file_types); |
| 298 | 414 | |
| 415 | + remove_filter('wp_handle_upload_prefilter', $prefix_upload); | |
| 416 | + | |
| 299 | 417 | if (is_wp_error($result)) { |
| 300 | 418 | return $result; |
| 301 | 419 | } |
| 302 | 420 | |
| @@ -314,10 +432,86 @@ | ||
| 314 | 432 | Logger::setId($importer->getId()); |
| 315 | 433 | |
| 316 | 434 | $allowed_file_types = $this->event_handler->run('importer.allowed_file_types', [$importer->getAllowedFileTypes()]); |
| 317 | 435 | $prefix = $this->get_importer_file_prefix($importer); |
| 318 | - $result = $this->filesystem->download_file($source, $filetype, $allowed_file_types, null, $prefix); | |
| 319 | 436 | |
| 437 | + try { | |
| 438 | + | |
| 439 | + if (preg_match('/^s?ftp?:\/\//', $source) === 1) { | |
| 440 | + | |
| 441 | + if (preg_match('/^(?<protocol>s?ftp):\/\/(?:(?<user>[^\:@]+)(?:\:(?<pass>[^@]+))?@)?(?<host>[^\:\/]+)(?:\:(?<port>[0-9]+))?(?:\/(?<path>.*))$/', $source, $matches) !== 1) { | |
| 442 | + | |
| 443 | + return new \WP_Error("IM_RM_FTP_PARSE", __("Unable to parse FTP connection string", 'jc-importer')); | |
| 444 | + } | |
| 445 | + | |
| 446 | + $protocol = isset($matches['protocol']) ? $matches['protocol'] : 'ftp'; | |
| 447 | + $user = isset($matches['user']) ? urldecode($matches['user']) : ''; | |
| 448 | + $pass = isset($matches['pass']) ? urldecode($matches['pass']) : ''; | |
| 449 | + $host = isset($matches['host']) ? $matches['host'] : false; | |
| 450 | + $port = isset($matches['port']) && !empty($matches['port']) ? $matches['port'] : intval(21); | |
| 451 | + $path = isset($matches['path']) ? $matches['path'] : false; | |
| 452 | + | |
| 453 | + if (!$host) { | |
| 454 | + return new \WP_Error("IM_RM_FTP_HOST", __("Unable to parse ftp host from connection string", 'jc-importer')); | |
| 455 | + } | |
| 456 | + | |
| 457 | + if (!$path) { | |
| 458 | + return new \WP_Error("IM_RM_FTP_HOST", __("Unable to parse ftp host from connection string", 'jc-importer')); | |
| 459 | + } | |
| 460 | + | |
| 461 | + /** | |
| 462 | + * @var \ImportWP\Common\Ftp\Ftp $ftp | |
| 463 | + */ | |
| 464 | + $ftp = Container::getInstance()->get('ftp'); | |
| 465 | + | |
| 466 | + $path = apply_filters('iwp/importer/remote_file', $path, $importer); | |
| 467 | + $path = apply_filters(sprintf('iwp/importer=%d/remote_file', $importer->getId()), $path, $importer); | |
| 468 | + | |
| 469 | + $filter_connection_args = [ | |
| 470 | + 'user' => $user, | |
| 471 | + 'pass' => $pass, | |
| 472 | + 'host' => $host, | |
| 473 | + 'port' => $port, | |
| 474 | + 'path' => $path, | |
| 475 | + ]; | |
| 476 | + $path = apply_filters(sprintf('iwp/importer/remote_file/source=%s', 'ftp'), $path, $importer, $filter_connection_args); | |
| 477 | + $path = apply_filters(sprintf('iwp/importer=%d/remote_file/source=ftp', $importer->getId(), 'ftp'), $path, $importer, $filter_connection_args); | |
| 478 | + | |
| 479 | + if ($protocol == 'sftp') { | |
| 480 | + | |
| 481 | + // require sftp package | |
| 482 | + require_once __DIR__ . '/../../../libs/autoload.php'; | |
| 483 | + | |
| 484 | + $sftp = new \phpseclib3\Net\SFTP($host, $port); | |
| 485 | + if (!$sftp->login($user, $pass)) { | |
| 486 | + return new \WP_Error('IWP_FTP_0', __("Unable to login to ftp server", 'jc-importer')); | |
| 487 | + } | |
| 488 | + | |
| 489 | + $wp_upload_dir = wp_upload_dir(); | |
| 490 | + | |
| 491 | + $dest = wp_unique_filename($wp_upload_dir['path'], basename($path)); | |
| 492 | + $wp_dest = $wp_upload_dir['path'] . '/' . $dest; | |
| 493 | + | |
| 494 | + if (!$sftp->get($path, $wp_dest)) { | |
| 495 | + return new \WP_Error('IWP_FTP_2', sprintf(__('Unable to download: %s file via sftp.', 'jc-importer'), $path)); | |
| 496 | + } | |
| 497 | + | |
| 498 | + $result = array( | |
| 499 | + 'dest' => $wp_dest, | |
| 500 | + 'type' => $this->filesystem->get_filetype($wp_dest), | |
| 501 | + 'mime' => $this->filesystem->get_file_mime($wp_dest) | |
| 502 | + ); | |
| 503 | + } else { | |
| 504 | + $result = $ftp->download_file($path, $host, $user, $pass, false, $port); | |
| 505 | + } | |
| 506 | + } else { | |
| 507 | + | |
| 508 | + $result = $this->filesystem->download_file($source, $filetype, $allowed_file_types, null, $prefix); | |
| 509 | + } | |
| 510 | + } catch (\Exception $e) { | |
| 511 | + return new \WP_Error($e->getCode(), $e->getMessage()); | |
| 512 | + } | |
| 513 | + | |
| 320 | 514 | if (is_wp_error($result)) { |
| 321 | 515 | return $result; |
| 322 | 516 | } |
| 323 | 517 | |
| @@ -328,16 +522,34 @@ | ||
| 328 | 522 | |
| 329 | 523 | return $attachment_id; |
| 330 | 524 | } |
| 331 | 525 | |
| 332 | - public function local_file($id, $source) | |
| 526 | + public function local_file($id, $source, $filetype = null) | |
| 333 | 527 | { |
| 334 | 528 | $importer = $this->get_importer($id); |
| 335 | 529 | Logger::setId($importer->getId()); |
| 336 | 530 | |
| 531 | + $allowed_bases = apply_filters('iwp/importer/local_file/allowed_directories', [ | |
| 532 | + realpath(WP_CONTENT_DIR) . DIRECTORY_SEPARATOR | |
| 533 | + ]); | |
| 534 | + | |
| 535 | + $source = realpath($source); | |
| 536 | + | |
| 537 | + $is_allowed = false; | |
| 538 | + foreach ($allowed_bases as $allowed_base) { | |
| 539 | + if ($allowed_base && strpos($source, $allowed_base) === 0) { | |
| 540 | + $is_allowed = true; | |
| 541 | + break; | |
| 542 | + } | |
| 543 | + } | |
| 544 | + | |
| 545 | + if (!$source || !$is_allowed) { | |
| 546 | + return new \WP_Error('IWP_LOCAL_FILE_1', __('Access to this file path is not allowed.', 'jc-importer')); | |
| 547 | + } | |
| 548 | + | |
| 337 | 549 | $allowed_file_types = $this->event_handler->run('importer.allowed_file_types', [$importer->getAllowedFileTypes()]); |
| 338 | 550 | $prefix = $this->get_importer_file_prefix($importer); |
| 339 | - $result = $this->filesystem->copy_file($source, $allowed_file_types, null, $prefix); | |
| 551 | + $result = $this->filesystem->copy_file($source, $allowed_file_types, null, $prefix, $filetype); | |
| 340 | 552 | |
| 341 | 553 | if (is_wp_error($result)) { |
| 342 | 554 | return $result; |
| 343 | 555 | } |
| @@ -371,9 +583,9 @@ | ||
| 371 | 583 | $file_path = apply_filters('iwp/importer/file_uploaded/file_path', $file_path, $importer_model); |
| 372 | 584 | |
| 373 | 585 | $file_id = $this->link_importer_file($id, $file_path); |
| 374 | 586 | if (!$file_id) { |
| 375 | - return new \WP_Error('IWP_IM_01', 'Unable to link importer file'); | |
| 587 | + return new \WP_Error('IWP_IM_01', __('Unable to link importer file', 'jc-importer')); | |
| 376 | 588 | } |
| 377 | 589 | |
| 378 | 590 | if (is_null($importer_model->getParser())) { |
| 379 | 591 | $importer_model->setParser($file_type); |
| @@ -434,9 +646,9 @@ | ||
| 434 | 646 | |
| 435 | 647 | $base .= str_pad($id, 2, STR_PAD_LEFT) . DIRECTORY_SEPARATOR; |
| 436 | 648 | if (!file_exists($base)) { |
| 437 | 649 | if (!is_writable(dirname($base)) || !mkdir($base)) { |
| 438 | - throw new \Exception("Unable to create directory: " . $base); | |
| 650 | + throw new \Exception(sprintf(__("Unable to create directory: %s", 'jc-importer'), $base)); | |
| 439 | 651 | } |
| 440 | 652 | } |
| 441 | 653 | |
| 442 | 654 | for ($i = 0; $i < ceil(strlen($session) / 2); $i++) { |
| @@ -443,9 +655,9 @@ | ||
| 443 | 655 | $base .= substr($session, $i * 2, 2) . DIRECTORY_SEPARATOR; |
| 444 | 656 | if (!file_exists($base)) { |
| 445 | 657 | |
| 446 | 658 | if (!is_writable(dirname($base)) || !mkdir($base)) { |
| 447 | - throw new \Exception("Unable to create directory: " . $base); | |
| 659 | + throw new \Exception(sprintf(__("Unable to create directory: %s", 'jc-importer'), $base)); | |
| 448 | 660 | } |
| 449 | 661 | } |
| 450 | 662 | |
| 451 | 663 | if ($i >= $max_depth - 2) { |
| @@ -472,13 +684,26 @@ | ||
| 472 | 684 | public function import($id, $user, $session = null) |
| 473 | 685 | { |
| 474 | 686 | Logger::timer(); |
| 475 | 687 | |
| 688 | + // Run pending DB/data migrations before loading importer config so cron, | |
| 689 | + // CLI, and REST imports do not depend on the admin setup wizard. | |
| 690 | + $migrations = new Migrations(); | |
| 691 | + if (!$migrations->isSetup()) { | |
| 692 | + $migrations->migrate(); | |
| 693 | + if ($id instanceof ImporterModel) { | |
| 694 | + $id = $id->getId(); | |
| 695 | + } | |
| 696 | + } | |
| 697 | + | |
| 476 | 698 | $importer_data = $this->get_importer($id); |
| 477 | 699 | $importer_id = $importer_data->getId(); |
| 478 | 700 | |
| 479 | - $config_data = get_site_option('iwp_importer_config_' . $importer_id, []); | |
| 701 | + // store current importer | |
| 702 | + iwp()->importer = $importer_data; | |
| 480 | 703 | |
| 704 | + $config_data = get_option('iwp_importer_config_' . $importer_id, []); | |
| 705 | + | |
| 481 | 706 | $this->event_handler->run('importer_manager.import', [$importer_data]); |
| 482 | 707 | |
| 483 | 708 | $state = new ImporterState($importer_id, $user); |
| 484 | 709 | |
| @@ -484,11 +709,16 @@ | ||
| 484 | 709 | |
| 485 | 710 | try { |
| 486 | 711 | |
| 487 | 712 | Logger::debug('IM -init_state'); |
| 713 | + | |
| 714 | + // 1. Set State Session, and load its state | |
| 488 | 715 | $state->init($session); |
| 489 | 716 | |
| 717 | + // if this is a new session, clear config files | |
| 490 | 718 | if ($state->has_status('init')) { |
| 719 | + // rest importer log. | |
| 720 | + Logger::clear($importer_id); | |
| 491 | 721 | Logger::debug('IM -clear_config_files'); |
| 492 | 722 | $this->clear_config_files($importer_id, false, true); |
| 493 | 723 | $config_data['features'] = [ |
| 494 | 724 | 'session_table' => true |
| @@ -512,8 +742,9 @@ | ||
| 512 | 742 | // mapper |
| 513 | 743 | Logger::debug('IM -get_importer_mapper'); |
| 514 | 744 | $mapper = $this->get_importer_mapper($importer_data, $template, $permission); |
| 515 | 745 | |
| 746 | + // if this is a new session, build config | |
| 516 | 747 | if ($state->has_status('init')) { |
| 517 | 748 | |
| 518 | 749 | Logger::debug('IM -generate_config'); |
| 519 | 750 | |
| @@ -533,8 +764,45 @@ | ||
| 533 | 764 | $version = 0; |
| 534 | 765 | } |
| 535 | 766 | update_post_meta($importer_id, '_iwp_version', $version); |
| 536 | 767 | $config_data['version'] = $version; |
| 768 | + | |
| 769 | + /** | |
| 770 | + * Fetch new file if setting is checked | |
| 771 | + * @since 2.7.15 | |
| 772 | + */ | |
| 773 | + $run_fetch_file = $importer_data->getSetting('run_fetch') || false; | |
| 774 | + $run_fetch_file = apply_filters('iwp/importer/run_fetch_file', $run_fetch_file); | |
| 775 | + if ($run_fetch_file) { | |
| 776 | + | |
| 777 | + add_filter('upload_dir', [$this, 'set_custom_upload_path']); | |
| 778 | + | |
| 779 | + $datasource = $importer_data->getDatasource(); | |
| 780 | + switch ($datasource) { | |
| 781 | + case 'remote': | |
| 782 | + $raw_source = $importer_data->getDatasourceSetting('remote_url'); | |
| 783 | + $source = apply_filters('iwp/importer/datasource', $raw_source, $raw_source, $importer_data); | |
| 784 | + $source = apply_filters('iwp/importer/datasource/remote', $source, $raw_source, $importer_data); | |
| 785 | + $attachment_id = $this->remote_file($importer_data, $source, $importer_data->getParser()); | |
| 786 | + break; | |
| 787 | + case 'local': | |
| 788 | + $raw_source = $importer_data->getDatasourceSetting('local_url'); | |
| 789 | + $source = apply_filters('iwp/importer/datasource', $raw_source, $raw_source, $importer_data); | |
| 790 | + $source = apply_filters('iwp/importer/datasource/local', $source, $raw_source, $importer_data); | |
| 791 | + $attachment_id = $this->local_file($importer_data, $source, $importer_data->getParser()); | |
| 792 | + break; | |
| 793 | + default: | |
| 794 | + // TODO: record error | |
| 795 | + $attachment_id = new \WP_Error('IWP_CRON_1', sprintf(__('Unable to get new file using datasource: %s', 'jc-importer'), $datasource)); | |
| 796 | + break; | |
| 797 | + } | |
| 798 | + | |
| 799 | + remove_filter('upload_dir', [$this, 'set_custom_upload_path']); | |
| 800 | + | |
| 801 | + if (is_wp_error($attachment_id)) { | |
| 802 | + throw new \Exception(sprintf(__('Importer Datasource: %s', 'jc-importer'), $attachment_id->get_error_message())); | |
| 803 | + } | |
| 804 | + } | |
| 537 | 805 | } |
| 538 | 806 | |
| 539 | 807 | $start = 0; |
| 540 | 808 | |
| @@ -551,21 +819,43 @@ | ||
| 551 | 819 | Logger::debug('IM -get_xml_file'); |
| 552 | 820 | $file = $this->get_xml_file($importer_data, $config); |
| 553 | 821 | Logger::debug('IM -load_parser'); |
| 554 | 822 | $parser = new XMLParser($file); |
| 823 | + } elseif ($importer_data->getParser() === 'json') { | |
| 824 | + Logger::debug('IM -get_json_file'); | |
| 825 | + $file = $this->get_json_file($importer_data, $config); | |
| 826 | + Logger::debug('IM -load_parser'); | |
| 827 | + $parser = new JSONParser($file); | |
| 555 | 828 | } else { |
| 556 | 829 | $parser = apply_filters('iwp/importer/init_parser', false, $importer_data, $config); |
| 557 | 830 | } |
| 558 | 831 | |
| 832 | + if (!$parser || !is_object($parser) || !method_exists($parser, 'file')) { | |
| 833 | + $parser_type = $importer_data->getParser(); | |
| 834 | + throw new \Exception( | |
| 835 | + sprintf( | |
| 836 | + __('Unable to load importer parser for type: %s', 'jc-importer'), | |
| 837 | + $parser_type ? $parser_type : __('unknown', 'jc-importer') | |
| 838 | + ) | |
| 839 | + ); | |
| 840 | + } | |
| 841 | + | |
| 842 | + // if this is a new session, set start / end rows to state | |
| 559 | 843 | if ($state->has_status('init')) { |
| 560 | 844 | |
| 561 | 845 | Logger::debug('IM -get_record_count'); |
| 562 | 846 | $end = $parser->file()->getRecordCount(); |
| 563 | 847 | |
| 848 | + // Capture cancelled status from file processor | |
| 849 | + $raw_state = ImporterState::get_state($importer_data->getId()); | |
| 850 | + if ($raw_state['status'] === 'cancelled') { | |
| 851 | + return $raw_state; | |
| 852 | + } | |
| 853 | + | |
| 564 | 854 | $config_data['start'] = $this->get_start($importer_data, $start); |
| 565 | 855 | $config_data['end'] = $this->get_end($importer_data, $config_data['start'], $end); |
| 566 | 856 | |
| 567 | - update_site_option('iwp_importer_config_' . $importer_id, $config_data); | |
| 857 | + update_option('iwp_importer_config_' . $importer_id, $config_data); | |
| 568 | 858 | |
| 569 | 859 | Logger::debug('IM -update_state'); |
| 570 | 860 | $state->update(function ($state) use ($config_data) { |
| 571 | 861 | |
| @@ -579,10 +869,17 @@ | ||
| 579 | 869 | }); |
| 580 | 870 | |
| 581 | 871 | Logger::debug('IM -write_status_session_to_file'); |
| 582 | 872 | Util::write_status_session_to_file($id, $state); |
| 873 | + | |
| 874 | + do_action('iwp/importer/init', $importer_data); | |
| 583 | 875 | } |
| 584 | 876 | |
| 877 | + | |
| 878 | + add_filter('iwp/importer/mapper/hash_check_enabled', function ($enabled) use ($importer_data) { | |
| 879 | + return $importer_data->getSetting('hash_check'); | |
| 880 | + }); | |
| 881 | + | |
| 585 | 882 | Logger::debug('IM -import'); |
| 586 | 883 | $importer = new \ImportWP\Common\Importer\Importer($config); |
| 587 | 884 | $importer->parser($parser); |
| 588 | 885 | $importer->mapper($mapper); |
| @@ -594,9 +891,11 @@ | ||
| 594 | 891 | } catch (\Exception $e) { |
| 595 | 892 | |
| 596 | 893 | // TODO: Missing template errors are currently not being logged to history, possibly others? |
| 597 | 894 | Logger::error('import -error=' . $e->getMessage(), $importer_id); |
| 598 | - return $state->error($e)->get_raw(); | |
| 895 | + $state->error($e); | |
| 896 | + Util::write_status_session_to_file($id, $state); | |
| 897 | + return $state->get_raw(); | |
| 599 | 898 | } |
| 600 | 899 | |
| 601 | 900 | /** |
| 602 | 901 | * @var Properties $properties |
| @@ -616,8 +915,38 @@ | ||
| 616 | 915 | return $data; |
| 617 | 916 | })->get_raw(); |
| 618 | 917 | } |
| 619 | 918 | |
| 919 | + public function pause_import($importer_id, $paused) | |
| 920 | + { | |
| 921 | + // TODO: set flag for paused. | |
| 922 | + $state = ImporterState::get_state($importer_id); | |
| 923 | + if ($paused === 'no') { | |
| 924 | + ImporterState::clear_flag($importer_id); | |
| 925 | + $state['status'] = 'running'; | |
| 926 | + } else { | |
| 927 | + ImporterState::set_paused($importer_id); | |
| 928 | + $state['status'] = 'paused'; | |
| 929 | + } | |
| 930 | + | |
| 931 | + // good chance this will be overwritten | |
| 932 | + ImporterState::set_state($importer_id, $state); | |
| 933 | + | |
| 934 | + return $state; | |
| 935 | + } | |
| 936 | + | |
| 937 | + public function stop_import($importer_id) | |
| 938 | + { | |
| 939 | + ImporterState::set_cancelled($importer_id); | |
| 940 | + | |
| 941 | + // good chance this will be overwritten | |
| 942 | + $state = ImporterState::get_state($importer_id); | |
| 943 | + $state['status'] = 'cancelled'; | |
| 944 | + ImporterState::set_state($importer_id, $state); | |
| 945 | + | |
| 946 | + return $state; | |
| 947 | + } | |
| 948 | + | |
| 620 | 949 | public function get_start($importer_data, $start) |
| 621 | 950 | { |
| 622 | 951 | $tmp_start = $importer_data->getStartRow(); |
| 623 | 952 | if (!is_null($tmp_start) && "" !== $tmp_start) { |
| @@ -650,9 +979,9 @@ | ||
| 650 | 979 | $templates = $this->get_templates(); |
| 651 | 980 | $template_name = $importer_model->getTemplate(); |
| 652 | 981 | |
| 653 | 982 | if (!isset($templates[$template_name])) { |
| 654 | - $exception_msg = "Unable to locate importer template: " . $template_name; | |
| 983 | + $exception_msg = sprintf(__("Unable to locate importer template: %s", 'jc-importer'), $template_name); | |
| 655 | 984 | Logger::error('import -get_importer_template=' . $exception_msg, $importer_model->getId()); |
| 656 | 985 | throw new \Exception($exception_msg); |
| 657 | 986 | } |
| 658 | 987 | |
| @@ -681,9 +1010,11 @@ | ||
| 681 | 1010 | $mappers = $this->event_handler->run('mappers.register', [[]]); // apply_filters('iwp/mappers/register', []); |
| 682 | 1011 | $mappers = array_merge($mappers, [ |
| 683 | 1012 | 'post' => PostMapper::class, |
| 684 | 1013 | 'user' => UserMapper::class, |
| 685 | - 'term' => TermMapper::class | |
| 1014 | + 'term' => TermMapper::class, | |
| 1015 | + 'attachment' => AttachmentMapper::class, | |
| 1016 | + 'comment' => CommentMapper::class, | |
| 686 | 1017 | ]); |
| 687 | 1018 | return $mappers; |
| 688 | 1019 | } |
| 689 | 1020 | |
| @@ -690,9 +1021,9 @@ | ||
| 690 | 1021 | public function get_mapper($key) |
| 691 | 1022 | { |
| 692 | 1023 | $mappers = $this->get_mappers(); |
| 693 | 1024 | if (!isset($mappers[$key])) { |
| 694 | - return new \WP_Error('IWP_IM_1', 'Unable to locate mapper: ' . $key); | |
| 1025 | + return new \WP_Error('IWP_IM_1', sprintf(__('Unable to locate mapper: %s', 'jc-importer'), $key)); | |
| 695 | 1026 | } |
| 696 | 1027 | |
| 697 | 1028 | return $mappers[$key]; |
| 698 | 1029 | } |
| @@ -704,8 +1035,10 @@ | ||
| 704 | 1035 | 'post' => PostTemplate::class, |
| 705 | 1036 | 'page' => PageTemplate::class, |
| 706 | 1037 | 'user' => UserTemplate::class, |
| 707 | 1038 | 'term' => TermTemplate::class, |
| 1039 | + 'attachment' => AttachmentTemplate::class, | |
| 1040 | + 'comment' => CommentTemplate::class, | |
| 708 | 1041 | 'custom-post-type' => CustomPostTypeTemplate::class, |
| 709 | 1042 | ]); |
| 710 | 1043 | return $templates; |
| 711 | 1044 | } |
| @@ -713,9 +1046,9 @@ | ||
| 713 | 1046 | public function get_template($key) |
| 714 | 1047 | { |
| 715 | 1048 | $templates = $this->get_templates(); |
| 716 | 1049 | if (!isset($templates[$key])) { |
| 717 | - return new \WP_Error('IWP_IM_1', 'Unable to locate template: ' . $key); | |
| 1050 | + return new \WP_Error('IWP_IM_1', sprintf(__('Unable to locate template: %s', 'jc-importer'), $key)); | |
| 718 | 1051 | } |
| 719 | 1052 | |
| 720 | 1053 | return $templates[$key]; |
| 721 | 1054 | } |
