| @@ -9,12 +9,21 @@ | ||
| 9 | 9 | use ImportWP\Common\Exporter\Mapper\MapperData; |
| 10 | 10 | use ImportWP\Common\Exporter\Mapper\PostMapper; |
| 11 | 11 | use ImportWP\Common\Exporter\Mapper\TaxMapper; |
| 12 | 12 | use ImportWP\Common\Exporter\Mapper\UserMapper; |
| 13 | +use ImportWP\Common\Exporter\State\ExporterState; | |
| 13 | 14 | use ImportWP\Common\Model\ExporterModel; |
| 15 | +use ImportWP\Common\Util\Logger; | |
| 16 | +use ImportWP\Common\Util\Util; | |
| 17 | +use ImportWP\Container; | |
| 14 | 18 | |
| 15 | 19 | class ExporterManager |
| 16 | 20 | { |
| 21 | + /** | |
| 22 | + * @var int | |
| 23 | + */ | |
| 24 | + protected $memory_limit; | |
| 25 | + | |
| 17 | 26 | public function __construct() |
| 18 | 27 | { |
| 19 | 28 | add_action('admin_init', [$this, 'download_file']); |
| 20 | 29 | } |
| @@ -30,9 +39,21 @@ | ||
| 30 | 39 | if (!isset($_GET['page'], $_GET['exporter'], $_GET['download']) || $_GET['page'] !== 'importwp') { |
| 31 | 40 | return; |
| 32 | 41 | } |
| 33 | 42 | |
| 43 | + if (!is_user_logged_in() || !current_user_can('manage_options')) { | |
| 44 | + wp_die(esc_html__('You do not have permission to download this file.', 'jc-importer'), '', array('response' => 403)); | |
| 45 | + } | |
| 46 | + | |
| 47 | + if (!isset($_GET['_wpnonce']) || !wp_verify_nonce(sanitize_key(wp_unslash($_GET['_wpnonce'])), 'iwp_export_download')) { | |
| 48 | + wp_die(esc_html__('Invalid download request.', 'jc-importer'), '', array('response' => 403)); | |
| 49 | + } | |
| 50 | + | |
| 34 | 51 | $exporter_data = $this->get_exporter(intval($_GET['exporter'])); |
| 52 | + if (!$exporter_data) { | |
| 53 | + wp_die(esc_html__('Invalid download request.', 'jc-importer'), '', array('response' => 403)); | |
| 54 | + } | |
| 55 | + | |
| 35 | 56 | $file = sanitize_key($_GET['download']); |
| 36 | 57 | |
| 37 | 58 | $download_data = get_post_meta($exporter_data->getId(), '_ewp_file_' . $file, true); |
| 38 | 59 | if ($download_data) { |
| @@ -43,11 +64,21 @@ | ||
| 43 | 64 | header('Content-type: "text/' . $download_data['type'] . '"; charset="utf8"'); |
| 44 | 65 | readfile($download_data['path']); |
| 45 | 66 | die(); |
| 46 | 67 | } |
| 68 | + | |
| 69 | + wp_die(esc_html__('Invalid download request.', 'jc-importer'), '', array('response' => 403)); | |
| 47 | 70 | } |
| 48 | 71 | |
| 49 | 72 | /** |
| 73 | + * @return string | |
| 74 | + */ | |
| 75 | + protected function generate_download_key() | |
| 76 | + { | |
| 77 | + return bin2hex(random_bytes(16)); | |
| 78 | + } | |
| 79 | + | |
| 80 | + /** | |
| 50 | 81 | * Get Exporters |
| 51 | 82 | * |
| 52 | 83 | * @return ExporterModel[] |
| 53 | 84 | */ |
| @@ -111,11 +142,338 @@ | ||
| 111 | 142 | $exporter = $this->get_exporter($id); |
| 112 | 143 | $exporter->delete(); |
| 113 | 144 | } |
| 114 | 145 | |
| 115 | - public function export($id) | |
| 146 | + /** | |
| 147 | + * @param \ImportWP\Common\Model\ExporterModel $exporter_data | |
| 148 | + * @return \ImportWP\Common\Exporter\File\File | |
| 149 | + */ | |
| 150 | + public function get_exporter_file($exporter_data) | |
| 116 | 151 | { |
| 152 | + | |
| 153 | + if ($exporter_data->getFileType() === 'csv') { | |
| 154 | + $file = new CSVFile($exporter_data); | |
| 155 | + } elseif ($exporter_data->getFileType() === 'xml') { | |
| 156 | + $file = new XMLFile($exporter_data); | |
| 157 | + } else { | |
| 158 | + $file = new JSONFile($exporter_data); | |
| 159 | + } | |
| 160 | + | |
| 161 | + return $file; | |
| 162 | + } | |
| 163 | + | |
| 164 | + /** | |
| 165 | + * @param \ImportWP\Common\Model\ExporterModel $exporter_data | |
| 166 | + * @return \ImportWP\Common\Exporter\Mapper\MapperInterface | |
| 167 | + */ | |
| 168 | + public function get_exporter_mapper($exporter_data) | |
| 169 | + { | |
| 170 | + $type = $exporter_data->getType(); | |
| 171 | + $matches = null; | |
| 172 | + if (preg_match('/^ewp_tax_(.*?)$/', $type, $matches) == 1) { | |
| 173 | + $taxonomy = $matches[1]; | |
| 174 | + $mapper = new TaxMapper($taxonomy); | |
| 175 | + } elseif (preg_match('/^ewp_comment_(.*?)$/', $type, $matches) == 1) { | |
| 176 | + $post_type = $matches[1]; | |
| 177 | + $mapper = new CommentMapper($post_type); | |
| 178 | + } elseif ($type === 'user') { | |
| 179 | + $mapper = new UserMapper(); | |
| 180 | + } else { | |
| 181 | + | |
| 182 | + $mapper = apply_filters('iwp/exporter/load_mapper', false, $type); | |
| 183 | + if (!$mapper) { | |
| 184 | + $mapper = new PostMapper($type); | |
| 185 | + } | |
| 186 | + } | |
| 187 | + | |
| 188 | + return $mapper; | |
| 189 | + } | |
| 190 | + | |
| 191 | + public function export($id, $user = null, $session = null) | |
| 192 | + { | |
| 193 | + Logger::timer(); | |
| 194 | + | |
| 195 | + if (is_null($user)) { | |
| 196 | + $user = uniqid('iwp', true); | |
| 197 | + } | |
| 198 | + | |
| 117 | 199 | $exporter_data = $this->get_exporter($id); |
| 200 | + $exporter_id = $exporter_data->getId(); | |
| 201 | + | |
| 202 | + $config_data = []; // get_option('iwp_exporter_config_' . $exporter_id, []); | |
| 203 | + | |
| 204 | + $state = new ExporterState($exporter_id, $user); | |
| 205 | + | |
| 206 | + try { | |
| 207 | + | |
| 208 | + | |
| 209 | + $state->init($session); | |
| 210 | + | |
| 211 | + $file = $this->get_exporter_file($exporter_data); | |
| 212 | + $mapper = $this->get_exporter_mapper($exporter_data); | |
| 213 | + | |
| 214 | + /** | |
| 215 | + * @var MapperInterface $mapper | |
| 216 | + */ | |
| 217 | + $mapper->set_filters($exporter_data->getFilters()); | |
| 218 | + | |
| 219 | + // Default to showing all fields, if none selected | |
| 220 | + if (empty($exporter_data->getFields(true))) { | |
| 221 | + | |
| 222 | + $fields = $mapper->get_fields(); | |
| 223 | + | |
| 224 | + switch ($exporter_data->getFileType()) { | |
| 225 | + case 'csv': | |
| 226 | + $fields = $this->flattenFields($fields); | |
| 227 | + break; | |
| 228 | + default: | |
| 229 | + $fields = $this->nestedFields($fields); | |
| 230 | + break; | |
| 231 | + } | |
| 232 | + | |
| 233 | + | |
| 234 | + $exporter_data->setFields($fields); | |
| 235 | + } | |
| 236 | + | |
| 237 | + | |
| 238 | + | |
| 239 | + if ($state->has_status('init')) { | |
| 240 | + | |
| 241 | + if (!$mapper->have_records($exporter_id)) { | |
| 242 | + throw new \Exception(__("No records found to export", 'jc-importer')); | |
| 243 | + } | |
| 244 | + | |
| 245 | + // write | |
| 246 | + $config_data['id'] = $state->get_session(); | |
| 247 | + $config_data['version'] = 1; | |
| 248 | + $config_data['records'] = $mapper->get_records(); | |
| 249 | + | |
| 250 | + // save | |
| 251 | + update_option('iwp_exporter_config_' . $exporter_id, $config_data); | |
| 252 | + | |
| 253 | + $state->update(function ($state) use ($config_data) { | |
| 254 | + $state['id'] = $config_data['id']; | |
| 255 | + $state['status'] = 'running'; | |
| 256 | + $state['progress']['export']['start'] = 0; | |
| 257 | + $state['progress']['export']['end'] = count($config_data['records']); | |
| 258 | + return $state; | |
| 259 | + }); | |
| 260 | + | |
| 261 | + $file->wipe(); | |
| 262 | + $file->start(); | |
| 263 | + } else { | |
| 264 | + $config_data = get_option('iwp_exporter_config_' . $exporter_id, []); | |
| 265 | + $mapper->set_records($config_data['records']); | |
| 266 | + } | |
| 267 | + | |
| 268 | + /** | |
| 269 | + * @var Properties $properties | |
| 270 | + */ | |
| 271 | + $properties = Container::getInstance()->get('properties'); | |
| 272 | + $this->process_chunk($exporter_data, $user, $state, $mapper, $file, $properties); | |
| 273 | + } catch (\Exception $e) { | |
| 274 | + | |
| 275 | + Logger::error('import -error=' . $e->getMessage(), $exporter_id); | |
| 276 | + return $state->error($e)->get_raw(); | |
| 277 | + } | |
| 278 | + | |
| 279 | + | |
| 280 | + return $state->update(function ($data) { | |
| 281 | + $data['duration'] = floatval($data['duration']) + Logger::timer(); | |
| 282 | + return $data; | |
| 283 | + })->get_raw(); | |
| 284 | + } | |
| 285 | + | |
| 286 | + public function process_chunk($exporter_data, $user, $exporter_state, $mapper, $file, $properties) | |
| 287 | + { | |
| 288 | + $id = $exporter_data->getId(); | |
| 289 | + | |
| 290 | + $exporter_state->populate([ | |
| 291 | + 'status' => 'processing' | |
| 292 | + ]); | |
| 293 | + ExporterState::set_state($id, $exporter_state->get_raw()); | |
| 294 | + | |
| 295 | + $time_limit = $properties->get_setting('timeout'); | |
| 296 | + Logger::info('time_limit ' . $time_limit . 's'); | |
| 297 | + | |
| 298 | + $start = microtime(true); | |
| 299 | + $max_record_time = 0; | |
| 300 | + $memory_max_usage = 0; | |
| 301 | + | |
| 302 | + $progress = $exporter_state->get_progress(); | |
| 303 | + $session = $exporter_state->get_session(); | |
| 304 | + $max_total = $progress['end'] - 1; | |
| 305 | + $i = $progress['start'] + $progress['current_row'] - 1; | |
| 306 | + | |
| 307 | + // limit to max 20 rows per chunk | |
| 308 | + $i_max = $i + apply_filters('iwp/exporter/chunk_max_records', 100); | |
| 309 | + | |
| 310 | + while ( | |
| 311 | + $i < $max_total | |
| 312 | + && (!defined('REST_REQUEST') || !REST_REQUEST || $i < $i_max) | |
| 313 | + && ( | |
| 314 | + $time_limit === 0 || $this->has_enough_time($start, $time_limit, $max_record_time) | |
| 315 | + ) | |
| 316 | + && $this->has_enough_memory($memory_max_usage) | |
| 317 | + ) { | |
| 318 | + $i++; | |
| 319 | + | |
| 320 | + $flag = ExporterState::get_flag($id); | |
| 321 | + | |
| 322 | + if (ExporterState::is_paused($flag)) { | |
| 323 | + | |
| 324 | + $exporter_state->populate([ | |
| 325 | + 'status' => 'paused' | |
| 326 | + ]); | |
| 327 | + | |
| 328 | + ExporterState::set_state($id, $exporter_state->get_raw()); | |
| 329 | + Util::write_status_session_to_file($id, $exporter_state); | |
| 330 | + return; | |
| 331 | + } | |
| 332 | + | |
| 333 | + if (ExporterState::is_cancelled($flag)) { | |
| 334 | + $exporter_state->populate([ | |
| 335 | + 'status' => 'cancelled' | |
| 336 | + ]); | |
| 337 | + | |
| 338 | + ExporterState::set_state($id, $exporter_state->get_raw()); | |
| 339 | + Util::write_status_session_to_file($id, $exporter_state); | |
| 340 | + return; | |
| 341 | + } | |
| 342 | + | |
| 343 | + $stats = [ | |
| 344 | + 'rows' => 0, | |
| 345 | + 'skips' => 0, | |
| 346 | + 'errors' => 0, | |
| 347 | + ]; | |
| 348 | + | |
| 349 | + $record_time = microtime(true); | |
| 350 | + | |
| 351 | + try { | |
| 352 | + | |
| 353 | + // TODO: Export row | |
| 354 | + | |
| 355 | + $mapper_data = new MapperData($mapper, $i); | |
| 356 | + if (!$mapper_data->skip()) { | |
| 357 | + $file->add($mapper_data); | |
| 358 | + } | |
| 359 | + | |
| 360 | + Logger::write('export:' . $i . ' -success'); | |
| 361 | + $stats['rows']++; | |
| 362 | + } catch (\Exception $e) { | |
| 363 | + $stats['errors']++; | |
| 364 | + Logger::error('export:' . $i . ' -error=' . $e->getMessage()); | |
| 365 | + } | |
| 366 | + | |
| 367 | + $exporter_state->update_importer_stats($stats); | |
| 368 | + | |
| 369 | + $exporter_state->increment_current_row(); | |
| 370 | + $progress = $exporter_state->get_progress(); | |
| 371 | + | |
| 372 | + ExporterState::set_state($id, $exporter_state->get_raw()); | |
| 373 | + | |
| 374 | + $max_record_time = max($max_record_time, microtime(true) - $record_time); | |
| 375 | + } | |
| 376 | + | |
| 377 | + // TODO: need a new state that will stop the running from happening more than once. | |
| 378 | + // if returning timeout then the cron will stop on older versions | |
| 379 | + if (defined('IWP_PRO_VERSION') && version_compare(IWP_PRO_VERSION, '2.8.0', '>')) { | |
| 380 | + // default status to idle after run | |
| 381 | + $exporter_state->populate([ | |
| 382 | + 'status' => 'timeout' | |
| 383 | + ]); | |
| 384 | + } else { | |
| 385 | + $exporter_state->populate([ | |
| 386 | + 'status' => 'running' | |
| 387 | + ]); | |
| 388 | + } | |
| 389 | + | |
| 390 | + $state_data = $exporter_state->get_raw(); | |
| 391 | + $progress = $exporter_state->get_progress(); | |
| 392 | + | |
| 393 | + if ($progress['end'] - $progress['start'] <= $progress['current_row']) { | |
| 394 | + switch ($exporter_state->get_section()) { | |
| 395 | + case 'export': | |
| 396 | + | |
| 397 | + // complete | |
| 398 | + $file->end(); | |
| 399 | + | |
| 400 | + $key = $this->generate_download_key(); | |
| 401 | + add_post_meta($exporter_data->getId(), '_ewp_file_' . $key, array( | |
| 402 | + 'url' => $file->get_file_url(), | |
| 403 | + 'path' => $file->get_file_path(), | |
| 404 | + 'type' => $exporter_data->getFileType() | |
| 405 | + )); | |
| 406 | + | |
| 407 | + $state_data['section'] = ''; | |
| 408 | + $state_data['status'] = 'complete'; | |
| 409 | + $state_data['file'] = $key; | |
| 410 | + | |
| 411 | + break; | |
| 412 | + } | |
| 413 | + } | |
| 414 | + | |
| 415 | + ExporterState::set_state($id, $state_data); | |
| 416 | + $exporter_state->populate($state_data); | |
| 417 | + } | |
| 418 | + | |
| 419 | + function has_enough_time($start, $time_limit, $max_record_time) | |
| 420 | + { | |
| 421 | + return (microtime(true) - $start) < $time_limit - $max_record_time; | |
| 422 | + } | |
| 423 | + | |
| 424 | + function get_memory_usage() | |
| 425 | + { | |
| 426 | + return memory_get_usage(true); | |
| 427 | + } | |
| 428 | + | |
| 429 | + function has_enough_memory($memory_max_usage) | |
| 430 | + { | |
| 431 | + $limit = $this->get_memory_limit(); | |
| 432 | + | |
| 433 | + // Has unlimited memory | |
| 434 | + if ($limit == '-1') { | |
| 435 | + return true; | |
| 436 | + } | |
| 437 | + | |
| 438 | + $limit *= 0.9; | |
| 439 | + $current_usage = $this->get_memory_usage(); | |
| 440 | + | |
| 441 | + if ($current_usage + $memory_max_usage < $limit) { | |
| 442 | + return true; | |
| 443 | + } | |
| 444 | + | |
| 445 | + Logger::error(sprintf("Not Enough Memory left to use %s, %s/%s", Logger::formatBytes($memory_max_usage, 2), Logger::formatBytes($current_usage, 2), Logger::formatBytes($limit, 2))); | |
| 446 | + | |
| 447 | + return false; | |
| 448 | + } | |
| 449 | + | |
| 450 | + function get_memory_limit($force = false) | |
| 451 | + { | |
| 452 | + if ($force || is_null($this->memory_limit)) { | |
| 453 | + | |
| 454 | + $memory_limit = ini_get('memory_limit'); | |
| 455 | + if (preg_match('/^(\d+)(.)$/', $memory_limit, $matches)) { | |
| 456 | + if ($matches[2] == 'G') { | |
| 457 | + $memory_limit = $matches[1] * 1024 * 1024 * 1024; // nnnM -> nnn MB | |
| 458 | + } elseif ($matches[2] == 'M') { | |
| 459 | + $memory_limit = $matches[1] * 1024 * 1024; // nnnM -> nnn MB | |
| 460 | + } else if ($matches[2] == 'K') { | |
| 461 | + $memory_limit = $matches[1] * 1024; // nnnK -> nnn KB | |
| 462 | + } | |
| 463 | + } | |
| 464 | + | |
| 465 | + $this->memory_limit = $memory_limit; | |
| 466 | + | |
| 467 | + Logger::info('memory_limit ' . $this->memory_limit . ' bytes'); | |
| 468 | + } | |
| 469 | + | |
| 470 | + return $this->memory_limit; | |
| 471 | + } | |
| 472 | + | |
| 473 | + public function export_old($id) | |
| 474 | + { | |
| 475 | + $exporter_data = $this->get_exporter($id); | |
| 118 | 476 | $previous_time = microtime(true); |
| 119 | 477 | |
| 120 | 478 | if ($exporter_data->getFileType() === 'csv') { |
| 121 | 479 | $file = new CSVFile($exporter_data); |
| @@ -192,9 +550,9 @@ | ||
| 192 | 550 | } |
| 193 | 551 | |
| 194 | 552 | $file->end(); |
| 195 | 553 | |
| 196 | - $key = md5(time()); | |
| 554 | + $key = $this->generate_download_key(); | |
| 197 | 555 | add_post_meta($exporter_data->getId(), '_ewp_file_' . $key, array( |
| 198 | 556 | 'url' => $file->get_file_url(), |
| 199 | 557 | 'path' => $file->get_file_path(), |
| 200 | 558 | 'type' => $exporter_data->getFileType() |
| @@ -324,6 +682,34 @@ | ||
| 324 | 682 | } |
| 325 | 683 | } |
| 326 | 684 | |
| 327 | 685 | return $output; |
| 686 | + } | |
| 687 | + | |
| 688 | + public function get_importer_map_fields($exporter_id) | |
| 689 | + { | |
| 690 | + $exporter_data = $this->get_exporter($exporter_id); | |
| 691 | + if (!$exporter_data) { | |
| 692 | + return new \WP_Error("Unable to get exporter"); | |
| 693 | + } | |
| 694 | + | |
| 695 | + $mapper = $this->get_exporter_mapper($exporter_data); | |
| 696 | + $file_type = $exporter_data->getFileType(); | |
| 697 | + | |
| 698 | + $fields = $exporter_data->getFields(true); | |
| 699 | + if (empty($fields)) { | |
| 700 | + | |
| 701 | + $fields = $mapper->get_fields(); | |
| 702 | + | |
| 703 | + switch ($file_type) { | |
| 704 | + case 'csv': | |
| 705 | + $fields = $this->flattenFields($fields); | |
| 706 | + break; | |
| 707 | + default: | |
| 708 | + $fields = $this->nestedFields($fields); | |
| 709 | + break; | |
| 710 | + } | |
| 711 | + } | |
| 712 | + | |
| 713 | + return $fields; | |
| 328 | 714 | } |
| 329 | 715 | } |
