PluginProbe
Import WP – CSV & XML Import Export for WordPress / 2.15.0
Import WP – CSV & XML Import Export for WordPress v2.15.0
2.15.1 2.15.0 2.14.24 2.14.23 2.7.0 2.7.1 2.7.10 2.7.11 2.7.12 2.7.13 2.7.14 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7 2.7.8 2.7.9 2.8.0 2.8.1 2.8.2 2.8.3 2.9.0 2.9.1 All 144 releases
jc-importer / class / Common / Exporter / ExporterManager.php

ExporterManager.php in Import WP – CSV & XML Import Export for WordPress 2.15.0, at class/Common/Exporter/ExporterManager.php

716 lines 21.3 KB
No matching file
Up and down to move Enter to open Esc to close
Raw Download Zip
1 <?php
2
3 namespace ImportWP\Common\Exporter;
4
5 use ImportWP\Common\Exporter\File\CSVFile;
6 use ImportWP\Common\Exporter\File\JSONFile;
7 use ImportWP\Common\Exporter\File\XMLFile;
8 use ImportWP\Common\Exporter\Mapper\CommentMapper;
9 use ImportWP\Common\Exporter\Mapper\MapperData;
10 use ImportWP\Common\Exporter\Mapper\PostMapper;
11 use ImportWP\Common\Exporter\Mapper\TaxMapper;
12 use ImportWP\Common\Exporter\Mapper\UserMapper;
13 use ImportWP\Common\Exporter\State\ExporterState;
14 use ImportWP\Common\Model\ExporterModel;
15 use ImportWP\Common\Util\Logger;
16 use ImportWP\Common\Util\Util;
17 use ImportWP\Container;
18
19 class ExporterManager
20 {
21 /**
22 * @var int
23 */
24 protected $memory_limit;
25
26 public function __construct()
27 {
28 add_action('admin_init', [$this, 'download_file']);
29 }
30
31 /**
32 * Download exporter file directly
33 *
34 * @return void
35 */
36 public function download_file()
37 {
38
39 if (!isset($_GET['page'], $_GET['exporter'], $_GET['download']) || $_GET['page'] !== 'importwp') {
40 return;
41 }
42
43 if (!is_user_logged_in() || !current_user_can('manage_options')) {
44 wp_die(esc_html__('You do not have permission to download this file.', 'jc-importer'), '', array('response' => 403));
45 }
46
47 if (!isset($_GET['_wpnonce']) || !wp_verify_nonce(sanitize_key(wp_unslash($_GET['_wpnonce'])), 'iwp_export_download')) {
48 wp_die(esc_html__('Invalid download request.', 'jc-importer'), '', array('response' => 403));
49 }
50
51 $exporter_data = $this->get_exporter(intval($_GET['exporter']));
52 if (!$exporter_data) {
53 wp_die(esc_html__('Invalid download request.', 'jc-importer'), '', array('response' => 403));
54 }
55
56 $file = sanitize_key($_GET['download']);
57
58 $download_data = get_post_meta($exporter_data->getId(), '_ewp_file_' . $file, true);
59 if ($download_data) {
60
61 delete_post_meta($exporter_data->getId(), '_ewp_file_' . $file, $download_data);
62
63 header('Content-disposition: attachment; filename="' . basename($download_data['path']) . '"');
64 header('Content-type: "text/' . $download_data['type'] . '"; charset="utf8"');
65 readfile($download_data['path']);
66 die();
67 }
68
69 wp_die(esc_html__('Invalid download request.', 'jc-importer'), '', array('response' => 403));
70 }
71
72 /**
73 * @return string
74 */
75 protected function generate_download_key()
76 {
77 return bin2hex(random_bytes(16));
78 }
79
80 /**
81 * Get Exporters
82 *
83 * @return ExporterModel[]
84 */
85 public function get_exporters()
86 {
87
88 $result = array();
89 $query = new \WP_Query(array(
90 'post_type' => EWP_POST_TYPE,
91 'posts_per_page' => -1,
92 ));
93
94 foreach ($query->posts as $post) {
95 $result[] = $this->get_exporter($post);
96 }
97 return $result;
98 }
99
100 /**
101 * Get Exporter
102 *
103 * @param int $id
104 * @return ExporterModel
105 */
106 public function get_exporter($id)
107 {
108 if ($id instanceof ExporterModel) {
109 return $id;
110 }
111
112 if (EWP_POST_TYPE !== get_post_type($id)) {
113 return false;
114 }
115
116 return new ExporterModel($id, $this->is_debug());
117 }
118
119 public function is_debug()
120 {
121
122 if (defined('IWP_DEBUG') && true === IWP_DEBUG) {
123 return true;
124 }
125
126 $uninstall_enabled = get_option('iwp_settings');
127 if (isset($uninstall_enabled['debug']) && true === $uninstall_enabled['debug']) {
128 return true;
129 }
130
131 return false;
132 }
133
134 /**
135 * Delete Importer
136 *
137 * @param int $id
138 * @return void
139 */
140 public function delete_exporter($id)
141 {
142 $exporter = $this->get_exporter($id);
143 $exporter->delete();
144 }
145
146 /**
147 * @param \ImportWP\Common\Model\ExporterModel $exporter_data
148 * @return \ImportWP\Common\Exporter\File\File
149 */
150 public function get_exporter_file($exporter_data)
151 {
152
153 if ($exporter_data->getFileType() === 'csv') {
154 $file = new CSVFile($exporter_data);
155 } elseif ($exporter_data->getFileType() === 'xml') {
156 $file = new XMLFile($exporter_data);
157 } else {
158 $file = new JSONFile($exporter_data);
159 }
160
161 return $file;
162 }
163
164 /**
165 * @param \ImportWP\Common\Model\ExporterModel $exporter_data
166 * @return \ImportWP\Common\Exporter\Mapper\MapperInterface
167 */
168 public function get_exporter_mapper($exporter_data)
169 {
170 $type = $exporter_data->getType();
171 $matches = null;
172 if (preg_match('/^ewp_tax_(.*?)$/', $type, $matches) == 1) {
173 $taxonomy = $matches[1];
174 $mapper = new TaxMapper($taxonomy);
175 } elseif (preg_match('/^ewp_comment_(.*?)$/', $type, $matches) == 1) {
176 $post_type = $matches[1];
177 $mapper = new CommentMapper($post_type);
178 } elseif ($type === 'user') {
179 $mapper = new UserMapper();
180 } else {
181
182 $mapper = apply_filters('iwp/exporter/load_mapper', false, $type);
183 if (!$mapper) {
184 $mapper = new PostMapper($type);
185 }
186 }
187
188 return $mapper;
189 }
190
191 public function export($id, $user = null, $session = null)
192 {
193 Logger::timer();
194
195 if (is_null($user)) {
196 $user = uniqid('iwp', true);
197 }
198
199 $exporter_data = $this->get_exporter($id);
200 $exporter_id = $exporter_data->getId();
201
202 $config_data = []; // get_option('iwp_exporter_config_' . $exporter_id, []);
203
204 $state = new ExporterState($exporter_id, $user);
205
206 try {
207
208
209 $state->init($session);
210
211 $file = $this->get_exporter_file($exporter_data);
212 $mapper = $this->get_exporter_mapper($exporter_data);
213
214 /**
215 * @var MapperInterface $mapper
216 */
217 $mapper->set_filters($exporter_data->getFilters());
218
219 // Default to showing all fields, if none selected
220 if (empty($exporter_data->getFields(true))) {
221
222 $fields = $mapper->get_fields();
223
224 switch ($exporter_data->getFileType()) {
225 case 'csv':
226 $fields = $this->flattenFields($fields);
227 break;
228 default:
229 $fields = $this->nestedFields($fields);
230 break;
231 }
232
233
234 $exporter_data->setFields($fields);
235 }
236
237
238
239 if ($state->has_status('init')) {
240
241 if (!$mapper->have_records($exporter_id)) {
242 throw new \Exception(__("No records found to export", 'jc-importer'));
243 }
244
245 // write
246 $config_data['id'] = $state->get_session();
247 $config_data['version'] = 1;
248 $config_data['records'] = $mapper->get_records();
249
250 // save
251 update_option('iwp_exporter_config_' . $exporter_id, $config_data);
252
253 $state->update(function ($state) use ($config_data) {
254 $state['id'] = $config_data['id'];
255 $state['status'] = 'running';
256 $state['progress']['export']['start'] = 0;
257 $state['progress']['export']['end'] = count($config_data['records']);
258 return $state;
259 });
260
261 $file->wipe();
262 $file->start();
263 } else {
264 $config_data = get_option('iwp_exporter_config_' . $exporter_id, []);
265 $mapper->set_records($config_data['records']);
266 }
267
268 /**
269 * @var Properties $properties
270 */
271 $properties = Container::getInstance()->get('properties');
272 $this->process_chunk($exporter_data, $user, $state, $mapper, $file, $properties);
273 } catch (\Exception $e) {
274
275 Logger::error('import -error=' . $e->getMessage(), $exporter_id);
276 return $state->error($e)->get_raw();
277 }
278
279
280 return $state->update(function ($data) {
281 $data['duration'] = floatval($data['duration']) + Logger::timer();
282 return $data;
283 })->get_raw();
284 }
285
286 public function process_chunk($exporter_data, $user, $exporter_state, $mapper, $file, $properties)
287 {
288 $id = $exporter_data->getId();
289
290 $exporter_state->populate([
291 'status' => 'processing'
292 ]);
293 ExporterState::set_state($id, $exporter_state->get_raw());
294
295 $time_limit = $properties->get_setting('timeout');
296 Logger::info('time_limit ' . $time_limit . 's');
297
298 $start = microtime(true);
299 $max_record_time = 0;
300 $memory_max_usage = 0;
301
302 $progress = $exporter_state->get_progress();
303 $session = $exporter_state->get_session();
304 $max_total = $progress['end'] - 1;
305 $i = $progress['start'] + $progress['current_row'] - 1;
306
307 // limit to max 20 rows per chunk
308 $i_max = $i + apply_filters('iwp/exporter/chunk_max_records', 100);
309
310 while (
311 $i < $max_total
312 && (!defined('REST_REQUEST') || !REST_REQUEST || $i < $i_max)
313 && (
314 $time_limit === 0 || $this->has_enough_time($start, $time_limit, $max_record_time)
315 )
316 && $this->has_enough_memory($memory_max_usage)
317 ) {
318 $i++;
319
320 $flag = ExporterState::get_flag($id);
321
322 if (ExporterState::is_paused($flag)) {
323
324 $exporter_state->populate([
325 'status' => 'paused'
326 ]);
327
328 ExporterState::set_state($id, $exporter_state->get_raw());
329 Util::write_status_session_to_file($id, $exporter_state);
330 return;
331 }
332
333 if (ExporterState::is_cancelled($flag)) {
334 $exporter_state->populate([
335 'status' => 'cancelled'
336 ]);
337
338 ExporterState::set_state($id, $exporter_state->get_raw());
339 Util::write_status_session_to_file($id, $exporter_state);
340 return;
341 }
342
343 $stats = [
344 'rows' => 0,
345 'skips' => 0,
346 'errors' => 0,
347 ];
348
349 $record_time = microtime(true);
350
351 try {
352
353 // TODO: Export row
354
355 $mapper_data = new MapperData($mapper, $i);
356 if (!$mapper_data->skip()) {
357 $file->add($mapper_data);
358 }
359
360 Logger::write('export:' . $i . ' -success');
361 $stats['rows']++;
362 } catch (\Exception $e) {
363 $stats['errors']++;
364 Logger::error('export:' . $i . ' -error=' . $e->getMessage());
365 }
366
367 $exporter_state->update_importer_stats($stats);
368
369 $exporter_state->increment_current_row();
370 $progress = $exporter_state->get_progress();
371
372 ExporterState::set_state($id, $exporter_state->get_raw());
373
374 $max_record_time = max($max_record_time, microtime(true) - $record_time);
375 }
376
377 // TODO: need a new state that will stop the running from happening more than once.
378 // if returning timeout then the cron will stop on older versions
379 if (defined('IWP_PRO_VERSION') && version_compare(IWP_PRO_VERSION, '2.8.0', '>')) {
380 // default status to idle after run
381 $exporter_state->populate([
382 'status' => 'timeout'
383 ]);
384 } else {
385 $exporter_state->populate([
386 'status' => 'running'
387 ]);
388 }
389
390 $state_data = $exporter_state->get_raw();
391 $progress = $exporter_state->get_progress();
392
393 if ($progress['end'] - $progress['start'] <= $progress['current_row']) {
394 switch ($exporter_state->get_section()) {
395 case 'export':
396
397 // complete
398 $file->end();
399
400 $key = $this->generate_download_key();
401 add_post_meta($exporter_data->getId(), '_ewp_file_' . $key, array(
402 'url' => $file->get_file_url(),
403 'path' => $file->get_file_path(),
404 'type' => $exporter_data->getFileType()
405 ));
406
407 $state_data['section'] = '';
408 $state_data['status'] = 'complete';
409 $state_data['file'] = $key;
410
411 break;
412 }
413 }
414
415 ExporterState::set_state($id, $state_data);
416 $exporter_state->populate($state_data);
417 }
418
419 function has_enough_time($start, $time_limit, $max_record_time)
420 {
421 return (microtime(true) - $start) < $time_limit - $max_record_time;
422 }
423
424 function get_memory_usage()
425 {
426 return memory_get_usage(true);
427 }
428
429 function has_enough_memory($memory_max_usage)
430 {
431 $limit = $this->get_memory_limit();
432
433 // Has unlimited memory
434 if ($limit == '-1') {
435 return true;
436 }
437
438 $limit *= 0.9;
439 $current_usage = $this->get_memory_usage();
440
441 if ($current_usage + $memory_max_usage < $limit) {
442 return true;
443 }
444
445 Logger::error(sprintf("Not Enough Memory left to use %s, %s/%s", Logger::formatBytes($memory_max_usage, 2), Logger::formatBytes($current_usage, 2), Logger::formatBytes($limit, 2)));
446
447 return false;
448 }
449
450 function get_memory_limit($force = false)
451 {
452 if ($force || is_null($this->memory_limit)) {
453
454 $memory_limit = ini_get('memory_limit');
455 if (preg_match('/^(\d+)(.)$/', $memory_limit, $matches)) {
456 if ($matches[2] == 'G') {
457 $memory_limit = $matches[1] * 1024 * 1024 * 1024; // nnnM -> nnn MB
458 } elseif ($matches[2] == 'M') {
459 $memory_limit = $matches[1] * 1024 * 1024; // nnnM -> nnn MB
460 } else if ($matches[2] == 'K') {
461 $memory_limit = $matches[1] * 1024; // nnnK -> nnn KB
462 }
463 }
464
465 $this->memory_limit = $memory_limit;
466
467 Logger::info('memory_limit ' . $this->memory_limit . ' bytes');
468 }
469
470 return $this->memory_limit;
471 }
472
473 public function export_old($id)
474 {
475 $exporter_data = $this->get_exporter($id);
476 $previous_time = microtime(true);
477
478 if ($exporter_data->getFileType() === 'csv') {
479 $file = new CSVFile($exporter_data);
480 } elseif ($exporter_data->getFileType() === 'xml') {
481 $file = new XMLFile($exporter_data);
482 } else {
483 $file = new JSONFile($exporter_data);
484 }
485
486 $type = $exporter_data->getType();
487 $matches = null;
488 if (preg_match('/^ewp_tax_(.*?)$/', $type, $matches) == 1) {
489 $taxonomy = $matches[1];
490 $mapper = new TaxMapper($taxonomy);
491 } elseif (preg_match('/^ewp_comment_(.*?)$/', $type, $matches) == 1) {
492 $post_type = $matches[1];
493 $mapper = new CommentMapper($post_type);
494 } elseif ($type === 'user') {
495 $mapper = new UserMapper();
496 } else {
497
498 $mapper = apply_filters('iwp/exporter/load_mapper', false, $type);
499 if (!$mapper) {
500 $mapper = new PostMapper($type);
501 }
502 }
503
504 $mapper->set_filters($exporter_data->getFilters());
505
506 // Default to showing all fields, if none selected
507 if (empty($exporter_data->getFields(true))) {
508
509 $fields = $mapper->get_fields();
510
511 switch ($exporter_data->getFileType()) {
512 case 'csv':
513 $fields = $this->flattenFields($fields);
514 break;
515 default:
516 $fields = $this->nestedFields($fields);
517 break;
518 }
519
520
521 $exporter_data->setFields($fields);
522 }
523
524 $file->start();
525
526 $columns = $exporter_data->getFields();
527
528 $total = 0;
529 $i = 0;
530
531 if ($mapper->have_records($exporter_data->getId())) {
532
533 $total = $mapper->found_records();
534
535 for ($i = 0; $i < $total; $i++) {
536
537 $mapper_data = new MapperData($mapper, $i);
538 if (!$mapper_data->skip()) {
539 $file->add($mapper_data);
540 }
541
542 $current_time = microtime(true);
543 $delta_time = $current_time - $previous_time;
544
545 if ($delta_time > 0.1) {
546 $exporter_data->set_status('running', $i, $total);
547 $previous_time = $current_time;
548 }
549 }
550 }
551
552 $file->end();
553
554 $key = $this->generate_download_key();
555 add_post_meta($exporter_data->getId(), '_ewp_file_' . $key, array(
556 'url' => $file->get_file_url(),
557 'path' => $file->get_file_path(),
558 'type' => $exporter_data->getFileType()
559 ));
560
561 $status = $exporter_data->set_status('complete', $i, $total);
562 $status['file'] = $key;
563 return $status;
564 // echo json_encode($status) . "\n";
565
566 // flush();
567 // ob_flush();
568 // die();
569 }
570
571 public function flattenFields($fields, $prefix = '')
572 {
573 $output = [];
574
575 $output = array_merge($output, array_reduce($fields['fields'], function ($carry, $item) use ($prefix) {
576
577 $carry[] = [
578 'parent' => 0,
579 'id' => count($carry) + 1,
580 'selection' => $prefix . $item,
581 'loop' => false,
582 'custom' => false
583 ];
584
585 return $carry;
586 }, []));
587
588 if (!empty($fields['children'])) {
589 foreach ($fields['children'] as $child) {
590 $output = array_merge($output, $this->flattenFields($child, $child['key'] . '.'));
591 }
592 }
593
594 return $output;
595 }
596
597 private $nested_field_counter = 0;
598
599 public function nestedFields($fields, $prefix = '', $parent = 0)
600 {
601 $output = [];
602
603 if ($parent == 0) {
604
605 $this->nested_field_counter = 1;
606 $output[] = [
607 'parent' => 0,
608 'id' => 1,
609 'selection' => '',
610 'label' => 'records',
611 'loop' => false,
612 ];
613
614 $this->nested_field_counter++;
615 $output[] = [
616 'parent' => 1,
617 'id' => 2,
618 'selection' => 'main',
619 'label' => 'record',
620 'loop' => true,
621 ];
622
623 $parent = 2;
624 } elseif ($fields['loop'] == true) {
625 $this->nested_field_counter++;
626 $output[] = [
627 'parent' => $parent,
628 'id' => $this->nested_field_counter,
629 'selection' => '',
630 'label' => $fields['key'] . '_wrapper',
631 'loop' => false,
632 ];
633 $parent = $this->nested_field_counter;
634
635 $this->nested_field_counter++;
636 $output[] = [
637 'parent' => $parent,
638 'id' => $this->nested_field_counter,
639 'selection' => $fields['key'],
640 'label' => $fields['key'],
641 'loop' => true,
642 ];
643
644 $parent = $this->nested_field_counter;
645 }
646
647 if (isset($fields['loop_fields']) && !empty($fields['loop_fields'])) {
648
649 $output = array_merge($output, array_reduce($fields['loop_fields'], function ($carry, $item) use ($parent) {
650
651 $this->nested_field_counter++;
652 $carry[] = [
653 'parent' => $parent,
654 'id' => $this->nested_field_counter,
655 'selection' => $item,
656 'loop' => false,
657 'custom' => false
658 ];
659
660 return $carry;
661 }, []));
662 } else {
663
664 $output = array_merge($output, array_reduce($fields['fields'], function ($carry, $item) use ($prefix, $parent) {
665
666 $this->nested_field_counter++;
667 $carry[] = [
668 'parent' => $parent,
669 'id' => $this->nested_field_counter,
670 'selection' => $prefix . $item,
671 'loop' => false,
672 'custom' => false
673 ];
674
675 return $carry;
676 }, []));
677 }
678
679 if (!empty($fields['children'])) {
680 foreach ($fields['children'] as $child) {
681 $output = array_merge($output, $this->nestedFields($child, $child['key'] . '.', $parent));
682 }
683 }
684
685 return $output;
686 }
687
688 public function get_importer_map_fields($exporter_id)
689 {
690 $exporter_data = $this->get_exporter($exporter_id);
691 if (!$exporter_data) {
692 return new \WP_Error("Unable to get exporter");
693 }
694
695 $mapper = $this->get_exporter_mapper($exporter_data);
696 $file_type = $exporter_data->getFileType();
697
698 $fields = $exporter_data->getFields(true);
699 if (empty($fields)) {
700
701 $fields = $mapper->get_fields();
702
703 switch ($file_type) {
704 case 'csv':
705 $fields = $this->flattenFields($fields);
706 break;
707 default:
708 $fields = $this->nestedFields($fields);
709 break;
710 }
711 }
712
713 return $fields;
714 }
715 }
716