← All changes | class/Common/Exporter/ExporterManager.php +292 -52 2.9.0 → 2.15.0 View file →

		@@ -1,9 +1,8 @@
1	1	<?php
2	2
3	3	namespace ImportWP\Common\Exporter;
4	4
5		-use Exception;
6	5	use ImportWP\Common\Exporter\File\CSVFile;
7	6	use ImportWP\Common\Exporter\File\JSONFile;
8	7	use ImportWP\Common\Exporter\File\XMLFile;
9	8	use ImportWP\Common\Exporter\Mapper\CommentMapper;
		@@ -12,14 +11,19 @@
12	11	use ImportWP\Common\Exporter\Mapper\TaxMapper;
13	12	use ImportWP\Common\Exporter\Mapper\UserMapper;
14	13	use ImportWP\Common\Exporter\State\ExporterState;
15	14	use ImportWP\Common\Model\ExporterModel;
16		-use ImportWP\Common\Runner\ExporterRunner;
17	15	use ImportWP\Common\Util\Logger;
	16	+use ImportWP\Common\Util\Util;
18	17	use ImportWP\Container;
19	18
20	19	class ExporterManager
21	20	{
	21	+ /**
	22	+ * @var int
	23	+ */
	24	+ protected $memory_limit;
	25	+
22	26	public function __construct()
23	27	{
24	28	add_action('admin_init', [$this, 'download_file']);
25	29	}
		@@ -35,9 +39,21 @@
35	39	if (!isset($_GET['page'], $_GET['exporter'], $_GET['download']) || $_GET['page'] !== 'importwp') {
36	40	return;
37	41	}
38	42
	43	+ if (!is_user_logged_in() || !current_user_can('manage_options')) {
	44	+ wp_die(esc_html__('You do not have permission to download this file.', 'jc-importer'), '', array('response' => 403));
	45	+ }
	46	+
	47	+ if (!isset($_GET['_wpnonce']) || !wp_verify_nonce(sanitize_key(wp_unslash($_GET['_wpnonce'])), 'iwp_export_download')) {
	48	+ wp_die(esc_html__('Invalid download request.', 'jc-importer'), '', array('response' => 403));
	49	+ }
	50	+
39	51	$exporter_data = $this->get_exporter(intval($_GET['exporter']));
	52	+ if (!$exporter_data) {
	53	+ wp_die(esc_html__('Invalid download request.', 'jc-importer'), '', array('response' => 403));
	54	+ }
	55	+
40	56	$file = sanitize_key($_GET['download']);
41	57
42	58	$download_data = get_post_meta($exporter_data->getId(), '_ewp_file_' . $file, true);
43	59	if ($download_data) {
		@@ -48,11 +64,21 @@
48	64	header('Content-type: "text/' . $download_data['type'] . '"; charset="utf8"');
49	65	readfile($download_data['path']);
50	66	die();
51	67	}
	68	+
	69	+ wp_die(esc_html__('Invalid download request.', 'jc-importer'), '', array('response' => 403));
52	70	}
53	71
54	72	/**
	73	+ * @return string
	74	+ */
	75	+ protected function generate_download_key()
	76	+ {
	77	+ return bin2hex(random_bytes(16));
	78	+ }
	79	+
	80	+ /**
55	81	* Get Exporters
56	82	*
57	83	* @return ExporterModel[]
58	84	*/
		@@ -116,8 +142,53 @@
116	142	$exporter = $this->get_exporter($id);
117	143	$exporter->delete();
118	144	}
119	145
	146	+ /**
	147	+ * @param \ImportWP\Common\Model\ExporterModel $exporter_data
	148	+ * @return \ImportWP\Common\Exporter\File\File
	149	+ */
	150	+ public function get_exporter_file($exporter_data)
	151	+ {
	152	+
	153	+ if ($exporter_data->getFileType() === 'csv') {
	154	+ $file = new CSVFile($exporter_data);
	155	+ } elseif ($exporter_data->getFileType() === 'xml') {
	156	+ $file = new XMLFile($exporter_data);
	157	+ } else {
	158	+ $file = new JSONFile($exporter_data);
	159	+ }
	160	+
	161	+ return $file;
	162	+ }
	163	+
	164	+ /**
	165	+ * @param \ImportWP\Common\Model\ExporterModel $exporter_data
	166	+ * @return \ImportWP\Common\Exporter\Mapper\MapperInterface
	167	+ */
	168	+ public function get_exporter_mapper($exporter_data)
	169	+ {
	170	+ $type = $exporter_data->getType();
	171	+ $matches = null;
	172	+ if (preg_match('/^ewp_tax_(.*?)$/', $type, $matches) == 1) {
	173	+ $taxonomy = $matches[1];
	174	+ $mapper = new TaxMapper($taxonomy);
	175	+ } elseif (preg_match('/^ewp_comment_(.*?)$/', $type, $matches) == 1) {
	176	+ $post_type = $matches[1];
	177	+ $mapper = new CommentMapper($post_type);
	178	+ } elseif ($type === 'user') {
	179	+ $mapper = new UserMapper();
	180	+ } else {
	181	+
	182	+ $mapper = apply_filters('iwp/exporter/load_mapper', false, $type);
	183	+ if (!$mapper) {
	184	+ $mapper = new PostMapper($type);
	185	+ }
	186	+ }
	187	+
	188	+ return $mapper;
	189	+ }
	190	+
120	191	public function export($id, $user = null, $session = null)
121	192	{
122	193	Logger::timer();
123	194
		@@ -127,9 +198,9 @@
127	198
128	199	$exporter_data = $this->get_exporter($id);
129	200	$exporter_id = $exporter_data->getId();
130	201
131		- $config_data = []; // get_site_option('iwp_exporter_config_' . $exporter_id, []);
	202	+ $config_data = []; // get_option('iwp_exporter_config_' . $exporter_id, []);
132	203
133	204	$state = new ExporterState($exporter_id, $user);
134	205
135	206	try {
		@@ -136,34 +207,11 @@
136	207
137	208
138	209	$state->init($session);
139	210
140		- if ($exporter_data->getFileType() === 'csv') {
141		- $file = new CSVFile($exporter_data);
142		- } elseif ($exporter_data->getFileType() === 'xml') {
143		- $file = new XMLFile($exporter_data);
144		- } else {
145		- $file = new JSONFile($exporter_data);
146		- }
	211	+ $file = $this->get_exporter_file($exporter_data);
	212	+ $mapper = $this->get_exporter_mapper($exporter_data);
147	213
148		- $type = $exporter_data->getType();
149		- $matches = null;
150		- if (preg_match('/^ewp_tax_(.*?)$/', $type, $matches) == 1) {
151		- $taxonomy = $matches[1];
152		- $mapper = new TaxMapper($taxonomy);
153		- } elseif (preg_match('/^ewp_comment_(.*?)$/', $type, $matches) == 1) {
154		- $post_type = $matches[1];
155		- $mapper = new CommentMapper($post_type);
156		- } elseif ($type === 'user') {
157		- $mapper = new UserMapper();
158		- } else {
159		-
160		- $mapper = apply_filters('iwp/exporter/load_mapper', false, $type);
161		- if (!$mapper) {
162		- $mapper = new PostMapper($type);
163		- }
164		- }
165		-
166	214	/**
167	215	* @var MapperInterface $mapper
168	216	*/
169	217	$mapper->set_filters($exporter_data->getFilters());
		@@ -190,9 +238,9 @@
190	238
191	239	if ($state->has_status('init')) {
192	240
193	241	if (!$mapper->have_records($exporter_id)) {
194		- throw new \Exception(("No records found to export"));
	242	+ throw new \Exception(__("No records found to export", 'jc-importer'));
195	243	}
196	244
197	245	// write
198	246	$config_data['id'] = $state->get_session();
		@@ -199,9 +247,9 @@
199	247	$config_data['version'] = 1;
200	248	$config_data['records'] = $mapper->get_records();
201	249
202	250	// save
203		- update_site_option('iwp_exporter_config_' . $exporter_id, $config_data);
	251	+ update_option('iwp_exporter_config_' . $exporter_id, $config_data);
204	252
205	253	$state->update(function ($state) use ($config_data) {
206	254	$state['id'] = $config_data['id'];
207	255	$state['status'] = 'running';
		@@ -212,9 +260,9 @@
212	260
213	261	$file->wipe();
214	262	$file->start();
215	263	} else {
216		- $config_data = get_site_option('iwp_exporter_config_' . $exporter_id, []);
	264	+ $config_data = get_option('iwp_exporter_config_' . $exporter_id, []);
217	265	$mapper->set_records($config_data['records']);
218	266	}
219	267
220	268	/**
		@@ -220,45 +268,209 @@
220	268	/**
221	269	* @var Properties $properties
222	270	*/
223	271	$properties = Container::getInstance()->get('properties');
224		- $runner = new ExporterRunner($properties, $mapper, $file);
225		- $runner->process($exporter_id, $user, $state);
	272	+ $this->process_chunk($exporter_data, $user, $state, $mapper, $file, $properties);
	273	+ } catch (\Exception $e) {
226	274
227		- // Lock the file to only write 1 end of file
228		- if ($state->has_status('end')) {
229		- $state->update(function ($state) use ($exporter_data, $file) {
	275	+ Logger::error('import -error=' . $e->getMessage(), $exporter_id);
	276	+ return $state->error($e)->get_raw();
	277	+ }
230	278
231		- if ($state['status'] !== 'end') {
232		- return $state;
233		- }
234	279
	280	+ return $state->update(function ($data) {
	281	+ $data['duration'] = floatval($data['duration']) + Logger::timer();
	282	+ return $data;
	283	+ })->get_raw();
	284	+ }
	285	+
	286	+ public function process_chunk($exporter_data, $user, $exporter_state, $mapper, $file, $properties)
	287	+ {
	288	+ $id = $exporter_data->getId();
	289	+
	290	+ $exporter_state->populate([
	291	+ 'status' => 'processing'
	292	+ ]);
	293	+ ExporterState::set_state($id, $exporter_state->get_raw());
	294	+
	295	+ $time_limit = $properties->get_setting('timeout');
	296	+ Logger::info('time_limit ' . $time_limit . 's');
	297	+
	298	+ $start = microtime(true);
	299	+ $max_record_time = 0;
	300	+ $memory_max_usage = 0;
	301	+
	302	+ $progress = $exporter_state->get_progress();
	303	+ $session = $exporter_state->get_session();
	304	+ $max_total = $progress['end'] - 1;
	305	+ $i = $progress['start'] + $progress['current_row'] - 1;
	306	+
	307	+ // limit to max 20 rows per chunk
	308	+ $i_max = $i + apply_filters('iwp/exporter/chunk_max_records', 100);
	309	+
	310	+ while (
	311	+ $i < $max_total
	312	+ && (!defined('REST_REQUEST') || !REST_REQUEST || $i < $i_max)
	313	+ && (
	314	+ $time_limit === 0 || $this->has_enough_time($start, $time_limit, $max_record_time)
	315	+ )
	316	+ && $this->has_enough_memory($memory_max_usage)
	317	+ ) {
	318	+ $i++;
	319	+
	320	+ $flag = ExporterState::get_flag($id);
	321	+
	322	+ if (ExporterState::is_paused($flag)) {
	323	+
	324	+ $exporter_state->populate([
	325	+ 'status' => 'paused'
	326	+ ]);
	327	+
	328	+ ExporterState::set_state($id, $exporter_state->get_raw());
	329	+ Util::write_status_session_to_file($id, $exporter_state);
	330	+ return;
	331	+ }
	332	+
	333	+ if (ExporterState::is_cancelled($flag)) {
	334	+ $exporter_state->populate([
	335	+ 'status' => 'cancelled'
	336	+ ]);
	337	+
	338	+ ExporterState::set_state($id, $exporter_state->get_raw());
	339	+ Util::write_status_session_to_file($id, $exporter_state);
	340	+ return;
	341	+ }
	342	+
	343	+ $stats = [
	344	+ 'rows' => 0,
	345	+ 'skips' => 0,
	346	+ 'errors' => 0,
	347	+ ];
	348	+
	349	+ $record_time = microtime(true);
	350	+
	351	+ try {
	352	+
	353	+ // TODO: Export row
	354	+
	355	+ $mapper_data = new MapperData($mapper, $i);
	356	+ if (!$mapper_data->skip()) {
	357	+ $file->add($mapper_data);
	358	+ }
	359	+
	360	+ Logger::write('export:' . $i . ' -success');
	361	+ $stats['rows']++;
	362	+ } catch (\Exception $e) {
	363	+ $stats['errors']++;
	364	+ Logger::error('export:' . $i . ' -error=' . $e->getMessage());
	365	+ }
	366	+
	367	+ $exporter_state->update_importer_stats($stats);
	368	+
	369	+ $exporter_state->increment_current_row();
	370	+ $progress = $exporter_state->get_progress();
	371	+
	372	+ ExporterState::set_state($id, $exporter_state->get_raw());
	373	+
	374	+ $max_record_time = max($max_record_time, microtime(true) - $record_time);
	375	+ }
	376	+
	377	+ // TODO: need a new state that will stop the running from happening more than once.
	378	+ // if returning timeout then the cron will stop on older versions
	379	+ if (defined('IWP_PRO_VERSION') && version_compare(IWP_PRO_VERSION, '2.8.0', '>')) {
	380	+ // default status to idle after run
	381	+ $exporter_state->populate([
	382	+ 'status' => 'timeout'
	383	+ ]);
	384	+ } else {
	385	+ $exporter_state->populate([
	386	+ 'status' => 'running'
	387	+ ]);
	388	+ }
	389	+
	390	+ $state_data = $exporter_state->get_raw();
	391	+ $progress = $exporter_state->get_progress();
	392	+
	393	+ if ($progress['end'] - $progress['start'] <= $progress['current_row']) {
	394	+ switch ($exporter_state->get_section()) {
	395	+ case 'export':
	396	+
	397	+ // complete
235	398	$file->end();
236	399
237		- $key = md5(time());
	400	+ $key = $this->generate_download_key();
238	401	add_post_meta($exporter_data->getId(), '_ewp_file_' . $key, array(
239	402	'url' => $file->get_file_url(),
240	403	'path' => $file->get_file_path(),
241	404	'type' => $exporter_data->getFileType()
242	405	));
243		- $state['file'] = $key;
244		- $state['status'] = 'complete';
245		- return $state;
246		- });
	406	+
	407	+ $state_data['section'] = '';
	408	+ $state_data['status'] = 'complete';
	409	+ $state_data['file'] = $key;
	410	+
	411	+ break;
247	412	}
248		- } catch (\Exception $e) {
	413	+ }
249	414
250		- Logger::error('import -error=' . $e->getMessage(), $exporter_id);
251		- return $state->error($e)->get_raw();
	415	+ ExporterState::set_state($id, $state_data);
	416	+ $exporter_state->populate($state_data);
	417	+ }
	418	+
	419	+ function has_enough_time($start, $time_limit, $max_record_time)
	420	+ {
	421	+ return (microtime(true) - $start) < $time_limit - $max_record_time;
	422	+ }
	423	+
	424	+ function get_memory_usage()
	425	+ {
	426	+ return memory_get_usage(true);
	427	+ }
	428	+
	429	+ function has_enough_memory($memory_max_usage)
	430	+ {
	431	+ $limit = $this->get_memory_limit();
	432	+
	433	+ // Has unlimited memory
	434	+ if ($limit == '-1') {
	435	+ return true;
252	436	}
253	437
	438	+ $limit *= 0.9;
	439	+ $current_usage = $this->get_memory_usage();
254	440
255		- return $state->update(function ($data) {
256		- $data['duration'] = floatval($data['duration']) + Logger::timer();
257		- return $data;
258		- })->get_raw();
	441	+ if ($current_usage + $memory_max_usage < $limit) {
	442	+ return true;
	443	+ }
	444	+
	445	+ Logger::error(sprintf("Not Enough Memory left to use %s, %s/%s", Logger::formatBytes($memory_max_usage, 2), Logger::formatBytes($current_usage, 2), Logger::formatBytes($limit, 2)));
	446	+
	447	+ return false;
259	448	}
260	449
	450	+ function get_memory_limit($force = false)
	451	+ {
	452	+ if ($force || is_null($this->memory_limit)) {
	453	+
	454	+ $memory_limit = ini_get('memory_limit');
	455	+ if (preg_match('/^(\d+)(.)$/', $memory_limit, $matches)) {
	456	+ if ($matches[2] == 'G') {
	457	+ $memory_limit = $matches[1] * 1024 * 1024 * 1024; // nnnM -> nnn MB
	458	+ } elseif ($matches[2] == 'M') {
	459	+ $memory_limit = $matches[1] * 1024 * 1024; // nnnM -> nnn MB
	460	+ } else if ($matches[2] == 'K') {
	461	+ $memory_limit = $matches[1] * 1024; // nnnK -> nnn KB
	462	+ }
	463	+ }
	464	+
	465	+ $this->memory_limit = $memory_limit;
	466	+
	467	+ Logger::info('memory_limit ' . $this->memory_limit . ' bytes');
	468	+ }
	469	+
	470	+ return $this->memory_limit;
	471	+ }
	472	+
261	473	public function export_old($id)
262	474	{
263	475	$exporter_data = $this->get_exporter($id);
264	476	$previous_time = microtime(true);
		@@ -338,9 +550,9 @@
338	550	}
339	551
340	552	$file->end();
341	553
342		- $key = md5(time());
	554	+ $key = $this->generate_download_key();
343	555	add_post_meta($exporter_data->getId(), '_ewp_file_' . $key, array(
344	556	'url' => $file->get_file_url(),
345	557	'path' => $file->get_file_path(),
346	558	'type' => $exporter_data->getFileType()
		@@ -470,6 +682,34 @@
470	682	}
471	683	}
472	684
473	685	return $output;
	686	+ }
	687	+
	688	+ public function get_importer_map_fields($exporter_id)
	689	+ {
	690	+ $exporter_data = $this->get_exporter($exporter_id);
	691	+ if (!$exporter_data) {
	692	+ return new \WP_Error("Unable to get exporter");
	693	+ }
	694	+
	695	+ $mapper = $this->get_exporter_mapper($exporter_data);
	696	+ $file_type = $exporter_data->getFileType();
	697	+
	698	+ $fields = $exporter_data->getFields(true);
	699	+ if (empty($fields)) {
	700	+
	701	+ $fields = $mapper->get_fields();
	702	+
	703	+ switch ($file_type) {
	704	+ case 'csv':
	705	+ $fields = $this->flattenFields($fields);
	706	+ break;
	707	+ default:
	708	+ $fields = $this->nestedFields($fields);
	709	+ break;
	710	+ }
	711	+ }
	712	+
	713	+ return $fields;
474	714	}
475	715	}