PluginProbe
Import WP – CSV & XML Import Export for WordPress / 2.15.0
Import WP – CSV & XML Import Export for WordPress v2.15.0
2.15.1 2.15.0 2.14.24 2.14.23 2.7.0 2.7.1 2.7.10 2.7.11 2.7.12 2.7.13 2.7.14 2.7.2 2.7.3 2.7.4 2.7.5 2.7.6 2.7.7 2.7.8 2.7.9 2.8.0 2.8.1 2.8.2 2.8.3 2.9.0 2.9.1 All 144 releases
← All changes | class/Common/Importer/File/JSONFile.php +458 -45 2.14.232.15.0 View file →
@@ -5,34 +5,166 @@
5 5 use ImportWP\Common\Importer\FileInterface;
6 6
7 7 class JSONFile extends AbstractIndexedFile implements FileInterface
8 8 {
9 - private $base_path;
9 + /**
10 + * Path segments to the array that holds records.
11 + * Empty means the root value is an array of records.
12 + *
13 + * @var string[]
14 + */
15 + private $base_path_segments = [];
10 16
11 - private $chunk;
17 + /**
18 + * Cache key fragment derived from the record path.
19 + *
20 + * @var string
21 + */
22 + private $record_path_cache_key = 'root';
23 +
24 + /**
25 + * Discovered record paths (path => sample count).
26 + *
27 + * @var array
28 + */
29 + private $path_list = [];
30 +
31 + private $chunk = '';
12 32 private $chunk_size = 8192;
13 33 private $chunk_offset = 0;
14 34
15 35 private $record_counter = 0;
16 36 private $record_start_index = 0;
37 + private $record_offset = 0;
17 38
18 - public function setRecordPath($path)
39 + // Stream parser state — must persist across chunk reads
40 + private $parse_segment_index = 0;
41 + private $parse_awaiting_value = false;
42 + private $parse_depth = 0;
43 + private $parse_search_depth = 0;
44 + private $parse_path_matched = false;
45 + private $parse_tracking = false;
46 + private $parse_record_depth = 0;
47 + private $parse_skip_depth = 0;
48 +
49 + /**
50 + * Set base path for records.
51 + *
52 + * Examples: "data", "results/items", "/" or "" for a root array.
53 + *
54 + * @param string $path
55 + */
56 + public function setRecordPath($path = '/')
19 57 {
20 - $this->base_path = "\"{$path}\"";
58 + if ($path === null) {
59 + $path = '/';
60 + }
61 +
62 + $path = trim((string) $path);
63 + if (strpos($path, '/') === 0) {
64 + $path = substr($path, 1);
65 + }
66 + $path = rtrim($path, '/');
67 +
68 + $segments = $path === '' ? [] : explode('/', $path);
69 + $segments = array_values(array_filter($segments, function ($value) {
70 + return $value !== '';
71 + }));
72 +
73 + $new_key = empty($segments) ? 'root' : implode('_', $segments);
74 + if ($new_key !== $this->record_path_cache_key) {
75 + $this->resetIndexState();
76 + }
77 +
78 + $this->base_path_segments = $segments;
79 + $this->record_path_cache_key = $new_key;
21 80 }
22 81
82 + public function getFileIndexKey()
83 + {
84 + return sprintf('file_index-%s', $this->record_path_cache_key);
85 + }
86 +
23 87 /**
24 - * Generate record file positions
88 + * Discover candidate record paths (arrays of objects).
25 89 *
26 - * Loop through each record and save each position
90 + * @return array path => count
27 91 */
92 + public function get_path_list()
93 + {
94 + $this->path_list = $this->config ? $this->config->get('json_path_list') : null;
95 + if (!is_array($this->path_list)) {
96 + $this->path_list = [];
97 + }
98 +
99 + if (!empty($this->path_list)) {
100 + return $this->path_list;
101 + }
102 +
103 + $decoded = $this->decodeFileSample();
104 + if (!is_array($decoded)) {
105 + return [];
106 + }
107 +
108 + $this->path_list = [];
109 + $this->collectPaths($decoded, '', $this->path_list);
110 + if ($this->config) {
111 + $this->config->set('json_path_list', $this->path_list);
112 + }
113 +
114 + return $this->path_list;
115 + }
116 +
117 + /**
118 + * Return records at the current record path by decoding JSON.
119 + * Used for preview so we are not dependent on a stream index.
120 + *
121 + * @param int $limit
122 + * @return array
123 + */
124 + public function getDecodedRecords($limit = 1)
125 + {
126 + $decoded = $this->decodeFileSample();
127 + if (!is_array($decoded)) {
128 + return [];
129 + }
130 +
131 + $records = $this->resolveRecordsAtPath($decoded);
132 + if (!is_array($records) || !$this->isList($records)) {
133 + return [];
134 + }
135 +
136 + if ($limit > 0) {
137 + return array_slice($records, 0, $limit);
138 + }
139 +
140 + return $records;
141 + }
142 +
143 + /**
144 + * Generate record file positions.
145 + */
28 146 public function generateIndex()
29 147 {
30 - $this->record_counter = 0;
148 + $this->record_counter = 0;
31 149 $this->record_start_index = 0;
150 + $this->chunk = '';
151 + $this->chunk_offset = 0;
32 152
153 + $this->parse_segment_index = 0;
154 + $this->parse_awaiting_value = false;
155 + $this->parse_depth = 0;
156 + $this->parse_search_depth = 0;
157 + $this->parse_path_matched = empty($this->base_path_segments);
158 + $this->parse_tracking = false;
159 + $this->parse_record_depth = 0;
160 + $this->parse_skip_depth = 0;
161 +
33 162 rewind($this->getFileHandle());
34 163 while (!feof($this->getFileHandle())) {
164 + if ($this->is_processing && $this->chunk_offset > $this->process_max_size) {
165 + break;
166 + }
35 167
36 168 $this->chunk .= $this->getChunk();
37 169 $this->processChunk();
38 170 }
@@ -38,9 +170,9 @@
38 170 }
39 171 }
40 172
41 173 /**
42 - * Read chunk from file
174 + * Read chunk from file.
43 175 *
44 176 * @return bool|string
45 177 */
46 178 public function getChunk()
@@ -49,13 +181,11 @@
49 181 }
50 182
51 183 public function processChunk()
52 184 {
53 -
54 185 $regex_parts = [
55 - '"[^"\\\\]*(?:\\\\.[^"\\\\]*)*"', // skip escaped \"
56 - "'[^'\\\\]*(?:\\\\.[^'\\\\]*)*'", // skip escaped \'
57 - '([^,"\'{}\[\]:\s]+)', // values not in quotes or double quotes
186 + '"[^"\\\\]*(?:\\\\.[^"\\\\]*)*"',
187 + '([^,"\'{}\[\]:\s]+)',
58 188 '({)',
59 189 '(})',
60 190 '(\[)',
61 191 '(\])',
@@ -62,39 +192,43 @@
62 192 '(:)',
63 193 '(,)',
64 194 ];
65 195
66 - $open_tags = ['{', '['];
67 - $close_tags = ['}', ']'];
68 -
69 196 $regex = '/' . implode('|', $regex_parts) . '/s';
70 - $track_depth = false;
71 - $track_depth_init = false;
72 - $depth = 0;
73 - $depth_to_capture = 1;
74 197
75 198 while (preg_match($regex, $this->chunk, $matches, PREG_OFFSET_CAPTURE) !== 0) {
76 199 list($captured, $offset) = $matches[0];
77 -
78 200 $this->record_offset = $offset;
79 - $depth_changed = false;
80 201
81 - if (true === $track_depth) {
202 + $is_string = strlen($captured) >= 2 && $captured[0] === '"';
203 + $is_open_object = $captured === '{';
204 + $is_close_object = $captured === '}';
205 + $is_open_array = $captured === '[';
206 + $is_close_array = $captured === ']';
207 + $is_colon = $captured === ':';
208 + $is_structure = $is_open_object || $is_close_object || $is_open_array || $is_close_array;
82 209
83 - if (in_array($captured, $open_tags)) {
84 - $depth++;
85 - $depth_changed = true;
86 - } elseif (in_array($captured, $close_tags)) {
87 - $depth--;
88 - $depth_changed = true;
210 + // Skipping an unmatched value subtree
211 + if ($this->parse_skip_depth > 0) {
212 + if ($is_open_object || $is_open_array) {
213 + $this->parse_skip_depth++;
214 + $this->parse_depth++;
215 + } elseif ($is_close_object || $is_close_array) {
216 + $this->parse_skip_depth--;
217 + $this->parse_depth--;
89 218 }
219 + $this->consumeToken($offset, $captured);
220 + continue;
221 + }
90 222
91 - if ($depth_changed) {
92 - if ($depth === 0) {
93 - $this->record_start_index = $this->chunk_offset + $this->record_offset;
94 - } elseif ($depth < -1) {
95 - $track_depth = false;
96 - } elseif ($depth === -1) {
223 + if ($this->parse_tracking) {
224 + if ($is_open_object || $is_open_array) {
225 + $this->parse_record_depth++;
226 + $this->parse_depth++;
227 + } elseif ($is_close_object || $is_close_array) {
228 + $this->parse_record_depth--;
229 + $this->parse_depth--;
230 + if ($this->parse_record_depth === 0 && $is_close_object) {
97 231 $this->setIndex(
98 232 $this->record_counter,
99 233 $this->record_start_index,
100 234 $this->chunk_offset + $this->record_offset + strlen($captured)
@@ -99,27 +233,306 @@
99 233 $this->record_start_index,
100 234 $this->chunk_offset + $this->record_offset + strlen($captured)
101 235 );
102 236 $this->record_counter++;
237 + $this->parse_tracking = false;
238 + } elseif ($this->parse_record_depth < 0) {
239 + $this->parse_tracking = false;
240 + $this->parse_path_matched = false;
241 + $this->parse_awaiting_value = false;
242 + $this->parse_segment_index = 0;
243 + $this->parse_search_depth = 0;
103 244 }
104 245 }
105 - } elseif ($track_depth_init && $depth_to_capture >= 0) {
106 - if (in_array($captured, $open_tags)) {
107 - $depth_to_capture--;
108 - if ($depth_to_capture < 0) {
109 - $track_depth = true;
110 - $this->record_start_index = $this->chunk_offset + $this->record_offset;
246 + $this->consumeToken($offset, $captured);
247 + continue;
248 + }
249 +
250 + if ($this->parse_path_matched) {
251 + if ($is_open_object) {
252 + $this->parse_tracking = true;
253 + $this->parse_record_depth = 1;
254 + $this->parse_depth++;
255 + $this->record_start_index = $this->chunk_offset + $this->record_offset;
256 + } elseif ($is_close_array || $is_close_object) {
257 + $this->parse_path_matched = false;
258 + $this->parse_awaiting_value = false;
259 + $this->parse_segment_index = 0;
260 + $this->parse_search_depth = 0;
261 + $this->parse_depth--;
262 + } elseif ($is_open_array) {
263 + $this->parse_depth++;
264 + }
265 + $this->consumeToken($offset, $captured);
266 + continue;
267 + }
268 +
269 + if ($this->parse_awaiting_value) {
270 + if ($is_colon) {
271 + $this->consumeToken($offset, $captured);
272 + continue;
273 + }
274 +
275 + $is_last_segment = $this->parse_segment_index >= count($this->base_path_segments);
276 +
277 + if ($is_last_segment && $is_open_array) {
278 + $this->parse_path_matched = true;
279 + $this->parse_awaiting_value = false;
280 + $this->parse_depth++;
281 + } elseif (!$is_last_segment && $is_open_object) {
282 + $this->parse_awaiting_value = false;
283 + $this->parse_depth++;
284 + $this->parse_search_depth = $this->parse_depth;
285 + } elseif ($is_open_object || $is_open_array) {
286 + $this->parse_awaiting_value = false;
287 + $this->parse_segment_index = 0;
288 + $this->parse_search_depth = 0;
289 + $this->parse_skip_depth = 1;
290 + $this->parse_depth++;
291 + } else {
292 + $this->parse_awaiting_value = false;
293 + $this->parse_segment_index = 0;
294 + $this->parse_search_depth = 0;
295 + }
296 +
297 + $this->consumeToken($offset, $captured);
298 + continue;
299 + }
300 +
301 + if (empty($this->base_path_segments)) {
302 + if ($is_open_array && $this->parse_depth === 0) {
303 + $this->parse_path_matched = true;
304 + $this->parse_depth++;
305 + } elseif ($is_structure) {
306 + if ($is_open_object || $is_open_array) {
307 + $this->parse_depth++;
308 + } else {
309 + $this->parse_depth--;
111 310 }
112 311 }
312 + $this->consumeToken($offset, $captured);
313 + continue;
113 314 }
114 315
316 + // Enter root object so we can match top-level path keys
317 + if ($is_open_object && $this->parse_depth === 0 && $this->parse_segment_index === 0 && !$this->parse_awaiting_value) {
318 + $this->parse_depth++;
319 + $this->parse_search_depth = 1;
320 + $this->consumeToken($offset, $captured);
321 + continue;
322 + }
115 323
116 - if ($captured === $this->base_path) {
117 - $track_depth_init = true;
324 + // Looking for the next path key at search_depth
325 + if ($is_string && $this->parse_depth === $this->parse_search_depth) {
326 + $key = $this->unquote($captured);
327 + if (isset($this->base_path_segments[$this->parse_segment_index]) && $key === $this->base_path_segments[$this->parse_segment_index]) {
328 + $this->parse_segment_index++;
329 + $this->parse_awaiting_value = true;
330 + }
331 + } elseif ($is_structure) {
332 + if ($is_open_object || $is_open_array) {
333 + $this->parse_depth++;
334 + } else {
335 + $this->parse_depth--;
336 + if ($this->parse_depth < $this->parse_search_depth) {
337 + $this->parse_segment_index = min($this->parse_segment_index, $this->segmentsReachedAtDepth($this->parse_depth));
338 + $this->parse_search_depth = max(0, $this->parse_depth);
339 + }
340 + }
118 341 }
119 342
120 - $string_offset = $offset + strlen($captured);
121 - $this->chunk_offset += $string_offset;
122 - $this->chunk = substr($this->chunk, $string_offset);
343 + $this->consumeToken($offset, $captured);
123 344 }
345 + }
346 +
347 + /**
348 + * @param int $depth
349 + * @return int
350 + */
351 + private function segmentsReachedAtDepth($depth)
352 + {
353 + return max(0, min($depth, count($this->base_path_segments)));
354 + }
355 +
356 + /**
357 + * @param int $offset
358 + * @param string $captured
359 + */
360 + private function consumeToken($offset, $captured)
361 + {
362 + $string_offset = $offset + strlen($captured);
363 + $this->chunk_offset += $string_offset;
364 + $this->chunk = substr($this->chunk, $string_offset);
365 + }
366 +
367 + /**
368 + * @param string $token
369 + * @return string
370 + */
371 + private function unquote($token)
372 + {
373 + if (strlen($token) >= 2 && $token[0] === '"' && substr($token, -1) === '"') {
374 + return stripcslashes(substr($token, 1, -1));
375 + }
376 +
377 + return $token;
378 + }
379 +
380 + /**
381 + * @return array|null
382 + */
383 + private function decodeFileSample()
384 + {
385 + $handle = $this->getFileHandle();
386 + $current = ftell($handle);
387 + rewind($handle);
388 +
389 + $sample = '';
390 + $max = max($this->process_max_size, 1000000);
391 + while (!feof($handle) && strlen($sample) < $max) {
392 + $sample .= fread($handle, $this->chunk_size);
393 + }
394 +
395 + $decoded = json_decode($sample, true);
396 + if (json_last_error() !== JSON_ERROR_NONE) {
397 + $decoded = $this->decodePartialSample($sample);
398 + }
399 +
400 + fseek($handle, $current);
401 +
402 + return is_array($decoded) ? $decoded : null;
403 + }
404 +
405 + /**
406 + * Walk to the record array for the current base path.
407 + *
408 + * @param array $decoded
409 + * @return array|null
410 + */
411 + private function resolveRecordsAtPath(array $decoded)
412 + {
413 + if (empty($this->base_path_segments)) {
414 + return $decoded;
415 + }
416 +
417 + $current = $decoded;
418 + foreach ($this->base_path_segments as $segment) {
419 + if (!is_array($current) || !array_key_exists($segment, $current)) {
420 + return null;
421 + }
422 + $current = $current[$segment];
423 + }
424 +
425 + return $current;
426 + }
427 +
428 + /**
429 + * Attempt to decode a truncated JSON sample by closing open braces/brackets.
430 + *
431 + * @param string $sample
432 + * @return array|null
433 + */
434 + private function decodePartialSample($sample)
435 + {
436 + $in_string = false;
437 + $escape = false;
438 + $stack = [];
439 +
440 + $len = strlen($sample);
441 + for ($i = 0; $i < $len; $i++) {
442 + $ch = $sample[$i];
443 + if ($in_string) {
444 + if ($escape) {
445 + $escape = false;
446 + } elseif ($ch === '\\') {
447 + $escape = true;
448 + } elseif ($ch === '"') {
449 + $in_string = false;
450 + }
451 + continue;
452 + }
453 +
454 + if ($ch === '"') {
455 + $in_string = true;
456 + } elseif ($ch === '{' || $ch === '[') {
457 + $stack[] = $ch;
458 + } elseif ($ch === '}' || $ch === ']') {
459 + array_pop($stack);
460 + }
461 + }
462 +
463 + if ($in_string) {
464 + $sample .= '"';
465 + }
466 +
467 + $sample = rtrim($sample);
468 + $sample = preg_replace('/[,:]\s*$/', '', $sample);
469 +
470 + while (!empty($stack)) {
471 + $open = array_pop($stack);
472 + $sample .= $open === '{' ? '}' : ']';
473 + }
474 +
475 + $decoded = json_decode($sample, true);
476 + return json_last_error() === JSON_ERROR_NONE ? $decoded : null;
477 + }
478 +
479 + /**
480 + * @param mixed $data
481 + * @param string $prefix
482 + * @param array $paths
483 + */
484 + private function collectPaths($data, $prefix, &$paths)
485 + {
486 + if (!is_array($data)) {
487 + return;
488 + }
489 +
490 + if ($this->isList($data)) {
491 + if ($this->isListOfObjects($data)) {
492 + $key = $prefix === '' ? '/' : $prefix;
493 + $paths[$key] = count($data);
494 + }
495 + return;
496 + }
497 +
498 + foreach ($data as $key => $value) {
499 + if (!is_array($value)) {
500 + continue;
501 + }
502 + $path = $prefix === '' ? (string) $key : $prefix . '/' . $key;
503 + $this->collectPaths($value, $path, $paths);
504 + }
505 + }
506 +
507 + /**
508 + * @param array $data
509 + * @return bool
510 + */
511 + private function isList(array $data)
512 + {
513 + if ($data === []) {
514 + return true;
515 + }
516 +
517 + return array_keys($data) === range(0, count($data) - 1);
518 + }
519 +
520 + /**
521 + * @param array $data
522 + * @return bool
523 + */
524 + private function isListOfObjects(array $data)
525 + {
526 + if (empty($data) || !$this->isList($data)) {
527 + return false;
528 + }
529 +
530 + foreach ($data as $item) {
531 + if (!is_array($item) || $this->isList($item)) {
532 + return false;
533 + }
534 + }
535 +
536 + return true;
124 537 }
125 538 }