PluginProbe
WP Synchro – The Ultimate WordPress Migration Tool / trunk
WP Synchro – The Ultimate WordPress Migration Tool vtrunk
1.16.1 1.16.0 trunk 1.0.0 1.0.1 1.0.2 1.0.3 1.0.4 1.0.5 1.1.0 1.10.0 1.11.0 1.11.1 1.11.2 1.11.3 1.11.4 1.11.5 1.12.0 1.13.0 1.14.0 1.15.0 1.2.0 1.3.0 1.3.1 1.3.2 All 45 releases
wpsynchro / src / Database / DatabaseFinalize.php

DatabaseFinalize.php in WP Synchro – The Ultimate WordPress Migration Tool trunk, at src/Database/DatabaseFinalize.php

498 lines 21.7 KB
No matching file
Up and down to move Enter to open Esc to close
Raw Download Zip
1 <?php
2
3 /**
4 * Class for handling database finalize
5 */
6
7 namespace WPSynchro\Database;
8
9 use WPSynchro\Masterdata\MasterdataSync;
10 use WPSynchro\Migration\MigrationController;
11 use WPSynchro\Transport\Destination;
12 use WPSynchro\Utilities\SyncTimerList;
13
14 class DatabaseFinalize
15 {
16 // Data objects
17 public $job = null;
18 public $migration = null;
19 public $databasesync = null;
20
21 // Dependencies
22 public $logger = null;
23 public $timer = null;
24
25 /**
26 * Constructor
27 */
28 public function __construct()
29 {
30 $this->logger = MigrationController::getInstance()->getLogger();
31 $this->timer = SyncTimerList::getInstance();
32 }
33
34 /**
35 * Calculate completion percent
36 */
37 public function finalize()
38 {
39 $sync_controller = MigrationController::getInstance();
40 $this->job = $sync_controller->job;
41 $this->migration = $sync_controller->migration;
42
43 $this->databasesync = new DatabaseSync();
44 $this->databasesync->job = $this->job;
45 $this->databasesync->migration = $this->migration;
46
47 $this->logger->log("INFO", "Starting database finalize with remaining time: " . $this->timer->getRemainingSyncTime());
48
49 // Prepare SQL statements, if not done yet
50 if (!$this->job->finalize_db_initialized) {
51 $this->job->finalize_progress_description = __("Preparing database finalize", "wpsynchro");
52 $this->logger->log("INFO", "Prepare SQL queries for database finalize");
53 $this->prepareSQLQueries();
54 $this->job->finalize_db_initialized = true;
55 $this->logger->log("INFO", "Done preparing SQL queries for database finalize");
56 $this->job->request_full_timeframe = true;
57 return;
58 }
59
60 // Execute a group of queries
61 if (count($this->job->finalize_db_sql_queries) > 0) {
62 $this->job->finalize_progress_description = sprintf(
63 __("Finalizing table %d out of %d", "wpsynchro"),
64 $this->job->finalize_db_sql_queries_count - count($this->job->finalize_db_sql_queries),
65 $this->job->finalize_db_sql_queries_count
66 );
67
68 $sql_group = array_pop($this->job->finalize_db_sql_queries);
69
70 // Execute a set of queries
71 $body = new \stdClass();
72 $body->sql_inserts = $sql_group;
73 $body->type = 'finalize'; // For executing sql
74
75 $this->logger->log("DEBUG", "Calling remote client db service with " . count($body->sql_inserts) . " SQL statements:", $sql_group);
76 $this->databasesync->callRemoteClientDBService($body, 'to');
77
78 $this->job->request_full_timeframe = true;
79 return;
80 }
81
82 // After all tables is renamed, we remove all the extra temporary tables on the target (only leftovers from other syncs)
83 if (count($this->job->errors) == 0 && !$this->job->finalize_db_excess_tables_initialized) {
84 $this->job->finalize_db_excess_table_queries = $this->cleanUpAfterFinalizing();
85 $this->job->finalize_db_excess_table_queries_count = count($this->job->finalize_db_excess_table_queries);
86 $this->job->finalize_db_excess_tables_initialized = true;
87 $this->job->request_full_timeframe = true;
88 return;
89 }
90
91 // If any excess SQL queries to clean up excess tables, run them
92 if (count($this->job->errors) == 0 && count($this->job->finalize_db_excess_table_queries) > 0) {
93 $this->job->finalize_progress_description = sprintf(
94 __("Removing old temporary table - %d out of %d", "wpsynchro"),
95 $this->job->finalize_db_excess_table_queries_count - count($this->job->finalize_db_excess_table_queries),
96 $this->job->finalize_db_excess_table_queries_count
97 );
98
99 $sql = array_pop($this->job->finalize_db_excess_table_queries);
100
101 // Execute a clean up query
102 $body = new \stdClass();
103 $body->sql_inserts = [$sql];
104 $body->type = 'finalize'; // For executing sql
105
106 $this->logger->log("DEBUG", "Calling remote client db service with for excess table cleanup - SQL statement:", $body->sql_inserts);
107 $this->databasesync->callRemoteClientDBService($body, 'to');
108
109 $this->job->request_full_timeframe = true;
110 return;
111 }
112
113 // Check for table case issues on the migration
114 if (count($this->job->errors) == 0) {
115 $this->job->finalize_progress_description = __("Check that all tables on target is in correct case", "wpsynchro");
116 $this->checkTableCasesCorrect($this->job->finalize_db_table_to_expect_on_target);
117 }
118
119 if (count($this->job->errors) > 0) {
120 // Errors during finalize
121 return;
122 } else {
123 // All good
124 $this->job->finalize_db_completed = true;
125 }
126 }
127
128 /**
129 * Prepare the list of sql queries to run for finalize
130 */
131 public function prepareSQLQueries()
132 {
133 // Handle preserving data
134 $sql_queries = [];
135 $sql_queries_last = [];
136
137 // Handle data to keep
138 $sql_queries[] = $this->handleDataToKeep();
139
140 // Get latest and greatest from target db
141 $dbtables = $this->retrieveDatabaseTables();
142
143 // Create lookup array
144 $to_table_lookup = [];
145 foreach ($dbtables as $to_table) {
146 $to_table_lookup[$to_table->name] = $to_table->rows;
147 }
148
149 // Run finalize checks
150 foreach ($this->job->from_dbmasterdata as $from_table) {
151 $from_rows = $from_table->rows;
152 // If its old temp table on source, just ignore
153 if (strpos($from_table->name, DatabaseSync::TMP_TABLE_PREFIX) > -1) {
154 $this->logger->log("DEBUG", "Table " . $from_table->name . " is a old temp table, so ignore");
155 continue;
156 }
157
158 // Check if table exists on "to", which it should
159 if (!isset($to_table_lookup[$from_table->temp_name])) {
160 // Not transferred - Error
161 $this->logger->log("CRITICAL", "Table " . $from_table->name . " does not exist on target, but it should. It is not transferred. Temp name is " . $from_table->temp_name);
162 $this->job->errors[] = sprintf(__("Finalize: Error in database migration for table %s - It is not transferred", "wpsynchro"), $from_table->name);
163 continue;
164 }
165
166 $to_rows = $to_table_lookup[$from_table->temp_name];
167 $this->checkRowCountCompare($from_table->name, $from_rows, $to_rows);
168 }
169
170 // Get tables to be renamed
171 foreach ($this->job->from_dbmasterdata as $table) {
172 if (!isset($from_table->temp_name) || strlen($from_table->temp_name) == 0) {
173 continue;
174 }
175
176 $table_name = $table->name;
177 $table_temp_name = $table->temp_name;
178
179 $sql_queries_in_group = [];
180
181 // If table prefix change is enabled
182 if ($this->migration->db_table_prefix_change) {
183 // Check if we need to change prefixes and therefore need to rewrite table name
184 $table_name = DatabaseHelperFunctions::handleTablePrefixChange($table_name, $this->job->from_wpdb_prefix, $this->job->to_wpdb_prefix);
185
186 // Handle the data updates in table when doing prefix change
187 $prefix_change_sql_queries = $this->handleDataChangeOnPrefixChange($table_name, $table_temp_name);
188 $sql_queries_in_group = array_merge($sql_queries_in_group, $prefix_change_sql_queries);
189 }
190
191 // Add tables to the list for "expected to be on target"
192 $this->job->finalize_db_table_to_expect_on_target[] = $table_name;
193
194 // Add sql statements
195 $this->logger->log("DEBUG", "Add drop table in database on " . $table_name . " and rename from " . $table_temp_name);
196 $sql_queries_in_group[] = 'DROP TABLE IF EXISTS `' . $table_name . '`';
197 $sql_queries_in_group[] = 'RENAME TABLE `' . $table_temp_name . '` TO `' . $table_name . '`';
198
199 // Check if it is special table
200 if ($table_name == $this->job->to_wp_users_table) {
201 $sql_queries_last[0] = $sql_queries_in_group;
202 } elseif ($table_name == $this->job->to_wp_usermeta_table) {
203 $sql_queries_last[1] = $sql_queries_in_group;
204 } elseif ($table_name == $this->job->to_wp_options_table) {
205 $sql_queries_last[2] = $sql_queries_in_group;
206 } else {
207 $sql_queries[] = $sql_queries_in_group;
208 }
209 }
210
211 // Handle multisite
212 $sql_queries = array_merge($sql_queries, $this->getMultisiteFinalizeSQL());
213
214 // Add the last queries
215 ksort($sql_queries_last);
216 foreach ($sql_queries_last as $query) {
217 $sql_queries[] = $query;
218 }
219
220 // Add views
221 foreach ($this->job->db_views_to_be_synced as $table) {
222 // Add sql statements
223 $this->logger->log("DEBUG", "Add drop view in database on " . $table->name . " and create it again");
224 $view_sql = [
225 'DROP VIEW IF EXISTS `' . $table->name . '`',
226 $table->create_table
227 ];
228 $sql_queries[] = $view_sql;
229 }
230
231 // Turn it around, so we can pop of from top
232 $sql_queries = array_reverse($sql_queries);
233
234 // Log sql queries
235 $this->logger->log("DEBUG", "Finalize SQL queries:", $sql_queries);
236 $this->job->finalize_db_sql_queries = $sql_queries;
237 $this->job->finalize_db_sql_queries_count = count($sql_queries);
238 }
239
240 /**
241 * Handle the data to keep (such as WP Synchro data etc.)
242 */
243 public function handleDataToKeep()
244 {
245 // Figure out if we actually migrate the options table
246 $target_options_table_tempname = "";
247 $sql_queries = [];
248 foreach ($this->job->from_dbmasterdata as $table) {
249 if ($table->name == $this->job->from_wp_options_table) {
250 $target_options_table_tempname = $table->temp_name;
251 break;
252 }
253 }
254 if ($target_options_table_tempname == "") {
255 return $sql_queries;
256 }
257
258
259 // Preserving data in options table, if it is migrated
260 if ($this->migration->include_all_database_tables || in_array($this->job->from_wp_options_table, $this->migration->only_include_database_table_names)) {
261 global $wpdb;
262
263 $delete_from_sql = "DELETE FROM `" . $target_options_table_tempname . "` WHERE option_name LIKE '" . $wpdb->esc_like("wpsynchro_") . "%'";
264 $insert_into_sql = "INSERT INTO `" . $target_options_table_tempname . "` (option_name,option_value,autoload) SELECT option_name,option_value,autoload FROM " . $this->job->to_wp_options_table . " where option_name like 'wpsynchro_%'";
265
266 $sql_queries[] = $delete_from_sql;
267 $this->logger->log("INFO", "Add sql statement to delete WP Synchro options: " . $delete_from_sql);
268 $sql_queries[] = $insert_into_sql;
269 $this->logger->log("INFO", "Add sql statement to copy current WP Synchro options to temp table: " . $insert_into_sql);
270
271 // Extract all the keys we want to preserve in options table
272 $preserve_options_keys = $this->migration->db_preserve_options_table_keys;
273 $custom_options_keys = explode(',', $this->migration->db_preserve_options_custom);
274 $preserve_options_keys = array_merge($preserve_options_keys, $custom_options_keys);
275 $preserve_options_keys = array_map('trim', $preserve_options_keys);
276
277 // Get the SQL for the preserve and add it to our list of SQL's to do
278 foreach ($preserve_options_keys as $preserve_key) {
279 $preserve_sql = $this->getPreserveOptionsFieldSQL($target_options_table_tempname, $preserve_key);
280 $sql_queries = array_merge($sql_queries, $preserve_sql);
281 }
282 }
283
284 return $sql_queries;
285 }
286
287 /**
288 * Add preserve data from options table SQL
289 */
290 public function getPreserveOptionsFieldSQL($target_options_table_tempname, $options_key)
291 {
292 $delete_from_sql = "DELETE FROM `" . $target_options_table_tempname . "` WHERE option_name = '{$options_key}'";
293 $insert_into_sql = "INSERT INTO `" . $target_options_table_tempname . "` (option_name,option_value,autoload) SELECT option_name,option_value,autoload FROM " . $this->job->to_wp_options_table . " WHERE option_name = '{$options_key}'";
294
295 $sql = [];
296 $sql[] = $delete_from_sql;
297 $this->logger->log("INFO", "Add sql statement to delete options field (db preserve): " . $delete_from_sql);
298 $sql[] = $insert_into_sql;
299 $this->logger->log("INFO", "Add sql statement to copy existing value from options table (db preserve): " . $insert_into_sql);
300
301 return $sql;
302 }
303
304 /**
305 * Handle data to be renamed inside tables when changing prefix
306 */
307 public function handleDataChangeOnPrefixChange($table_name, $table_temp_name)
308 {
309
310 $source_prefix = $this->job->from_wpdb_prefix;
311 $target_prefix = $this->job->to_wpdb_prefix;
312 $sql_queries = [];
313 global $wpdb;
314
315 if ($source_prefix != $target_prefix) {
316 // Add sql queries to change meta data if options table or user_meta table
317 $temp_wp_usermeta = $this->job->to_wpdb_prefix . "usermeta";
318 if ($table_name == $this->job->to_wp_usermeta_table || $table_name == $temp_wp_usermeta) {
319 // Update prefixes in usermeta table
320 $sql_queries[] = "DELETE FROM `" . $table_temp_name . "` WHERE meta_key LIKE '" . $wpdb->esc_like($target_prefix) . "%'";
321 $sql_queries[] = "UPDATE `" . $table_temp_name . "` SET meta_key = REPLACE(meta_key, '" . $source_prefix . "', '" . $target_prefix . "') WHERE meta_key LIKE '" . $wpdb->esc_like($source_prefix) . "%'";
322 $this->logger->log("DEBUG", "update data in temp table " . $table_temp_name . " (" . $table_name . ") to replace source prefix " . $source_prefix . " with target prefix " . $target_prefix);
323 } elseif ($table_name == $this->job->to_wp_options_table) {
324 // Update prefix in options table
325 $sql_queries[] = "DELETE FROM `" . $table_temp_name . "` WHERE option_name LIKE '" . $wpdb->esc_like($target_prefix) . "%'";
326 $sql_queries[] = "UPDATE `" . $table_temp_name . "` SET option_name = REPLACE(option_name, '" . $source_prefix . "', '" . $target_prefix . "') WHERE option_name LIKE '" . $wpdb->esc_like($source_prefix) . "%'";
327 $this->logger->log("DEBUG", "update data in temp table " . $table_temp_name . " (" . $table_name . ") to replace source prefix " . $source_prefix . " with target prefix " . $target_prefix);
328 }
329 }
330 return $sql_queries;
331 }
332
333 /**
334 * Retrieve new database data from target
335 */
336 public function retrieveDatabaseTables($temp_table = true)
337 {
338 // Retrieve new db tables list from destination
339 $masterdata_obj = new MasterdataSync();
340 $data_to_retrieve = ["dbdetails"];
341 $masterdata_obj->migration = $this->migration;
342 $masterdata_obj->job = $this->job;
343 $masterdata_obj->logger = $this->logger;
344 $this->logger->log("DEBUG", "Retrieving new masterdata from target");
345 $masterdata = $masterdata_obj->retrieveMasterdata(new Destination(Destination::TARGET), $data_to_retrieve);
346
347 if (!is_object($masterdata) || !isset($masterdata->tmptables_dbdetails)) {
348 $this->job->errors[] = __("Could not retrieve data from remote site for finalizing", "wpsynchro");
349 $this->logger->log("CRITICAL", "Could not retrieve data from target site for finalizing");
350 return;
351 }
352 $this->logger->log("DEBUG", "Retrieving new masterdata completed");
353
354 if ($temp_table) {
355 return $masterdata->tmptables_dbdetails;
356 } else {
357 return $masterdata->dbdetails;
358 }
359 }
360
361 /**
362 * Try to clean up if any temporary tables are left on target
363 */
364 public function cleanUpAfterFinalizing()
365 {
366 $temp_tables_left = $this->retrieveDatabaseTables();
367 $sql_queries = [];
368 foreach ($temp_tables_left as $table) {
369 $sql_queries[] = 'DROP TABLE IF EXISTS `' . $table->name . '`';
370 $this->logger->log("DEBUG", "Add sql to delete excess temp table: " . $table->name);
371 }
372
373 if (count($sql_queries) == 0) {
374 $this->logger->log("DEBUG", "No excess temp tables to delete");
375 }
376 return $sql_queries;
377 }
378
379 /**
380 * Check that tables have correct case
381 */
382 public function checkTableCasesCorrect($tables_to_be_expected_on_target)
383 {
384 $tables_on_target = $this->retrieveDatabaseTables(false);
385
386 $tables_check_ignore = $this->getMultisiteTableExclusions();
387
388 foreach ($tables_to_be_expected_on_target as $checktablename) {
389 // Check if table name is excluded from check, such as due to multisite stuff
390 if (in_array($checktablename, $tables_check_ignore)) {
391 continue;
392 }
393
394 $found = false;
395 foreach ($tables_on_target as $targettable) {
396 if ($checktablename == $targettable->name) {
397 $found = true;
398 break;
399 }
400 }
401
402 if (!$found) {
403 // Not found in correct case, now check for case insensitive
404 $found_case_insensitive = false;
405 $found_table_case = "";
406 foreach ($tables_on_target as $targettable) {
407 $found_table_case = $targettable->name;
408 if (strcasecmp($checktablename, $targettable->name) == 0) {
409 $found_case_insensitive = true;
410 break;
411 }
412 }
413
414 if ($found_case_insensitive) {
415 $warningmsg = sprintf(__("Finalize: Table %s is not found with the correct case. We found a table called %s. This may or may not give you problems. This happens due to SQL server configuration.", "wpsynchro"), $checktablename, $found_table_case);
416 $this->job->warnings[] = $warningmsg;
417 $this->logger->log("WARNING", $warningmsg);
418 } else {
419 $warningmsg = sprintf(__("Finalize: Table %s is not found on target. It may be a problem with the rename from temp table name.", "wpsynchro"), $checktablename, $found_table_case);
420 $this->job->warnings[] = $warningmsg;
421 $this->logger->log("WARNING", $warningmsg);
422 }
423 }
424 }
425 }
426
427 /**
428 * Function to help with finalizing database data and checks if rows are with reasonable limits
429 */
430 public function checkRowCountCompare($from_tablename, $from_rows, $to_rows)
431 {
432
433 $margin_for_warning_rows_equal = 5; // 5%
434
435 // If from has no rows, the to table should also be empty
436 if ($from_rows == 0 && $to_rows != 0) {
437 $this->job->errors[] = sprintf(__("Finalize: Error in database migration for table %s - It should not contain any rows", "wpsynchro"), $from_tablename);
438 return;
439 }
440
441 // If from has rows, but the to table is empty, could be memory limit hit, exceeding post max size or mysql max_packet_size
442 if ($from_rows > 0 && $to_rows == 0) {
443 $this->job->errors[] = sprintf(__("Finalize: Error in database migration for table %s - No rows has been transferred, but should contain %d rows. Normally this is because the ressource limits has been hit and the database content is too large. Contact support if this continues to fail.", "wpsynchro"), $from_tablename, $from_rows);
444 return;
445 }
446
447 // Check that rows approximately equal. Could have been changed a bit while synching, which is okay, but raises a warning if too much. Its okay if it is bigger
448 if ($to_rows < ((1 - ($margin_for_warning_rows_equal / 100)) * $from_rows)) {
449 $this->job->warnings[] = sprintf(__("Finalize: Warning in database migration for table %s - It differs more than %d%% in size, which indicate something has gone wrong during transfer. We found %d rows, but expected around %d rows.", "wpsynchro"), $from_tablename, $margin_for_warning_rows_equal, $to_rows, $from_rows);
450 }
451 }
452
453 /**
454 * Handle the multisite finalize sql
455 */
456 public function getMultisiteFinalizeSQL()
457 {
458 $multisite_sql = [];
459 return $multisite_sql;
460 }
461
462 /**
463 * Handle which tables to not check for on the target - aka those that might be renamed or removed, such as users table on multisite
464 */
465 public function getMultisiteTableExclusions()
466 {
467 $tables_to_exclude = [];
468 return $tables_to_exclude;
469 }
470
471 /**
472 * Get percentage completed for database finalize
473 */
474 public function getPercentCompletedForDatabaseFinalize()
475 {
476 // We have four primary steps - Initialize, rename queries, drop excess tables and the last checks
477 $completion = 0;
478
479 if ($this->job->finalize_db_initialized) {
480 $completion += 10;
481 }
482
483 if ($this->job->finalize_db_sql_queries_count > 0) {
484 $completion += 80 * (count($this->job->finalize_db_sql_queries) / $this->job->finalize_db_sql_queries_count);
485 } else {
486 $completion += 80;
487 }
488
489 if ($this->job->finalize_db_excess_table_queries_count > 0) {
490 $completion += 10 * (count($this->job->finalize_db_excess_table_queries) / $this->job->finalize_db_excess_table_queries_count);
491 } else {
492 $completion += 10;
493 }
494
495 return 1 - ($completion / 100);
496 }
497 }
498