better-backup-v3.php
2 years ago
better-backup.php
2 years ago
better-restore.php
2 years ago
even-better-restore-v3.php
2 years ago
even-better-restore-v4.php
2 years ago
manager.php
2 years ago
search-replace.php
2 years ago
smart-sort.php
2 years ago
smart-sort.php
1162 lines
| 1 | <?php |
| 2 | |
| 3 | /** |
| 4 | * Author: Mikołaj `iClyde` Chodorowski |
| 5 | * Contact: kontakt@iclyde.pl |
| 6 | * Package: Backup Migration – WP Plugin |
| 7 | * Version: 2.0 |
| 8 | */ |
| 9 | |
| 10 | // Namespace |
| 11 | namespace BMI\Plugin\Database; |
| 12 | |
| 13 | // Use |
| 14 | use BMI\Plugin\BMI_Logger AS Logger; |
| 15 | use BMI\Plugin\Progress\BMI_ZipProgress AS Progress; |
| 16 | |
| 17 | |
| 18 | // // Exit on direct access |
| 19 | if (!defined('ABSPATH')) exit; |
| 20 | |
| 21 | /** |
| 22 | * Database sort method |
| 23 | * Usage: Create new object and use "sortUnsorted" function |
| 24 | */ |
| 25 | class BMI_Database_Sorting { |
| 26 | |
| 27 | /** |
| 28 | * Path to database files made with V2 engine |
| 29 | */ |
| 30 | private $db_files_root; |
| 31 | private $output_file_stream; |
| 32 | private $name_of_finished_tables_dir = 'bmi_converted_completed_tables'; |
| 33 | private $progress; |
| 34 | private $isCLI; |
| 35 | private $unwantedTables = [ |
| 36 | 'wfblockediplog', |
| 37 | 'wfblocks7', |
| 38 | 'wfcrawlers', |
| 39 | 'wffilechanges', |
| 40 | 'wffilemods', |
| 41 | 'wfhits', |
| 42 | 'wfhoover', |
| 43 | 'wfissues', |
| 44 | 'wfknownfilelist', |
| 45 | 'wflivetraffichuman', |
| 46 | 'wflocs', |
| 47 | 'wflogins', |
| 48 | 'wfnotifications', |
| 49 | 'wfpendingissues', |
| 50 | 'wfreversecache', |
| 51 | 'wfsnipcache', |
| 52 | 'wfstatus', |
| 53 | 'wftrafficrate', |
| 54 | 'actionscheduler_logs', |
| 55 | 'slim_stats', |
| 56 | 'woocommerce_sessions', |
| 57 | 'yoast_indexable', |
| 58 | 'slim_events', |
| 59 | 'cerber_files', |
| 60 | 'cerber_traffic', |
| 61 | 'cerber_log', |
| 62 | 'cerber_countries', |
| 63 | 'cerber_blocks', |
| 64 | 'cerber_acl' |
| 65 | ]; |
| 66 | |
| 67 | /** |
| 68 | * __construct - description |
| 69 | * |
| 70 | * @param {string} $db_root Path to database files |
| 71 | * @return void |
| 72 | */ |
| 73 | function __construct($db_root, &$progress, $isCLI) { |
| 74 | |
| 75 | /** |
| 76 | * Set path to the root directory |
| 77 | */ |
| 78 | $this->db_files_root = $db_root; |
| 79 | $this->progress = $progress; |
| 80 | $this->isCLI = $isCLI; |
| 81 | |
| 82 | } |
| 83 | |
| 84 | /** |
| 85 | * sortUnsorted - Main function which dynamically sorts the tables (main function) |
| 86 | * |
| 87 | * @param {array} $processData = [] Data of process to be continued or empty array to start |
| 88 | * Requires keys [ 'index', 'insert', 'file', 'table_name', 'dir', 'unique', 'destinationFile' ] |
| 89 | * Optional key: convertionFinished |
| 90 | * |
| 91 | * @return array of [ 'index', 'insert', 'file', 'table_name', 'dir', 'unique', 'destinationFile' ] |
| 92 | */ |
| 93 | public function sortUnsorted($processData = []) { |
| 94 | |
| 95 | // Sort the SQL files into directories |
| 96 | $this->putDatabaseFilesInDirectories(); |
| 97 | |
| 98 | // Split files into partial files (query per batch) |
| 99 | $process = $this->splitFilesInSeparateDirectories($processData); |
| 100 | |
| 101 | // Check if all tables finished and move them outside completed directory |
| 102 | if ($this->doesTablesFinished()) { |
| 103 | |
| 104 | // Move them outside |
| 105 | $this->moveAllConvertedTables(); |
| 106 | |
| 107 | // Check if process finished |
| 108 | if ($this->isCLI || (is_array($process) && array_key_exists('completed', $process) && $process['completed'] == 'yes')) { |
| 109 | $process['convertionFinished'] = 'yes'; |
| 110 | } |
| 111 | |
| 112 | } |
| 113 | |
| 114 | // Return process for that |
| 115 | return $process; |
| 116 | |
| 117 | } |
| 118 | |
| 119 | /** |
| 120 | * doesTablesFinished - Checks if there is any table that didn't finish |
| 121 | * |
| 122 | * @return {bool} true or false |
| 123 | */ |
| 124 | private function doesTablesFinished() { |
| 125 | |
| 126 | // Scan all files in root |
| 127 | $allFiles = scandir($this->db_files_root); |
| 128 | $files = array_diff($allFiles, array('.', '..', $this->name_of_finished_tables_dir)); |
| 129 | |
| 130 | // If no other directories here that means nothing to convert left |
| 131 | if (sizeof($files) <= 0) { |
| 132 | |
| 133 | // Return true for that |
| 134 | return true; |
| 135 | |
| 136 | } else { |
| 137 | |
| 138 | // Otherwise it's false |
| 139 | return false; |
| 140 | |
| 141 | } |
| 142 | |
| 143 | } |
| 144 | |
| 145 | /** |
| 146 | * putDatabaseFilesInDirectories - It puts all SQL files into table directories |
| 147 | * Does nothing if the root directory does not contain any SQL files |
| 148 | * |
| 149 | * @return void |
| 150 | */ |
| 151 | private function putDatabaseFilesInDirectories() { |
| 152 | |
| 153 | // Scan all files in root |
| 154 | $allFiles = scandir($this->db_files_root); |
| 155 | $files = array_diff($allFiles, array('.', '..')); |
| 156 | |
| 157 | // For each found file make sure it's SQL and run merge function |
| 158 | foreach ($files as $index => $filename) { |
| 159 | |
| 160 | // Exclude non .sql files |
| 161 | if (substr($filename, -4) != '.sql') continue; |
| 162 | |
| 163 | // It may be directory with .sql ending, make sure it's not |
| 164 | $pathToFile = $this->db_files_root . DIRECTORY_SEPARATOR . $filename; |
| 165 | if (is_dir($pathToFile)) continue; |
| 166 | |
| 167 | // Merge the file |
| 168 | $this->mergeFileWithDirectory($filename); |
| 169 | |
| 170 | } |
| 171 | |
| 172 | } |
| 173 | |
| 174 | /** |
| 175 | * mergeFileWithDirectory - Merges the file into proper directory |
| 176 | * |
| 177 | * @param {string} $filename File name which is located in root |
| 178 | * @return bool true on success false on failed file move |
| 179 | */ |
| 180 | private function mergeFileWithDirectory($filename) { |
| 181 | |
| 182 | // Suffix size which stands for ending ".sql" |
| 183 | $suffix_size = 4; |
| 184 | |
| 185 | // Match output for table name with different name than just table |
| 186 | // It may happen when our support used their tools for optimization |
| 187 | $match_output = []; |
| 188 | preg_match('/_(\d+)\_of\_(\d+)/', $filename, $match_output); |
| 189 | |
| 190 | // If that's the case, add suffix size as we don't need that info here |
| 191 | if (sizeof($match_output) > 0) { |
| 192 | $suffix_size += strlen($match_output[0]); |
| 193 | } |
| 194 | |
| 195 | // If we know the suffix size, get table name |
| 196 | $table_name = substr($filename, 0, -$suffix_size); |
| 197 | |
| 198 | // Prepare future paths of the file |
| 199 | $dest_dir = $this->db_files_root . DIRECTORY_SEPARATOR . $table_name; |
| 200 | $source_path = $this->db_files_root . DIRECTORY_SEPARATOR . $filename; |
| 201 | $dest_path = $dest_dir . DIRECTORY_SEPARATOR . $filename; |
| 202 | |
| 203 | // Check if the directory exists and if it's not make new one |
| 204 | if (!file_exists($dest_dir) && !is_dir($dest_dir)) { |
| 205 | @mkdir($dest_dir, 0755, true); |
| 206 | } |
| 207 | |
| 208 | // Check if the files is not an duplicate and make sure both will be merged |
| 209 | $tries = 2; |
| 210 | while (file_exists($dest_path)) { |
| 211 | |
| 212 | // If file exists already add number at the suffix and try again until unique name |
| 213 | $dest_path = substr($dest_path, 0, -4) . '_' . $tries . '.sql'; |
| 214 | $tries++; |
| 215 | |
| 216 | } |
| 217 | |
| 218 | // If we're sure move the file to correct directory |
| 219 | if (rename($source_path, $dest_path)) { |
| 220 | |
| 221 | // Return true on success |
| 222 | return true; |
| 223 | |
| 224 | } else { |
| 225 | |
| 226 | // Return false on fail |
| 227 | return false; |
| 228 | |
| 229 | } |
| 230 | |
| 231 | } |
| 232 | |
| 233 | /** |
| 234 | * countAllFilesAndQueries - Counts all files inside directory and returns all query count |
| 235 | * |
| 236 | * @return {array} of total_queries, total_size, all_tables and all_files |
| 237 | */ |
| 238 | public function countAllFilesAndQueries() { |
| 239 | |
| 240 | // Scan all files in root |
| 241 | $allFiles = scandir($this->db_files_root); |
| 242 | $folders = array_diff($allFiles, array('.', '..', $this->name_of_finished_tables_dir)); |
| 243 | |
| 244 | // Tables |
| 245 | $tables = []; |
| 246 | |
| 247 | // Files |
| 248 | $files = []; |
| 249 | |
| 250 | // Total size |
| 251 | $total_size = 0; |
| 252 | |
| 253 | // For each found file make sure it's SQL and run merge function |
| 254 | foreach ($folders as $index => $filename) { |
| 255 | |
| 256 | // Path to that file/directory |
| 257 | $pathToFile = $this->db_files_root . DIRECTORY_SEPARATOR . $filename; |
| 258 | |
| 259 | // Hanlde only dirs |
| 260 | if (is_dir($pathToFile)) { |
| 261 | |
| 262 | // Add table information |
| 263 | $tables[$filename] = 0; |
| 264 | |
| 265 | // Scan SQL files |
| 266 | $allSQLFiles = scandir($pathToFile); |
| 267 | $allSQLFilesParsed = array_diff($allSQLFiles, array('.', '..')); |
| 268 | |
| 269 | // Go throught these files |
| 270 | foreach ($allSQLFilesParsed as $indexSQL => $filenameSQL) { |
| 271 | |
| 272 | // Exclude non .sql files |
| 273 | if (substr($filenameSQL, -4) != '.sql') continue; |
| 274 | |
| 275 | // Get size of that file |
| 276 | $size = filesize($pathToFile . DIRECTORY_SEPARATOR . $filenameSQL); |
| 277 | |
| 278 | // Add size in total and table |
| 279 | $tables[$filename] += $size; |
| 280 | $total_size += $size; |
| 281 | |
| 282 | // Add that file to all files |
| 283 | $files[] = $filenameSQL; |
| 284 | |
| 285 | } |
| 286 | |
| 287 | } |
| 288 | |
| 289 | } |
| 290 | |
| 291 | // Prepare stats variable |
| 292 | $stats = [ |
| 293 | 'total_queries' => (sizeof($files) * 5), |
| 294 | 'total_size' => $total_size, |
| 295 | 'all_tables' => $tables, |
| 296 | 'all_files' => $files |
| 297 | ]; |
| 298 | |
| 299 | return $stats; |
| 300 | |
| 301 | } |
| 302 | |
| 303 | /** |
| 304 | * countTablesDuringProcess - Counts tables even during process |
| 305 | * |
| 306 | * @return array $done, $progress, $total |
| 307 | */ |
| 308 | private function countTablesDuringProcess() { |
| 309 | |
| 310 | $theFinalCount = 0; |
| 311 | $progressCount = 0; |
| 312 | $doneCount = 0; |
| 313 | |
| 314 | // Scan all files in root (now everything should be directory) |
| 315 | $allFiles = scandir($this->db_files_root); |
| 316 | $files = array_diff($allFiles, array('.', '..', $this->name_of_finished_tables_dir)); |
| 317 | |
| 318 | foreach ($files as $index => $filename) { |
| 319 | if (is_dir($this->db_files_root . DIRECTORY_SEPARATOR . $filename)) { |
| 320 | $theFinalCount++; |
| 321 | $progressCount++; |
| 322 | } |
| 323 | } |
| 324 | |
| 325 | $doneDir = $this->db_files_root . DIRECTORY_SEPARATOR . $this->name_of_finished_tables_dir; |
| 326 | if (file_exists($doneDir) && is_dir($doneDir)) { |
| 327 | |
| 328 | $allDoneFiles = scandir($doneDir); |
| 329 | $doneFiles = array_diff($allDoneFiles, array('.', '..')); |
| 330 | |
| 331 | foreach ($doneFiles as $index => $filename) { |
| 332 | if (is_dir($doneDir . DIRECTORY_SEPARATOR . $filename)) { |
| 333 | $theFinalCount++; |
| 334 | $doneCount++; |
| 335 | } |
| 336 | } |
| 337 | |
| 338 | } |
| 339 | |
| 340 | return [ |
| 341 | 'done' => $doneCount, |
| 342 | 'in_progress' => $progressCount, |
| 343 | 'total' => $theFinalCount |
| 344 | ]; |
| 345 | |
| 346 | } |
| 347 | |
| 348 | /** |
| 349 | * logFinished - Shows progress of the process in the log file |
| 350 | * |
| 351 | * @return void |
| 352 | */ |
| 353 | private function logFinished($dir, $filename) { |
| 354 | |
| 355 | $counts = $this->countTablesDuringProcess(); |
| 356 | |
| 357 | $progress = ($counts['done'] + 1) . '/' . $counts['total'] . ' (' . number_format((($counts['done'] + 1) / $counts['total']) * 100, 2) . '%)'; |
| 358 | $translated = __('Finished %progress%: %filename% table, total of %parts% parts.', 'backup-backup'); |
| 359 | |
| 360 | // Get part amount |
| 361 | $partsAll = scandir($dir); |
| 362 | $parts = sizeof(array_diff($partsAll, array('.', '..', $this->name_of_finished_tables_dir))); |
| 363 | |
| 364 | $theLog = str_replace('%progress%', $progress, $translated); |
| 365 | $theLog = str_replace('%filename%', $filename, $theLog); |
| 366 | $theLog = str_replace('%parts%', $parts, $theLog); |
| 367 | |
| 368 | $this->progress->log($theLog, 'SUCCESS'); |
| 369 | |
| 370 | $percentage = number_format((($counts['done'] + 1) / $counts['total']) * 100, 2); |
| 371 | $this->progress->progress((50 + ($percentage / 4)), 'INFO'); |
| 372 | |
| 373 | } |
| 374 | |
| 375 | /** |
| 376 | * splitInDirectories - Splits file into parts inside proper directory |
| 377 | * @param {array} $processData Process data to be continued |
| 378 | * |
| 379 | * @return void |
| 380 | */ |
| 381 | private function splitFilesInSeparateDirectories($processData = []) { |
| 382 | |
| 383 | // Local process variable |
| 384 | $process = []; |
| 385 | |
| 386 | // Scan all files in root (now everything should be directory) |
| 387 | $allFiles = scandir($this->db_files_root); |
| 388 | $files = array_diff($allFiles, array('.', '..', $this->name_of_finished_tables_dir)); |
| 389 | |
| 390 | // Loop all tables and split them one by one |
| 391 | foreach ($files as $index => $filename) { |
| 392 | |
| 393 | // Prepate path to the SQL file which should be converted |
| 394 | $dir = $this->db_files_root . DIRECTORY_SEPARATOR . $filename; |
| 395 | |
| 396 | // Always double check if it's directory otherwise we may get some unexpected errors |
| 397 | if (!is_dir($dir)) continue; |
| 398 | |
| 399 | // Log the new file which will be processed |
| 400 | $destinationFile = $this->makeDestinationFilename($dir, $filename); |
| 401 | $destinationIndex = $destinationFile['index']; |
| 402 | |
| 403 | // Log only if it's first batch |
| 404 | if ($destinationIndex == 1) { |
| 405 | $this->progress->log(str_replace('%s', $filename, __('Starting conversion of: %s table.', 'backup-backup')), 'INFO'); |
| 406 | } |
| 407 | |
| 408 | // Dynamically find file to shrink and split it |
| 409 | $process = $this->splitFileInTableDirectory($dir, $filename, $processData); |
| 410 | |
| 411 | // If it's not CLI use batching |
| 412 | if (!$this->isCLI) { |
| 413 | |
| 414 | // If process finished |
| 415 | if (is_array($process) && array_key_exists('completed', $process) && $process['completed'] == 'yes') { |
| 416 | |
| 417 | // Show in the log file the progress |
| 418 | $this->logFinished($dir, $filename); |
| 419 | |
| 420 | // Check if directory for completed tables exist and make it if it is not |
| 421 | $doneDirectory = $this->db_files_root . DIRECTORY_SEPARATOR . $this->name_of_finished_tables_dir; |
| 422 | if (!file_exists($doneDirectory) && !is_dir($doneDirectory)) { |
| 423 | @mkdir($doneDirectory, 0755, true); |
| 424 | } |
| 425 | |
| 426 | // Move the table into completed directory |
| 427 | rename($dir, $doneDirectory . DIRECTORY_SEPARATOR . $filename); |
| 428 | |
| 429 | // Return process even empty |
| 430 | return $process; |
| 431 | |
| 432 | } else { |
| 433 | |
| 434 | // If didn't finished that table return process data for next batch to finish |
| 435 | return $process; |
| 436 | |
| 437 | } |
| 438 | |
| 439 | } else { |
| 440 | |
| 441 | // Show in the log file the progress |
| 442 | $this->logFinished($dir, $filename); |
| 443 | |
| 444 | // Check if directory for completed tables exist and make it if it is not |
| 445 | $doneDirectory = $this->db_files_root . DIRECTORY_SEPARATOR . $this->name_of_finished_tables_dir; |
| 446 | if (!file_exists($doneDirectory) && !is_dir($doneDirectory)) { |
| 447 | @mkdir($doneDirectory, 0755, true); |
| 448 | } |
| 449 | |
| 450 | // Move the table into completed directory |
| 451 | rename($dir, $doneDirectory . DIRECTORY_SEPARATOR . $filename); |
| 452 | |
| 453 | } |
| 454 | |
| 455 | } |
| 456 | |
| 457 | } |
| 458 | |
| 459 | /** |
| 460 | * splitFileInTableDirectory - Splits file into parts for easier progress read and performance |
| 461 | * |
| 462 | * @param {string} $dir Path to parent directory of table SQL files |
| 463 | * @param {string} $table_name Table name (current directory name) |
| 464 | * @param {array} $processData Process data to be continued |
| 465 | * @return void |
| 466 | */ |
| 467 | private function splitFileInTableDirectory($dir, $table_name, $processData = []) { |
| 468 | |
| 469 | // Prepare local var for process data |
| 470 | $process = []; |
| 471 | |
| 472 | // Get all database files inside that directory (it can contain multiple files alrady) |
| 473 | $allFiles = scandir($dir); |
| 474 | $files = array_diff($allFiles, array('.', '..')); |
| 475 | |
| 476 | // Loop throught all of them and apply splitting |
| 477 | foreach ($files as $index => $filename) { |
| 478 | |
| 479 | // Just to triple check that parent is a valid directory |
| 480 | if (!is_dir($dir)) continue; |
| 481 | |
| 482 | // Prepare exact path to the SQL file |
| 483 | $file = $dir . DIRECTORY_SEPARATOR . $filename; |
| 484 | |
| 485 | // Run splitting function for that file |
| 486 | $process = $this->splitDatabaseFile($file, $dir, $table_name, $processData); |
| 487 | |
| 488 | // Break each query if not CLI |
| 489 | if (!$this->isCLI) { |
| 490 | break; |
| 491 | } |
| 492 | |
| 493 | } |
| 494 | |
| 495 | // If CLI only rename files without return |
| 496 | if ($this->isCLI) { |
| 497 | |
| 498 | // Rename splited files |
| 499 | $this->renameSplitedFiles($dir); |
| 500 | |
| 501 | } else { |
| 502 | |
| 503 | // Check if process for that table completed and rename if so |
| 504 | if (is_array($process) && array_key_exists('completed', $process) && $process['completed'] == 'yes') { |
| 505 | |
| 506 | // Rename splited files |
| 507 | $this->renameSplitedFiles($dir); |
| 508 | |
| 509 | // Return empty process data as we will iterate new file in next batch |
| 510 | return [ |
| 511 | 'index' => 0, |
| 512 | 'insert' => 'no', |
| 513 | 'completed' => 'yes' |
| 514 | ]; |
| 515 | |
| 516 | } else { |
| 517 | |
| 518 | // Or just return process data for future use (next batch) |
| 519 | return $process; |
| 520 | |
| 521 | } |
| 522 | |
| 523 | } |
| 524 | |
| 525 | } |
| 526 | |
| 527 | /** |
| 528 | * logPartial - Logs partial progress to the output log |
| 529 | * |
| 530 | * @param {string} $dir Directory of the table |
| 531 | * @param {string} $table_name Name of the table |
| 532 | * @return void |
| 533 | */ |
| 534 | private function logPartial($dir, $table_name) { |
| 535 | |
| 536 | // Get part amount |
| 537 | $partsAll = scandir($dir); |
| 538 | $parts = sizeof(array_diff($partsAll, array('.', '..', $this->name_of_finished_tables_dir))); |
| 539 | $parts = $parts - 1; |
| 540 | |
| 541 | // Prepare log string |
| 542 | $logString = __('Finished part %part% of %table% table.', 'backup-backup'); |
| 543 | $logString = str_replace('%table%', $table_name, $logString); |
| 544 | $logString = str_replace('%part%', $parts, $logString); |
| 545 | |
| 546 | // Log the string to output |
| 547 | $this->progress->log($logString, 'INFO'); |
| 548 | |
| 549 | } |
| 550 | |
| 551 | /** |
| 552 | * splitDatabaseFile - Splits a SQL file into parts |
| 553 | * |
| 554 | * @param {string} $file Name of the SQL File |
| 555 | * @param {string} $dir Full path to the SQL file |
| 556 | * @param {string} $table_name Name of the table to which SQL file belong |
| 557 | * @param {array} $processData Process data to be continued |
| 558 | * @return void |
| 559 | */ |
| 560 | private function splitDatabaseFile($file, $dir, $table_name, $processData = []) { |
| 561 | |
| 562 | // Make destionation file and prepare output file stream |
| 563 | $destinationFile = $this->makeDestinationFilename($dir, $table_name); |
| 564 | $destinationFile = $destinationFile['file']; |
| 565 | |
| 566 | // Local variables required for the process |
| 567 | $table_unique = 0; |
| 568 | $is_converted = false; |
| 569 | $query_started = false; |
| 570 | $custom_vars_started = false; |
| 571 | $values_started = false; |
| 572 | $insert_next = false; |
| 573 | $last_seek = 0; |
| 574 | |
| 575 | // Resolve not finished process |
| 576 | if (!$this->isCLI && is_array($processData) && !empty($processData)) { |
| 577 | |
| 578 | // Should be file path $file_path & $file (check if the file exist) |
| 579 | if (array_key_exists('file', $processData) && file_exists($processData['file'])) { |
| 580 | |
| 581 | // Set existing file path |
| 582 | $file = $processData['file']; |
| 583 | |
| 584 | // Should be int<make sure> of $last_seek |
| 585 | if (array_key_exists('index', $processData)) { |
| 586 | $last_seek = intval($processData['index']) + 1; |
| 587 | } |
| 588 | |
| 589 | // Yes | No values {string} of $insert_next |
| 590 | if (array_key_exists('insert', $processData)) { |
| 591 | $ins = $processData['insert']; |
| 592 | |
| 593 | if ($ins == 'yes') $insert_next = true; |
| 594 | else $insert_next = false; |
| 595 | } |
| 596 | |
| 597 | // Table name of $table_name |
| 598 | if (array_key_exists('table_name', $processData)) { |
| 599 | $table_name = $processData['table_name']; |
| 600 | } |
| 601 | |
| 602 | // Directory of that table $dir |
| 603 | if (array_key_exists('dir', $processData)) { |
| 604 | $dir = $processData['dir']; |
| 605 | } |
| 606 | |
| 607 | // Unique for that table $table_unique |
| 608 | if (array_key_exists('unique', $processData)) { |
| 609 | $table_unique = $processData['unique']; |
| 610 | } |
| 611 | |
| 612 | // Destination File (output) of $destinationFile |
| 613 | if (array_key_exists('destinationFile', $processData)) { |
| 614 | $destinationFile = $processData['destinationFile']; |
| 615 | } |
| 616 | |
| 617 | } |
| 618 | |
| 619 | } |
| 620 | |
| 621 | // Open stream for output file |
| 622 | $this->output_file_stream = fopen($destinationFile, 'a+'); |
| 623 | |
| 624 | // Keep original filename |
| 625 | $file_path = $file; |
| 626 | |
| 627 | // Open SplObject for seek-based reading |
| 628 | $file = new \SplFileObject($file); |
| 629 | |
| 630 | // Go to last line to check how many lines are there |
| 631 | $file->seek($file->getSize()); |
| 632 | |
| 633 | // Calculate total lines in that file |
| 634 | $total_lines = $file->key() + 1; |
| 635 | |
| 636 | for ($i = $last_seek; $i < $total_lines; ++$i) { |
| 637 | |
| 638 | // Start seek frrom the loop index |
| 639 | $file->seek($i); |
| 640 | |
| 641 | // Get line into string and trim it (it contains new line at the end) |
| 642 | $line = trim($file->current()); |
| 643 | |
| 644 | // Variable for that line, if it's a comment or not |
| 645 | $hasComment = false; |
| 646 | |
| 647 | // Check this only if the size of line is below 24 |
| 648 | // Otherwise it's not a comment line and we can ignore that check for better performance |
| 649 | if (strlen($line) < 24) { |
| 650 | |
| 651 | // List of all our comments |
| 652 | $c_fs = "/* CONVERTED DB FILE */"; |
| 653 | $q_s = "/* QUERY START */"; |
| 654 | $q_e = "/* QUERY END */"; |
| 655 | $cq_s = "/* CUSTOM VARS START */"; |
| 656 | $cq_e = "/* CUSTOM VARS END */"; |
| 657 | $v_s = "/* VALUES START */"; |
| 658 | $v_e = "/* VALUES END */"; |
| 659 | |
| 660 | // If it's converted already ignore the file |
| 661 | if ($line === $c_fs) { |
| 662 | |
| 663 | // Mark as converted (it output file be removed by this) |
| 664 | $is_converted = true; |
| 665 | |
| 666 | // And break the loop, no need to continue |
| 667 | break; |
| 668 | |
| 669 | } |
| 670 | |
| 671 | // Check if it's a query start comment and adjust local variables for that for future know |
| 672 | if ($line === $q_s) { |
| 673 | $query_started = true; |
| 674 | $hasComment = true; |
| 675 | } |
| 676 | |
| 677 | // Check if it's a custom variable section start comment and adjust local variables for that for future know |
| 678 | if ($line === $cq_s) { |
| 679 | $custom_vars_started = true; |
| 680 | $hasComment = true; |
| 681 | } |
| 682 | |
| 683 | // Check if it's a values section start comment and adjust local variables for that for future know |
| 684 | if ($line === $v_s) { |
| 685 | $values_started = true; |
| 686 | $hasComment = true; |
| 687 | } |
| 688 | |
| 689 | // If we are behind a heading insert this line into new output file |
| 690 | if ($insert_next && $hasComment) { |
| 691 | |
| 692 | // Add new line as we trimed it before |
| 693 | $line .= "\n"; |
| 694 | |
| 695 | // Push this to output file |
| 696 | $this->insertIntoOutputFile($line, $table_name); |
| 697 | |
| 698 | // If we already added this line continue to new line |
| 699 | continue; |
| 700 | |
| 701 | } |
| 702 | |
| 703 | // Handle query ending comment |
| 704 | if ($line === $q_e) { |
| 705 | |
| 706 | // Adjust local variable for that file |
| 707 | $query_started = false; |
| 708 | |
| 709 | // If it's below heading, add new output and generate new heading for output file |
| 710 | if ($insert_next) { |
| 711 | |
| 712 | // Add double ending – it looks much better for our eyes :) |
| 713 | $line .= "\n\n"; |
| 714 | |
| 715 | // Add this line |
| 716 | $this->insertIntoOutputFile($line, $table_name); |
| 717 | |
| 718 | // Close the file as we split per query, we will need new one for new query |
| 719 | if (is_resource($this->output_file_stream) || get_resource_type($this->output_file_stream) === 'file') { |
| 720 | fclose($this->output_file_stream); |
| 721 | } |
| 722 | |
| 723 | // Check if it's end of the file add 4 lines offset for empty lines |
| 724 | // If it's not the end make new heading, otherwise remove new output file |
| 725 | if (($i + 4) < $total_lines) { |
| 726 | |
| 727 | // Make new destination file |
| 728 | $destinationFile = $this->makeDestinationFilename($dir, $table_name); |
| 729 | $destinationFile = $destinationFile['file']; |
| 730 | |
| 731 | // Check if we really need that table contents |
| 732 | if ($this->checkIfContainName($table_name, $this->unwantedTables)) { |
| 733 | |
| 734 | $this->progress->log(str_replace('%s', $table_name, __('Cleaning up contents of %s table.', 'backup-backup')), 'INFO'); |
| 735 | |
| 736 | $insert_next = false; |
| 737 | |
| 738 | if (!$this->isCLI) { |
| 739 | |
| 740 | $file = null; |
| 741 | |
| 742 | if (is_resource($this->output_file_stream) || get_resource_type($this->output_file_stream) === 'file') { |
| 743 | fclose($this->output_file_stream); |
| 744 | } |
| 745 | |
| 746 | if (file_exists($destinationFile)) { |
| 747 | @unlink($destinationFile); |
| 748 | } |
| 749 | |
| 750 | if (file_exists($file_path)) { |
| 751 | @unlink($file_path); |
| 752 | } |
| 753 | |
| 754 | // Return empty for new file |
| 755 | return [ |
| 756 | 'index' => 0, |
| 757 | 'insert' => 'no', |
| 758 | 'completed' => 'yes' |
| 759 | ]; |
| 760 | |
| 761 | } |
| 762 | |
| 763 | } else { |
| 764 | |
| 765 | // Log partial progress |
| 766 | $this->logPartial($dir, $table_name); |
| 767 | |
| 768 | // Open new stream for new output file |
| 769 | $this->output_file_stream = fopen($destinationFile, 'a+'); |
| 770 | |
| 771 | // Generate new heading |
| 772 | $heading = $this->generateNewHeading($table_unique, $table_name); |
| 773 | |
| 774 | // Put the heading into new file |
| 775 | $this->insertIntoOutputFile($heading, $table_name); |
| 776 | |
| 777 | // Technically only few bytes but with 200 tables it may be few MBs :) |
| 778 | // Always unset variables if you don't need them anymore |
| 779 | unset($heading); |
| 780 | |
| 781 | } |
| 782 | |
| 783 | if (!$this->isCLI) { |
| 784 | |
| 785 | // Check if the output file is open and close it if it is |
| 786 | $file = null; |
| 787 | if (is_resource($this->output_file_stream) || get_resource_type($this->output_file_stream) === 'file') { |
| 788 | fclose($this->output_file_stream); |
| 789 | } |
| 790 | |
| 791 | // Return latest insert query |
| 792 | return [ |
| 793 | 'index' => $i, |
| 794 | 'insert' => ($insert_next == true ? 'yes' : 'no'), |
| 795 | 'file' => $file_path, |
| 796 | 'table_name' => $table_name, |
| 797 | 'dir' => $dir, |
| 798 | 'unique' => $table_unique, |
| 799 | 'destinationFile' => $destinationFile |
| 800 | ]; |
| 801 | |
| 802 | } |
| 803 | |
| 804 | } else { |
| 805 | |
| 806 | // Log partial progress |
| 807 | $this->logPartial($dir, $table_name); |
| 808 | |
| 809 | } |
| 810 | |
| 811 | // Continue to new line as we handled this one already |
| 812 | continue; |
| 813 | |
| 814 | } |
| 815 | |
| 816 | } |
| 817 | |
| 818 | // Handle Custom Variables ending comment |
| 819 | if ($line === $cq_e) { |
| 820 | |
| 821 | // Adjust local variables for next loops |
| 822 | $custom_vars_started = false; |
| 823 | |
| 824 | } |
| 825 | |
| 826 | // Handle values ending |
| 827 | if ($line === $v_e) { |
| 828 | |
| 829 | // Adjust local variables for next loops |
| 830 | $values_started = false; |
| 831 | |
| 832 | // If it's after heading add this line into new output file |
| 833 | if ($insert_next) { |
| 834 | |
| 835 | // Add new line as we trimed it before |
| 836 | $line .= "\n"; |
| 837 | |
| 838 | // Insert that line into new file |
| 839 | $this->insertIntoOutputFile($line, $table_name); |
| 840 | |
| 841 | // As we added the line continue to new one |
| 842 | continue; |
| 843 | |
| 844 | } |
| 845 | |
| 846 | } |
| 847 | |
| 848 | } |
| 849 | |
| 850 | // Check if the line is not a comment |
| 851 | if (!$hasComment) { |
| 852 | |
| 853 | // If the line should be a part of query handle that case |
| 854 | if ($query_started === true) { |
| 855 | |
| 856 | // If it's after heading allow to add this line to new output file |
| 857 | if ($insert_next) { |
| 858 | |
| 859 | // Add new line as we trimed it earlier |
| 860 | $line .= "\n"; |
| 861 | |
| 862 | // Insert the line into new file |
| 863 | $this->insertIntoOutputFile($line, $table_name); |
| 864 | |
| 865 | // As we added that line continue to new one |
| 866 | continue; |
| 867 | |
| 868 | } |
| 869 | |
| 870 | } |
| 871 | |
| 872 | // Handle custom variables |
| 873 | if ($custom_vars_started === true) { |
| 874 | |
| 875 | // We are interested only into PRE TABLE NAME which contains unique timestamp |
| 876 | if (strpos($line, 'PRE_TABLE_NAME')) { |
| 877 | |
| 878 | // Prepare variable for preg match output |
| 879 | $output = []; |
| 880 | |
| 881 | // Run the preg match function to find the |
| 882 | preg_match('/\`(.*)\`/', $line, $output); |
| 883 | |
| 884 | // Get the unique value from regex output |
| 885 | $unique = substr($output[1], 0, 10); |
| 886 | |
| 887 | // Set the local variable to found unique for future use in the heading |
| 888 | $table_unique = $unique; |
| 889 | |
| 890 | // As we went throught old heading let the script know that it can insert new lines in next loops |
| 891 | $insert_next = true; |
| 892 | |
| 893 | // Prepare new heading for file which will be used in next loops |
| 894 | $heading = $this->generateNewHeading($unique, $table_name); |
| 895 | |
| 896 | // Insert that heading |
| 897 | $this->insertIntoOutputFile($heading, $table_name); |
| 898 | |
| 899 | } |
| 900 | |
| 901 | } |
| 902 | |
| 903 | // Handle casual variable line |
| 904 | if ($values_started === true) { |
| 905 | |
| 906 | // If it can insert new output handle it |
| 907 | if ($insert_next) { |
| 908 | |
| 909 | // Insert that line into new file |
| 910 | $this->insertIntoOutputFile($line, $table_name); |
| 911 | |
| 912 | // Free up the memory as this line may be actually above few MBs. |
| 913 | unset($line); |
| 914 | |
| 915 | // Continue to new line as we inserted this one already |
| 916 | continue; |
| 917 | |
| 918 | } |
| 919 | |
| 920 | } |
| 921 | |
| 922 | } |
| 923 | |
| 924 | // Free up the memory as this line may be actually above few MBs. |
| 925 | unset($line); |
| 926 | |
| 927 | } |
| 928 | |
| 929 | // $total_lines |
| 930 | // File size: $file->getSize() / 1024 / 1024 |
| 931 | |
| 932 | // Check if the output file is open and close it if it is |
| 933 | if (is_resource($this->output_file_stream) || get_resource_type($this->output_file_stream) === 'file') { |
| 934 | fclose($this->output_file_stream); |
| 935 | } |
| 936 | |
| 937 | // If the file was already converted |
| 938 | if ($is_converted === false) { |
| 939 | |
| 940 | // Empty the SplFileObject stream |
| 941 | $file = null; |
| 942 | |
| 943 | // Remove new output file as there's nothing to do with it |
| 944 | @unlink($file_path); |
| 945 | |
| 946 | } else { |
| 947 | |
| 948 | // If the convert finished, remove source file as we now have splited files |
| 949 | @unlink($destinationFile); |
| 950 | |
| 951 | } |
| 952 | |
| 953 | // Empty SplFileObject stream |
| 954 | $file = null; |
| 955 | |
| 956 | // Only if not CLI |
| 957 | if (!$this->isCLI) { |
| 958 | |
| 959 | // Return empty process data as we will iterate new file in next batch |
| 960 | return [ |
| 961 | 'index' => 0, |
| 962 | 'insert' => 'no', |
| 963 | 'completed' => 'yes' |
| 964 | ]; |
| 965 | |
| 966 | } |
| 967 | |
| 968 | } |
| 969 | |
| 970 | /** |
| 971 | * makeHeading - Generates new heading (converted) for output file |
| 972 | * |
| 973 | * @param {string/int} $unique Unique timestamp generated during backup (all tables have to be part of it) |
| 974 | * @param {string} $table_name Table name of current file, required to generate the heading |
| 975 | * @return {string} New heading |
| 976 | */ |
| 977 | private function generateNewHeading($unique, $table_name) { |
| 978 | |
| 979 | $heading = ''; |
| 980 | |
| 981 | $heading .= "/* CONVERTED DB FILE */\n\n"; |
| 982 | $heading .= "/* QUERY START */\n"; |
| 983 | $heading .= "SET foreign_key_checks = 0;\n"; |
| 984 | $heading .= "/* QUERY END */\n"; |
| 985 | $heading .= "\n"; |
| 986 | $heading .= "/* QUERY START */\n"; |
| 987 | $heading .= "SET SQL_MODE = '';\n"; |
| 988 | $heading .= "/* QUERY END */\n"; |
| 989 | $heading .= "\n"; |
| 990 | $heading .= "/* QUERY START */\n"; |
| 991 | $heading .= "SET time_zone = '+00:00';\n"; |
| 992 | $heading .= "/* QUERY END */\n"; |
| 993 | $heading .= "\n"; |
| 994 | $heading .= "/* QUERY START */\n"; |
| 995 | $heading .= "SET NAMES 'utf8';\n"; |
| 996 | $heading .= "/* QUERY END */\n"; |
| 997 | $heading .= "\n"; |
| 998 | $heading .= "/* CUSTOM VARS START */\n"; |
| 999 | $heading .= "/* REAL_TABLE_NAME: `" . $table_name . "`; */\n"; |
| 1000 | $heading .= "/* PRE_TABLE_NAME: `" . $unique . "_" . $table_name . "`; */\n"; |
| 1001 | $heading .= "/* CUSTOM VARS END */\n"; |
| 1002 | $heading .= "\n"; |
| 1003 | |
| 1004 | return $heading; |
| 1005 | |
| 1006 | } |
| 1007 | |
| 1008 | /** |
| 1009 | * insertIntoOutputFile - Puts and output data into new (current) file |
| 1010 | * |
| 1011 | * @param {string} &$line Pointer to line, to save the memory |
| 1012 | * @param {string} $table_name Table name, may be useful for future use |
| 1013 | * @return void |
| 1014 | */ |
| 1015 | private function insertIntoOutputFile(&$line, $table_name) { |
| 1016 | |
| 1017 | if (is_resource($this->output_file_stream) || get_resource_type($this->output_file_stream) === 'file') { |
| 1018 | fwrite($this->output_file_stream, $line); |
| 1019 | } |
| 1020 | |
| 1021 | } |
| 1022 | |
| 1023 | /** |
| 1024 | * destinationFile - Makes unique destionation file name |
| 1025 | * |
| 1026 | * @param {string} $dir Full path to the SQL file |
| 1027 | * @param {string} $table_name Name of the table to which SQL file belong |
| 1028 | * @return {string} name of the destionation SQL file |
| 1029 | */ |
| 1030 | private function makeDestinationFilename($dir, $table_name) { |
| 1031 | |
| 1032 | // Start from file number one |
| 1033 | $i = 1; |
| 1034 | |
| 1035 | // Make an assumption that it will be the first file |
| 1036 | $file = $table_name . '_' . $i . '.sql'; |
| 1037 | |
| 1038 | // Loop until it find unqiue name |
| 1039 | while (file_exists($dir . DIRECTORY_SEPARATOR . $file)) { |
| 1040 | |
| 1041 | // Increment each iteration for new filename |
| 1042 | $i++; |
| 1043 | |
| 1044 | // Set new name to check |
| 1045 | $file = $table_name . '_' . $i . '.sql'; |
| 1046 | |
| 1047 | } |
| 1048 | |
| 1049 | // Return final file name |
| 1050 | return ['file' => $dir . DIRECTORY_SEPARATOR . $file, 'index' => $i]; |
| 1051 | |
| 1052 | } |
| 1053 | |
| 1054 | /** |
| 1055 | * |
| 1056 | * renameSplitedFiles - Renames all files after splitting into sorted SQL files |
| 1057 | * It also gives better progress experience for the user |
| 1058 | * |
| 1059 | * @param {string} $directory Path to directory where the rename function should run |
| 1060 | * @return void |
| 1061 | */ |
| 1062 | private function renameSplitedFiles($directory) { |
| 1063 | |
| 1064 | // Scan all files after splitting |
| 1065 | $allFiles = scandir($directory); |
| 1066 | $files = array_diff($allFiles, array('.', '..')); |
| 1067 | |
| 1068 | // Loop throught them and add proper number for each |
| 1069 | foreach ($files as $index => $filename) { |
| 1070 | |
| 1071 | // Make new destination name |
| 1072 | $newName = substr($filename, 0, -4) . '_of_' . sizeof($files) . '.sql'; |
| 1073 | |
| 1074 | // Current file path |
| 1075 | $file = $directory . DIRECTORY_SEPARATOR . $filename; |
| 1076 | |
| 1077 | // Final path (after rename) |
| 1078 | $finalPathWithName = $directory . DIRECTORY_SEPARATOR . $newName; |
| 1079 | |
| 1080 | // Finally rename the file |
| 1081 | rename($file, $finalPathWithName); |
| 1082 | |
| 1083 | } |
| 1084 | |
| 1085 | } |
| 1086 | |
| 1087 | /** |
| 1088 | * checkIfContainName - Checks if A (string) is part of B array |
| 1089 | * |
| 1090 | * @return bool |
| 1091 | */ |
| 1092 | private function checkIfContainName($tableName, $unwantedTables) { |
| 1093 | |
| 1094 | $found = false; |
| 1095 | |
| 1096 | $tableName = str_replace('-', '', $tableName); |
| 1097 | $tableName = str_replace('_', '', $tableName); |
| 1098 | $tableName = strtolower($tableName); |
| 1099 | |
| 1100 | for ($i = 0; $i < sizeof($unwantedTables); ++$i) { |
| 1101 | |
| 1102 | $name = $unwantedTables[$i]; |
| 1103 | $name = str_replace('-', '', $name); |
| 1104 | $name = str_replace('_', '', $name); |
| 1105 | $name = strtolower($name); |
| 1106 | |
| 1107 | if (strpos($tableName, $name) !== false) { |
| 1108 | $found = true; |
| 1109 | break; |
| 1110 | } |
| 1111 | |
| 1112 | } |
| 1113 | |
| 1114 | return $found; |
| 1115 | |
| 1116 | } |
| 1117 | |
| 1118 | /** |
| 1119 | * moveAllConvertedTables - Moves all completed tables outside |
| 1120 | * |
| 1121 | * @return void |
| 1122 | */ |
| 1123 | private function moveAllConvertedTables() { |
| 1124 | |
| 1125 | // Finished directory path |
| 1126 | $finishedDBDir = $this->db_files_root . DIRECTORY_SEPARATOR . $this->name_of_finished_tables_dir; |
| 1127 | |
| 1128 | // Check if it exists |
| 1129 | if (!file_exists($finishedDBDir) || !is_dir($finishedDBDir)) { |
| 1130 | return; |
| 1131 | } |
| 1132 | |
| 1133 | // Scan all tables in finished directory |
| 1134 | $allFiles = scandir($finishedDBDir); |
| 1135 | $files = array_diff($allFiles, array('.', '..')); |
| 1136 | |
| 1137 | // Loop throught them and move outside |
| 1138 | foreach ($files as $index => $filename) { |
| 1139 | |
| 1140 | // Old path |
| 1141 | $old = $this->db_files_root . DIRECTORY_SEPARATOR . $this->name_of_finished_tables_dir . DIRECTORY_SEPARATOR . $filename; |
| 1142 | |
| 1143 | // New path |
| 1144 | $new = $this->db_files_root . DIRECTORY_SEPARATOR . $filename; |
| 1145 | |
| 1146 | // Move that directory to parent |
| 1147 | rename($old, $new); |
| 1148 | |
| 1149 | } |
| 1150 | |
| 1151 | // Unlink directory that won't be used as it's (and it should) be empty. |
| 1152 | if (file_exists($finishedDBDir) && is_dir($finishedDBDir)) { |
| 1153 | @rmdir($finishedDBDir); |
| 1154 | } |
| 1155 | |
| 1156 | // Return void |
| 1157 | return; |
| 1158 | |
| 1159 | } |
| 1160 | |
| 1161 | } |
| 1162 |