| 1 |
<?php |
| 2 |
|
| 3 |
/** |
| 4 |
* Class for handling service "PopulateFileList" - Returns file lists |
| 5 |
* Call should already be verified by permissions callback |
| 6 |
*/ |
| 7 |
|
| 8 |
namespace WPSynchro\API; |
| 9 |
|
| 10 |
use WPSynchro\Utilities\DatabaseTables; |
| 11 |
use WPSynchro\Files\PopulateFileListFilterIterator; |
| 12 |
use WPSynchro\Files\PopulateFileListState; |
| 13 |
use WPSynchro\Transport\TransferAccessKey; |
| 14 |
use WPSynchro\Files\FileHelperFunctions; |
| 15 |
use WPSynchro\Transport\TransferFile; |
| 16 |
use WPSynchro\Utilities\ErrorHandler\CustomPHPErrorHandler; |
| 17 |
use WPSynchro\Utilities\CommonFunctions; |
| 18 |
use WPSynchro\Transport\ReturnResult; |
| 19 |
use WPSynchro\Transport\Transfer; |
| 20 |
use WPSynchro\Utilities\SyncTimerList; |
| 21 |
|
| 22 |
class PopulateFileList extends WPSynchroService |
| 23 |
{ |
| 24 |
// ID (generated) |
| 25 |
public $id; |
| 26 |
// |
| 27 |
public $exclusions; |
| 28 |
public $file_list = []; |
| 29 |
public $file_excludes_in_webroot = []; |
| 30 |
public $state = null; |
| 31 |
public $timer = null; |
| 32 |
public $basepath = null; |
| 33 |
// Debugs |
| 34 |
public $debugs = []; |
| 35 |
// Dependencies |
| 36 |
public $errorhandler = null; |
| 37 |
public $common = null; |
| 38 |
|
| 39 |
public function __construct() |
| 40 |
{ |
| 41 |
// Generate a ID to be used for logging and such |
| 42 |
$this->id = uniqid(); |
| 43 |
|
| 44 |
// Start custom error handler |
| 45 |
$this->errorhandler = new CustomPHPErrorHandler(); |
| 46 |
$this->errorhandler->addErrorHandler(); |
| 47 |
$this->errorhandler->instant_callable = [$this, "addLog"]; |
| 48 |
|
| 49 |
// Init timer |
| 50 |
$this->timer = SyncTimerList::getInstance(); |
| 51 |
$this->timer->init(); |
| 52 |
|
| 53 |
// Common functions |
| 54 |
$this->common = new CommonFunctions(); |
| 55 |
} |
| 56 |
|
| 57 |
public function service() |
| 58 |
{ |
| 59 |
// Get transfer object, so we can get data |
| 60 |
$transfer = new Transfer(); |
| 61 |
$transfer->setEncryptionKey(TransferAccessKey::getAccessKey()); |
| 62 |
$transfer->populateFromString($this->getRequestBody()); |
| 63 |
$body = $transfer->getDataObject(); |
| 64 |
|
| 65 |
// Extract parameters |
| 66 |
$section = $body->section; |
| 67 |
$type = $body->type; |
| 68 |
$allotted_time = $body->allotted_time * 0.9; |
| 69 |
$job_id = $body->requestid; |
| 70 |
$this->exclusions = $body->exclusions; |
| 71 |
|
| 72 |
// Initialize and get current state |
| 73 |
$already_running = $this->initPopulate($job_id, $section->id); |
| 74 |
if (!$already_running || $this->state->state == 'completed') { |
| 75 |
// If it is already running or completed, we just return |
| 76 |
$returnresult = new ReturnResult(); |
| 77 |
$returnresult->init(); |
| 78 |
$returnresult->setDataObject(null); |
| 79 |
return $returnresult->echoDataFromServiceAndExit(); |
| 80 |
} |
| 81 |
|
| 82 |
// Get some helpers |
| 83 |
$this->timer->addOtherSyncTimeLimit($allotted_time); |
| 84 |
$this->file_excludes_in_webroot = FileHelperFunctions::getWPFilesInWebrootToExclude(); |
| 85 |
|
| 86 |
// Do some setup |
| 87 |
if ($type == "source") { |
| 88 |
$this->basepath = $section->source_basepath; |
| 89 |
} else { |
| 90 |
$this->basepath = $section->target_basepath; |
| 91 |
} |
| 92 |
|
| 93 |
$section->temp_locations_in_basepath = (array) $section->temp_locations_in_basepath; |
| 94 |
|
| 95 |
// If we are just getting started, set the root entries in database |
| 96 |
if ($this->state->state == 'start') { |
| 97 |
$this->handleStart($section); |
| 98 |
} |
| 99 |
|
| 100 |
// From this point on, we are in running mode, meaning doing expansion of dirs until no more time |
| 101 |
$this->handleRunning(); |
| 102 |
|
| 103 |
// Update state in db |
| 104 |
$this->state->in_progress = false; |
| 105 |
update_option("wpsynchro_filepopulation_current", $this->state, false); |
| 106 |
|
| 107 |
$returnresult = new ReturnResult(); |
| 108 |
$returnresult->init(); |
| 109 |
$returnresult->setDataObject(null); |
| 110 |
|
| 111 |
// Restore error handler |
| 112 |
$this->errorhandler->removeErrorHandler(); |
| 113 |
|
| 114 |
// Add debugs of this run to db |
| 115 |
$this->addLogs("DEBUG", $this->debugs); |
| 116 |
|
| 117 |
return $returnresult->echoDataFromServiceAndExit(); |
| 118 |
} |
| 119 |
|
| 120 |
/** |
| 121 |
* Get status on populate and initialize if needed |
| 122 |
*/ |
| 123 |
public function initPopulate($job_id, $section_id) |
| 124 |
{ |
| 125 |
// Figure out where we are in the process |
| 126 |
$this->state = get_option('wpsynchro_filepopulation_current'); |
| 127 |
|
| 128 |
// Needed for compatibility with pre 1.6.0 |
| 129 |
if (is_object($this->state) && get_class($this->state) !== PopulateFileListState::class) { |
| 130 |
$this->state = null; |
| 131 |
} |
| 132 |
|
| 133 |
// Check if we are currently running, with the right job and section |
| 134 |
if (is_object($this->state) && $this->state->job_id == $job_id && $this->state->section_id == $section_id) { |
| 135 |
$this->debugs[] = "File population state exist with section_id: " . $section_id . " - Checking if it is in progress or if we should run"; |
| 136 |
if ($this->state->in_progress) { |
| 137 |
// Return that we should not run |
| 138 |
$this->debugs[] = "Section id: " . $section_id . " is in progress - Aborting"; |
| 139 |
return false; |
| 140 |
} else { |
| 141 |
// Set it to running |
| 142 |
$this->debugs[] = "Section id: " . $section_id . " is not running"; |
| 143 |
if ($this->state->state !== "completed") { |
| 144 |
$this->debugs[] = "Section id: " . $section_id . " is not running and not completed, so we are starting"; |
| 145 |
$this->state->in_progress = true; |
| 146 |
update_option("wpsynchro_filepopulation_current", $this->state, false); |
| 147 |
} |
| 148 |
} |
| 149 |
} else { |
| 150 |
// Does not exist or has wrong job_id or section_id, so create new and clear database |
| 151 |
$this->debugs[] = "Section id: " . $section_id . " has no known state, so we start from scratch"; |
| 152 |
$this->state = new PopulateFileListState(); |
| 153 |
$this->state->job_id = $job_id; |
| 154 |
$this->state->section_id = $section_id; |
| 155 |
$this->state->in_progress = true; |
| 156 |
update_option("wpsynchro_filepopulation_current", $this->state, false); |
| 157 |
update_option("wpsynchro_filepopulation_current_download_id", 0, false); |
| 158 |
update_option("wpsynchro_filepopulation_problems", [], false); |
| 159 |
|
| 160 |
// Delete/create database table |
| 161 |
$database_tables = new DatabaseTables(); |
| 162 |
if (!$database_tables->createFilePopulationTable()) { |
| 163 |
$this->addLog("ERROR", __("Could not create database table needed for file population on the source site - This is normally because database user does not have access to create tables on the database.", "wpsynchro")); |
| 164 |
} |
| 165 |
} |
| 166 |
|
| 167 |
return true; |
| 168 |
} |
| 169 |
|
| 170 |
/** |
| 171 |
* Handle running phase of section population |
| 172 |
*/ |
| 173 |
public function handleRunning() |
| 174 |
{ |
| 175 |
$this->debugs[] = "Start running"; |
| 176 |
|
| 177 |
// Get dirs that need expanding |
| 178 |
global $wpdb; |
| 179 |
|
| 180 |
// Run until we have run 3 seconds and then do not start new expand's |
| 181 |
while ($this->timer->getElapsedOverallTimer() < 10) { |
| 182 |
$dir_to_expand = $wpdb->get_row("select * from " . $wpdb->prefix . "wpsynchro_file_population_list where is_expanded=0 and is_dir=1 order by id limit 1"); |
| 183 |
if ($dir_to_expand == null) { |
| 184 |
// No more dirs to expand, so we are complete |
| 185 |
$this->debugs[] = "Got no more dirs to expand, so setting section to completed"; |
| 186 |
$this->state->state = "completed"; |
| 187 |
return; |
| 188 |
} |
| 189 |
$this->debugs[] = "Expanding dir: " . $this->basepath . $dir_to_expand->source_file; |
| 190 |
|
| 191 |
// Set current item in state, so we can continue from it later |
| 192 |
$resume = false; |
| 193 |
if ($this->state->current_dir_state->id == 0) { |
| 194 |
$this->state->current_dir_state->id = $dir_to_expand->id; |
| 195 |
} else { |
| 196 |
$resume = true; |
| 197 |
} |
| 198 |
|
| 199 |
// Get files/dirs in dir |
| 200 |
$filter_iterator = $this->getPathIterator($this->basepath . $dir_to_expand->source_file); |
| 201 |
if ($filter_iterator === false) { |
| 202 |
return; |
| 203 |
} |
| 204 |
$fileobj_list = []; |
| 205 |
$is_completed = true; |
| 206 |
$check_timeout_timer = microtime(true); |
| 207 |
foreach ($filter_iterator as $fileinfo) { |
| 208 |
if ($fileinfo->isDot()) { |
| 209 |
continue; |
| 210 |
} |
| 211 |
|
| 212 |
// Check if we should continue last session |
| 213 |
if ($resume) { |
| 214 |
$path = $fileinfo->getPathname(); |
| 215 |
if ($path == $this->state->current_dir_state->current_path) { |
| 216 |
$resume = false; |
| 217 |
} |
| 218 |
continue; |
| 219 |
} |
| 220 |
|
| 221 |
// Get pathname and set it in state, so we can continue, if we have to break out |
| 222 |
$pathname = $fileinfo->getPathname(); |
| 223 |
$this->state->current_dir_state->current_path = $pathname; |
| 224 |
|
| 225 |
if (!$fileinfo->isReadable()) { |
| 226 |
continue; |
| 227 |
} |
| 228 |
|
| 229 |
// Get data on this file |
| 230 |
$path = $this->common->fixPath($pathname); |
| 231 |
|
| 232 |
// Get info on file |
| 233 |
$fileobj_list[] = $this->getFileObject($path, ($fileinfo->isDir() ? 0 : $fileinfo->getSize()), $fileinfo->isDir()); |
| 234 |
$this->state->files_found++; |
| 235 |
|
| 236 |
// Update database for every X files, so status can pick it up |
| 237 |
if ($this->state->files_found % 1000 == 0) { |
| 238 |
update_option("wpsynchro_filepopulation_current", $this->state, false); |
| 239 |
} |
| 240 |
|
| 241 |
// check that file list does not contain more than X files - If so, break out, so we can write to db |
| 242 |
if (count($fileobj_list) > 990) { |
| 243 |
$is_completed = false; |
| 244 |
break; |
| 245 |
} |
| 246 |
|
| 247 |
// check every 2 seconds, if we need to break out because of time, to make sure we can also write to db before going back |
| 248 |
if ((microtime(true) - $check_timeout_timer) > 2) { |
| 249 |
// Check if we have less than X seconds back and abort if so, completing whatever state we got so far |
| 250 |
if ($this->timer->getRemainingSyncTime() < 10) { |
| 251 |
$is_completed = false; |
| 252 |
break; |
| 253 |
} |
| 254 |
$check_timeout_timer = microtime(true); |
| 255 |
} |
| 256 |
} |
| 257 |
|
| 258 |
// If completed, mark the dir as expanded |
| 259 |
if ($is_completed) { |
| 260 |
$this->setPathExpanded($dir_to_expand->id); |
| 261 |
} |
| 262 |
|
| 263 |
// Insert data to db |
| 264 |
$this->insertPathsToDB($fileobj_list); |
| 265 |
|
| 266 |
// Reset current dir, to prevent it continuing |
| 267 |
if ($is_completed) { |
| 268 |
$this->state->resetCurrentDirState(); |
| 269 |
} |
| 270 |
} |
| 271 |
} |
| 272 |
|
| 273 |
/** |
| 274 |
* Get file/dir iterator on single dir |
| 275 |
*/ |
| 276 |
public function getPathIterator($path) |
| 277 |
{ |
| 278 |
try { |
| 279 |
$dir_iterator = new \DirectoryIterator($path); |
| 280 |
$filter_iterator = new PopulateFileListFilterIterator($dir_iterator); |
| 281 |
$filter_iterator::$FILTERS = $this->exclusions; |
| 282 |
$filter_iterator::$common = $this->common; |
| 283 |
$filter_iterator::$file_excludes = $this->file_excludes_in_webroot; |
| 284 |
return $filter_iterator; |
| 285 |
} catch (\Throwable $e) { // For PHP 7 |
| 286 |
$this->addLog("ERROR", "Error during file population on " . get_home_url() . ": Can't read path: " . $path . ". Reason can be that path does not exist anymore or a permission issue."); |
| 287 |
} catch (\Exception $e) { // For PHP 5 |
| 288 |
$this->addLog("ERROR", "Error during file population on " . get_home_url() . ": Can't read path: " . $path . ". Reason can be that path does not exist anymore or a permission issue."); |
| 289 |
} |
| 290 |
|
| 291 |
return false; |
| 292 |
} |
| 293 |
|
| 294 |
/** |
| 295 |
* Handle start phase of section population |
| 296 |
*/ |
| 297 |
public function handleStart($section) |
| 298 |
{ |
| 299 |
// If we have a preset to move all files, iterate through the file location. |
| 300 |
if ($section->type == "sync_preset_all_files") { |
| 301 |
$this->debugs[] = "Start new with preset all files: " . $this->basepath; |
| 302 |
// If preset is set, populate locations in basepath with all content of web root |
| 303 |
$section->temp_locations_in_basepath = []; |
| 304 |
$filter_iterator = $this->getPathIterator($this->basepath); |
| 305 |
|
| 306 |
if ($filter_iterator === false) { |
| 307 |
return; |
| 308 |
} |
| 309 |
|
| 310 |
foreach ($filter_iterator as $fileinfo) { |
| 311 |
if ($fileinfo->isDot()) { |
| 312 |
continue; |
| 313 |
} |
| 314 |
|
| 315 |
$section->temp_locations_in_basepath["/" . $fileinfo->getFilename()] = true; |
| 316 |
} |
| 317 |
$this->debugs[] = "All files preset generated list of basepaths: " . print_r($section->temp_locations_in_basepath, true); |
| 318 |
} |
| 319 |
|
| 320 |
// Check what work we need to do |
| 321 |
$paths = []; |
| 322 |
if (count($section->temp_locations_in_basepath) > 0) { |
| 323 |
foreach ($section->temp_locations_in_basepath as $relativepath => $whatever) { |
| 324 |
$paths[] = trailingslashit($this->basepath) . trim($relativepath, "/"); |
| 325 |
} |
| 326 |
} else { |
| 327 |
$paths[] = $this->basepath; |
| 328 |
} |
| 329 |
|
| 330 |
$this->debugs[] = "All files preset list of paths: " . print_r($paths, true); |
| 331 |
|
| 332 |
$fileobj_list = []; |
| 333 |
foreach ($paths as $path) { |
| 334 |
$path = $this->common->fixPath($path); |
| 335 |
if (file_exists($path)) { |
| 336 |
$fileobj_list[] = $this->getFileObject($path, (is_dir($path) ? 0 : filesize($path)), is_dir($path)); |
| 337 |
$this->debugs[] = "All files preset - Found path: " . $path; |
| 338 |
$this->state->files_found++; |
| 339 |
} |
| 340 |
} |
| 341 |
|
| 342 |
// Add filelist to database, so we can start digging in |
| 343 |
$this->insertPathsToDB($fileobj_list); |
| 344 |
|
| 345 |
// Change state to running |
| 346 |
$this->state->state = 'running'; |
| 347 |
} |
| 348 |
|
| 349 |
/** |
| 350 |
* Get standard file structure from data |
| 351 |
*/ |
| 352 |
public function getFileObject($name, $size, $is_dir, $hash = null) |
| 353 |
{ |
| 354 |
|
| 355 |
$file_tmp = new TransferFile(); |
| 356 |
$file_tmp->source_file = str_replace($this->basepath, "", $name); |
| 357 |
$file_tmp->size = $size; |
| 358 |
$file_tmp->is_dir = $is_dir; |
| 359 |
|
| 360 |
if ($hash == null) { |
| 361 |
if ($file_tmp->is_dir) { |
| 362 |
$file_tmp->hash = null; |
| 363 |
} else { |
| 364 |
if ($file_tmp->size == 0) { |
| 365 |
$file_tmp->hash = 'd41d8cd98f00b204e9800998ecf8427e'; |
| 366 |
} else { |
| 367 |
if (file_exists($name) && is_readable($name)) { |
| 368 |
$file_tmp->hash = md5_file($name); |
| 369 |
} else { |
| 370 |
$file_tmp->hash = "file_not_exist"; |
| 371 |
} |
| 372 |
} |
| 373 |
} |
| 374 |
} else { |
| 375 |
$file_tmp->hash = $hash; |
| 376 |
} |
| 377 |
|
| 378 |
return $file_tmp; |
| 379 |
} |
| 380 |
|
| 381 |
/** |
| 382 |
* Insert path to database |
| 383 |
*/ |
| 384 |
public function insertPathsToDB($pathlist) |
| 385 |
{ |
| 386 |
global $wpdb; |
| 387 |
$insert_query_part = "INSERT INTO " . $wpdb->prefix . "wpsynchro_file_population_list (source_file, hash, is_expanded, is_dir, size) VALUES "; |
| 388 |
$insert_value_part_arr = []; |
| 389 |
$insert_counter = 0; |
| 390 |
$insert_total_counter = 0; |
| 391 |
|
| 392 |
foreach ($pathlist as $path) { |
| 393 |
$insert_counter++; |
| 394 |
$insert_total_counter++; |
| 395 |
if ($insert_counter > 995) { |
| 396 |
$wpdb->query($insert_query_part . " " . implode(",", $insert_value_part_arr)); |
| 397 |
$insert_counter = 0; |
| 398 |
$insert_value_part_arr = []; |
| 399 |
} |
| 400 |
|
| 401 |
// Add value part |
| 402 |
$insert_value_part_arr[] = $wpdb->prepare( |
| 403 |
"(%s,%s,%d,%d,%d)", |
| 404 |
$path->source_file, |
| 405 |
$path->hash, |
| 406 |
($path->is_dir ? 0 : 1), |
| 407 |
$path->is_dir, |
| 408 |
$path->size |
| 409 |
); |
| 410 |
} |
| 411 |
if ($insert_counter > 0) { |
| 412 |
$wpdb->query($insert_query_part . " " . implode(",", $insert_value_part_arr)); |
| 413 |
} |
| 414 |
} |
| 415 |
|
| 416 |
/** |
| 417 |
* Set path as expanded in db |
| 418 |
*/ |
| 419 |
public function setPathExpanded($id) |
| 420 |
{ |
| 421 |
global $wpdb; |
| 422 |
$wpdb->update( |
| 423 |
$wpdb->prefix . "wpsynchro_file_population_list", |
| 424 |
[ |
| 425 |
'is_expanded' => 1, |
| 426 |
], |
| 427 |
['id' => $id], |
| 428 |
[ |
| 429 |
'%d' |
| 430 |
], |
| 431 |
['%d'] |
| 432 |
); |
| 433 |
} |
| 434 |
|
| 435 |
/** |
| 436 |
* Add single log |
| 437 |
*/ |
| 438 |
public function addLog($type, $msg) |
| 439 |
{ |
| 440 |
return $this->addLogs($type, [$msg]); |
| 441 |
} |
| 442 |
|
| 443 |
/** |
| 444 |
* Add error/debug/other message to database, so it can be picked up by populate status |
| 445 |
*/ |
| 446 |
public function addLogs($type, $msgs = []) |
| 447 |
{ |
| 448 |
if (empty($msgs)) { |
| 449 |
return; |
| 450 |
} |
| 451 |
|
| 452 |
$log = get_option("wpsynchro_filepopulation_problems"); |
| 453 |
if (!is_array($log)) { |
| 454 |
$log = []; |
| 455 |
} |
| 456 |
|
| 457 |
$hostname = parse_url(get_home_url(), \PHP_URL_HOST); |
| 458 |
|
| 459 |
foreach ($msgs as $msg) { |
| 460 |
if (!isset($log[$type])) { |
| 461 |
$log[$type] = []; |
| 462 |
} |
| 463 |
$log[$type][] = "[FILE POPULATION " . $this->id . " " . $hostname . "] " . $msg; |
| 464 |
} |
| 465 |
|
| 466 |
update_option("wpsynchro_filepopulation_problems", $log, false); |
| 467 |
} |
| 468 |
} |
| 469 |
|
