| 1 |
<?php |
| 2 |
|
| 3 |
if (!defined('ABSPATH')) { |
| 4 |
exit; |
| 5 |
} |
| 6 |
|
| 7 |
/** |
| 8 |
* Unified rebuild health state for view-build and logs-hits rebuild pipelines. |
| 9 |
* |
| 10 |
* Single wp_options row for health gate state (failure count, exponential |
| 11 |
* backoff, trial tokens) and adaptive chunk sizing for hits rebuild. |
| 12 |
* |
| 13 |
* Created for task c632 (Bruno 507 rebuild loop). |
| 14 |
*/ |
| 15 |
class ABJ_404_Solution_RebuildHealthState { |
| 16 |
|
| 17 |
const OPTION_NAME = 'abj404_rebuild_health'; |
| 18 |
const TRIAL_LOCK_OPTION = 'abj404_rebuild_health_trial_lock'; |
| 19 |
const MAX_COOLDOWN_SECONDS = 86400; |
| 20 |
const TRIAL_TTL_SECONDS = 300; |
| 21 |
const FAILURE_THRESHOLD = 3; |
| 22 |
const INITIAL_COOLDOWN_SECONDS = 300; |
| 23 |
const DISK_ERROR_COOLDOWN_SECONDS = 86400; |
| 24 |
const DAILY_MAINTENANCE_RECOVERY_SECONDS = 86400; |
| 25 |
const MAX_CHUNK_SIZE = 100000; |
| 26 |
const MIN_CHUNK_SIZE = 100; |
| 27 |
const CHUNK_GROWTH_FACTOR = 1.5; |
| 28 |
|
| 29 |
/** @var ABJ_404_Solution_Clock */ |
| 30 |
private $clock; |
| 31 |
/** @var ABJ_404_Solution_Logging|null */ |
| 32 |
private $logger; |
| 33 |
/** @var string|null Exact row value acquired by this instance. */ |
| 34 |
private $trialLockValue; |
| 35 |
|
| 36 |
/** |
| 37 |
* @param ABJ_404_Solution_Clock $clock |
| 38 |
* @param ABJ_404_Solution_Logging|null $logger |
| 39 |
*/ |
| 40 |
public function __construct(ABJ_404_Solution_Clock $clock, $logger = null) { |
| 41 |
$this->clock = $clock; |
| 42 |
$this->logger = $logger; |
| 43 |
} |
| 44 |
|
| 45 |
/** @return bool */ |
| 46 |
public function mayStartExpensiveRebuild(): bool { |
| 47 |
$state = $this->readState(); |
| 48 |
if ($state === null) { |
| 49 |
$this->log('warn', 'Rebuild health state is corrupt; refusing expensive rebuild.'); |
| 50 |
return false; |
| 51 |
} |
| 52 |
$gate = $state['gate']; |
| 53 |
$trial = $state['trial']; |
| 54 |
$now = $this->clock->now(); |
| 55 |
if ($this->dailyRecoveryIsActive($trial, $now)) { |
| 56 |
return true; |
| 57 |
} |
| 58 |
$rawNext = $gate['next_allowed_at'] ?? 0; |
| 59 |
$nextAllowed = is_numeric($rawNext) ? intval($rawNext) : 0; |
| 60 |
if ($this->trialIsActive($trial, $now)) { |
| 61 |
return !$this->gateHasOpenFailureWindow($gate); |
| 62 |
} |
| 63 |
return $nextAllowed <= $now; |
| 64 |
} |
| 65 |
|
| 66 |
/** @return bool */ |
| 67 |
public function beginExpensiveRebuildAttempt(): bool { |
| 68 |
if (!$this->mayStartExpensiveRebuild()) { |
| 69 |
return false; |
| 70 |
} |
| 71 |
$state = $this->readState(); |
| 72 |
if ($state === null) { |
| 73 |
return false; |
| 74 |
} |
| 75 |
if ($this->dailyRecoveryIsActive($state['trial'], $this->clock->now())) { |
| 76 |
return true; |
| 77 |
} |
| 78 |
$gate = $state['gate']; |
| 79 |
if (!$this->gateHasOpenFailureWindow($gate)) { |
| 80 |
return true; |
| 81 |
} |
| 82 |
return $this->acquireTrialToken() !== null; |
| 83 |
} |
| 84 |
|
| 85 |
/** @return bool */ |
| 86 |
public function beginDailyMaintenanceRebuildAttempt(): bool { |
| 87 |
if ($this->beginExpensiveRebuildAttempt()) { |
| 88 |
return true; |
| 89 |
} |
| 90 |
$state = $this->readState(); |
| 91 |
if ($state === null) { |
| 92 |
return false; |
| 93 |
} |
| 94 |
$now = $this->clock->now(); |
| 95 |
$rawLastDaily = $state['gate']['last_daily_maintenance_attempt_ts'] ?? 0; |
| 96 |
$lastDaily = is_numeric($rawLastDaily) ? intval($rawLastDaily) : 0; |
| 97 |
if ($lastDaily > 0 && ($now - $lastDaily) < self::DAILY_MAINTENANCE_RECOVERY_SECONDS) { |
| 98 |
return false; |
| 99 |
} |
| 100 |
if ($this->acquireTrialToken() === null) { |
| 101 |
return false; |
| 102 |
} |
| 103 |
$this->mutateState(function (array $state) use ($now): array { |
| 104 |
$state['gate']['last_daily_maintenance_attempt_ts'] = $now; |
| 105 |
$state['trial']['daily_recovery_until'] = $now + self::TRIAL_TTL_SECONDS; |
| 106 |
return $state; |
| 107 |
}); |
| 108 |
return true; |
| 109 |
} |
| 110 |
|
| 111 |
/** Give up the trial lock row. |
| 112 |
* |
| 113 |
* Goes through the same exclusive-row primitive the claim uses rather than |
| 114 |
* delete_option(), so there is one write path to this row instead of two |
| 115 |
* that have to be kept consistent with each other. |
| 116 |
* |
| 117 |
* @return void |
| 118 |
*/ |
| 119 |
private function releaseTrialLock(): void { |
| 120 |
if ($this->trialLockValue === null) { |
| 121 |
return; |
| 122 |
} |
| 123 |
(new ABJ_404_Solution_ExclusiveOptionRow())->releaseIfValueIs(array( |
| 124 |
'optionName' => self::TRIAL_LOCK_OPTION, |
| 125 |
'value' => $this->trialLockValue, |
| 126 |
)); |
| 127 |
$this->trialLockValue = null; |
| 128 |
} |
| 129 |
|
| 130 |
/** @return string|null */ |
| 131 |
public function acquireTrialToken(): ?string { |
| 132 |
$now = $this->clock->now(); |
| 133 |
// Read the row from the table rather than through get_option(), whose |
| 134 |
// per-request cache answers a racing request with its own write, and |
| 135 |
// clear an expired holder conditionally on the exact value read so a |
| 136 |
// fresh holder is never displaced. The claim underneath is a single |
| 137 |
// atomic INSERT: several requests that all saw the same expired trial |
| 138 |
// lock still produce exactly one token. |
| 139 |
$lockRow = new ABJ_404_Solution_ExclusiveOptionRow(); |
| 140 |
$existing = $lockRow->valueOf(self::TRIAL_LOCK_OPTION); |
| 141 |
$existingExpiryPart = explode(':', $existing, 2)[0]; |
| 142 |
$existingExpires = is_numeric($existingExpiryPart) ? intval($existingExpiryPart) : 0; |
| 143 |
if ($existingExpires > 0 && $existingExpires <= $now) { |
| 144 |
$lockRow->releaseIfValueIs(array('optionName' => self::TRIAL_LOCK_OPTION, 'value' => $existing)); |
| 145 |
} |
| 146 |
$claimValue = ABJ_404_Solution_ExclusiveOptionRow::uniqueClaimValue( |
| 147 |
(string)($now + self::TRIAL_TTL_SECONDS) |
| 148 |
); |
| 149 |
$added = $lockRow->claim(array('optionName' => self::TRIAL_LOCK_OPTION, 'value' => $claimValue)); |
| 150 |
if (!$added) { return null; } |
| 151 |
$this->trialLockValue = $claimValue; |
| 152 |
try { |
| 153 |
$token = bin2hex(random_bytes(8)); |
| 154 |
} catch (\Throwable $t) { |
| 155 |
// allow-silent-catch: random_bytes unavailable on some hosts; fallback token is sufficient for trial lock dedup. |
| 156 |
$token = (string)mt_rand() . '_' . (string)$now; |
| 157 |
} |
| 158 |
$this->mutateState(function (array $state) use ($token, $now): array { $state['trial'] = array('token' => $token, 'started_at' => $now, 'ttl' => self::TRIAL_TTL_SECONDS); return $state; }); |
| 159 |
return $token; |
| 160 |
} |
| 161 |
|
| 162 |
/** |
| 163 |
* @param string $msg |
| 164 |
* @param string $class |
| 165 |
* @return void |
| 166 |
*/ |
| 167 |
public function recordFailure(string $msg, string $class = 'unknown'): void { |
| 168 |
$now = $this->clock->now(); |
| 169 |
$this->mutateState(function (array $state) use ($msg, $class, $now): array { |
| 170 |
$gate = $state['gate']; |
| 171 |
$fc = is_numeric($gate['failure_count'] ?? 0) ? intval($gate['failure_count']) : 0; |
| 172 |
$gate['failure_count'] = $fc + 1; |
| 173 |
$gate['last_failure_ts'] = $now; |
| 174 |
$gate['last_failure_msg'] = substr($msg, 0, 500); |
| 175 |
$gate['last_failure_class'] = $class; |
| 176 |
$cd = is_numeric($gate['cooldown_seconds'] ?? self::INITIAL_COOLDOWN_SECONDS) ? intval($gate['cooldown_seconds']) : self::INITIAL_COOLDOWN_SECONDS; |
| 177 |
if ($class === 'disk') { |
| 178 |
$gate['cooldown_seconds'] = self::DISK_ERROR_COOLDOWN_SECONDS; |
| 179 |
$gate['next_allowed_at'] = $now + self::DISK_ERROR_COOLDOWN_SECONDS; |
| 180 |
} elseif (($fc + 1) >= self::FAILURE_THRESHOLD) { |
| 181 |
$gate['next_allowed_at'] = $now + $cd; |
| 182 |
$gate['cooldown_seconds'] = min(self::MAX_COOLDOWN_SECONDS, $cd * 2); |
| 183 |
} |
| 184 |
$state['gate'] = $gate; |
| 185 |
$state['trial'] = array('token' => '', 'started_at' => 0, 'ttl' => self::TRIAL_TTL_SECONDS, 'daily_recovery_until' => 0); |
| 186 |
return $state; |
| 187 |
}); |
| 188 |
$this->releaseTrialLock(); |
| 189 |
$this->log('warn', sprintf('Rebuild health: failure (class=%s). Message: %s', $class, substr($msg, 0, 200))); |
| 190 |
} |
| 191 |
|
| 192 |
/** @return void */ |
| 193 |
public function recordSuccess(): void { |
| 194 |
$now = $this->clock->now(); |
| 195 |
$this->mutateState(function (array $state) use ($now): array { |
| 196 |
$state['gate'] = array('failure_count' => 0, 'next_allowed_at' => 0, 'last_failure_ts' => 0, 'last_failure_msg' => '', 'last_failure_class' => '', 'cooldown_seconds' => self::INITIAL_COOLDOWN_SECONDS, 'last_success_ts' => $now); |
| 197 |
$state['trial'] = array('token' => '', 'started_at' => 0, 'ttl' => self::TRIAL_TTL_SECONDS, 'daily_recovery_until' => 0); |
| 198 |
return $state; |
| 199 |
}); |
| 200 |
$this->releaseTrialLock(); |
| 201 |
} |
| 202 |
|
| 203 |
/** @return void */ |
| 204 |
public function reset(): void { |
| 205 |
if (function_exists('update_option')) { update_option(self::OPTION_NAME, $this->defaultState(), false); } |
| 206 |
$this->releaseTrialLock(); |
| 207 |
} |
| 208 |
|
| 209 |
/** @return array{failure_count: int, last_failure_msg: string, last_failure_class: string, cooldown_seconds: int, next_allowed_at: int}|null */ |
| 210 |
public function getNoticePayload(): ?array { |
| 211 |
$state = $this->readState(); |
| 212 |
if ($state === null) { return array('failure_count' => 0, 'last_failure_msg' => 'Health state corrupt.', 'last_failure_class' => 'unknown', 'cooldown_seconds' => 0, 'next_allowed_at' => 0); } |
| 213 |
$gate = $state['gate']; |
| 214 |
$rawNext = $gate['next_allowed_at'] ?? 0; |
| 215 |
$nextAllowed = is_numeric($rawNext) ? intval($rawNext) : 0; |
| 216 |
if ($nextAllowed <= $this->clock->now()) { return null; } |
| 217 |
$rawFc = $gate['failure_count'] ?? 0; |
| 218 |
$rawMsg = $gate['last_failure_msg'] ?? ''; |
| 219 |
$rawCls = $gate['last_failure_class'] ?? ''; |
| 220 |
$rawCd = $gate['cooldown_seconds'] ?? 0; |
| 221 |
return array('failure_count' => is_numeric($rawFc) ? intval($rawFc) : 0, 'last_failure_msg' => is_string($rawMsg) ? $rawMsg : '', 'last_failure_class' => is_string($rawCls) ? $rawCls : '', 'cooldown_seconds' => is_numeric($rawCd) ? intval($rawCd) : 0, 'next_allowed_at' => $nextAllowed); |
| 222 |
} |
| 223 |
|
| 224 |
/** @param string $errorMessage @return string */ |
| 225 |
public function classifyError(string $errorMessage): string { |
| 226 |
$lower = strtolower($errorMessage); |
| 227 |
foreach (array('507', 'incorrect key file', 'is full', 'table full', 'no space left', 'disk quota exceeded') as $p) { if (strpos($lower, $p) !== false) { return 'disk'; } } |
| 228 |
foreach (array('max_statement_time exceeded', 'query execution was interrupted') as $p) { if (strpos($lower, $p) !== false) { return 'timeout'; } } |
| 229 |
if (strpos($lower, 'illegal mix of collations') !== false) { return 'collation'; } |
| 230 |
return 'unknown'; |
| 231 |
} |
| 232 |
|
| 233 |
/** @param int $idRange @return int */ |
| 234 |
public function getHitsChunkSize(int $idRange): int { |
| 235 |
$state = $this->readState(); |
| 236 |
$current = $state !== null ? ($state['hits_chunk_size']['current'] ?? null) : null; |
| 237 |
if ($current !== null && is_numeric($current)) { return max(self::MIN_CHUNK_SIZE, min(self::MAX_CHUNK_SIZE, intval($current))); } |
| 238 |
$estimated = $idRange <= 0 ? self::MAX_CHUNK_SIZE : max(self::MIN_CHUNK_SIZE, min(self::MAX_CHUNK_SIZE, intval($idRange / 10))); |
| 239 |
$this->mutateState(function (array $state) use ($estimated): array { $state['hits_chunk_size']['current'] = $estimated; return $state; }); |
| 240 |
return $estimated; |
| 241 |
} |
| 242 |
|
| 243 |
/** @param int $size @return void */ |
| 244 |
public function recordHitsChunkSuccess(int $size): void { |
| 245 |
$this->mutateState(function (array $state) use ($size): array { $state['hits_chunk_size']['last_successful'] = $size; if ($state['hits_chunk_size']['current'] === null) { $state['hits_chunk_size']['current'] = $size; } return $state; }); |
| 246 |
} |
| 247 |
|
| 248 |
/** @return void */ |
| 249 |
public function recordHitsChunkFailure(): void { |
| 250 |
$this->mutateState(function (array $state): array { $c = $state['hits_chunk_size']['current'] ?? self::MAX_CHUNK_SIZE; $state['hits_chunk_size']['current'] = intval(max(self::MIN_CHUNK_SIZE, intval(is_numeric($c) ? intval($c) : self::MAX_CHUNK_SIZE) / 2)); return $state; }); |
| 251 |
} |
| 252 |
|
| 253 |
/** @param int $lastChunkSize @return void */ |
| 254 |
public function recordFullRebuildSuccess(int $lastChunkSize): void { |
| 255 |
$this->mutateState(function (array $state) use ($lastChunkSize): array { $state['hits_chunk_size']['current'] = min(self::MAX_CHUNK_SIZE, intval($lastChunkSize * self::CHUNK_GROWTH_FACTOR)); $state['hits_chunk_size']['last_successful'] = $lastChunkSize; return $state; }); |
| 256 |
} |
| 257 |
|
| 258 |
/** |
| 259 |
* @return array<string, array<string, mixed>>|null |
| 260 |
*/ |
| 261 |
public function readState(): ?array { |
| 262 |
if (!function_exists('get_option')) { return $this->defaultState(); } |
| 263 |
$raw = get_option(self::OPTION_NAME, null); |
| 264 |
if ($raw === null || $raw === false) { return $this->defaultState(); } |
| 265 |
if (!is_array($raw) || !isset($raw['gate']) || !is_array($raw['gate'])) { return null; } |
| 266 |
return $this->mergeDefaults($raw); |
| 267 |
} |
| 268 |
|
| 269 |
/** |
| 270 |
* @param callable $mutator |
| 271 |
* @return void |
| 272 |
*/ |
| 273 |
private function mutateState(callable $mutator): void { |
| 274 |
$state = $this->readState(); |
| 275 |
if ($state === null) { $state = $this->defaultState(); } |
| 276 |
$mutated = $mutator($state); |
| 277 |
if (is_array($mutated)) { |
| 278 |
$state = $mutated; |
| 279 |
} |
| 280 |
if (function_exists('update_option')) { update_option(self::OPTION_NAME, $state, false); } |
| 281 |
} |
| 282 |
|
| 283 |
/** |
| 284 |
* @param array<int|string, mixed> $raw |
| 285 |
* @return array<string, array<string, mixed>> |
| 286 |
*/ |
| 287 |
private function mergeDefaults(array $raw): array { |
| 288 |
$d = $this->defaultState(); |
| 289 |
$gate = is_array($raw['gate'] ?? null) ? $raw['gate'] : array(); |
| 290 |
$trial = is_array($raw['trial'] ?? null) ? $raw['trial'] : array(); |
| 291 |
$chunk = is_array($raw['hits_chunk_size'] ?? null) ? $raw['hits_chunk_size'] : array(); |
| 292 |
return array('gate' => array_merge($d['gate'], $gate), 'trial' => array_merge($d['trial'], $trial), 'hits_chunk_size' => array_merge($d['hits_chunk_size'], $chunk)); |
| 293 |
} |
| 294 |
|
| 295 |
/** @return array<string, array<string, mixed>> */ |
| 296 |
private function defaultState(): array { |
| 297 |
return array( |
| 298 |
'gate' => array('failure_count' => 0, 'next_allowed_at' => 0, 'last_failure_ts' => 0, 'last_failure_msg' => '', 'last_failure_class' => '', 'cooldown_seconds' => self::INITIAL_COOLDOWN_SECONDS, 'last_success_ts' => 0), |
| 299 |
'trial' => array('token' => '', 'started_at' => 0, 'ttl' => self::TRIAL_TTL_SECONDS, 'daily_recovery_until' => 0), |
| 300 |
'hits_chunk_size' => array('last_successful' => null, 'current' => null), |
| 301 |
); |
| 302 |
} |
| 303 |
|
| 304 |
/** |
| 305 |
* @param array<string, mixed> $gate |
| 306 |
* @return bool |
| 307 |
*/ |
| 308 |
private function gateHasOpenFailureWindow(array $gate): bool { |
| 309 |
$rawFailureCount = $gate['failure_count'] ?? 0; |
| 310 |
$failureCount = is_numeric($rawFailureCount) ? intval($rawFailureCount) : 0; |
| 311 |
$rawNext = $gate['next_allowed_at'] ?? 0; |
| 312 |
$nextAllowed = is_numeric($rawNext) ? intval($rawNext) : 0; |
| 313 |
$rawLastFailure = $gate['last_failure_ts'] ?? 0; |
| 314 |
$lastFailure = is_numeric($rawLastFailure) ? intval($rawLastFailure) : 0; |
| 315 |
$rawLastSuccess = $gate['last_success_ts'] ?? 0; |
| 316 |
$lastSuccess = is_numeric($rawLastSuccess) ? intval($rawLastSuccess) : 0; |
| 317 |
return $failureCount > 0 || $nextAllowed > 0 || $lastFailure > $lastSuccess; |
| 318 |
} |
| 319 |
|
| 320 |
/** |
| 321 |
* @param array<string, mixed> $trial |
| 322 |
* @param int $now |
| 323 |
* @return bool |
| 324 |
*/ |
| 325 |
private function trialIsActive(array $trial, int $now): bool { |
| 326 |
$rawToken = $trial['token'] ?? ''; |
| 327 |
$trialToken = is_string($rawToken) ? $rawToken : ''; |
| 328 |
$rawStarted = $trial['started_at'] ?? 0; |
| 329 |
$trialStarted = is_numeric($rawStarted) ? intval($rawStarted) : 0; |
| 330 |
$rawTtl = $trial['ttl'] ?? 0; |
| 331 |
$trialTtl = is_numeric($rawTtl) ? intval($rawTtl) : 0; |
| 332 |
return $trialToken !== '' && $trialStarted > 0 && ($now - $trialStarted) < $trialTtl; |
| 333 |
} |
| 334 |
|
| 335 |
/** |
| 336 |
* @param array<string, mixed> $trial |
| 337 |
* @param int $now |
| 338 |
* @return bool |
| 339 |
*/ |
| 340 |
private function dailyRecoveryIsActive(array $trial, int $now): bool { |
| 341 |
$rawUntil = $trial['daily_recovery_until'] ?? 0; |
| 342 |
$until = is_numeric($rawUntil) ? intval($rawUntil) : 0; |
| 343 |
return $until > $now; |
| 344 |
} |
| 345 |
|
| 346 |
/** |
| 347 |
* @param string $level |
| 348 |
* @param string $message |
| 349 |
* @return void |
| 350 |
*/ |
| 351 |
private function log(string $level, string $message): void { |
| 352 |
if ($this->logger === null) { return; } |
| 353 |
if ($level === 'warn' && method_exists($this->logger, 'warn')) { $this->logger->warn($message); } |
| 354 |
elseif (method_exists($this->logger, 'debugMessage')) { $this->logger->debugMessage($message); } |
| 355 |
} |
| 356 |
} |
| 357 |
|
