| 1 |
<?php |
| 2 |
|
| 3 |
declare (strict_types=1); |
| 4 |
namespace WindPress\WindPress\Core\Scanner; |
| 5 |
|
| 6 |
use WindPressDeps\Symfony\Component\Finder\Glob; |
| 7 |
/** |
| 8 |
* Resumable file discovery and bounded content reads. |
| 9 |
*/ |
| 10 |
class FileScanner |
| 11 |
{ |
| 12 |
private const CURSOR_TTL = 900; |
| 13 |
public static function scan_local(string $root, array $patterns, array $exclude_patterns = [], $cursor = \false, bool $batched = \true, array $limits = []): array |
| 14 |
{ |
| 15 |
$configuration = self::local_configuration($root, $patterns, $exclude_patterns); |
| 16 |
[$root, $patterns, $exclude_patterns, $allowed_roots] = $configuration; |
| 17 |
$scope = self::configuration_scope($configuration); |
| 18 |
return self::scan($scope, $cursor, $batched, $limits, static function () use ($root, $patterns, $exclude_patterns, $allowed_roots): array { |
| 19 |
$roots = []; |
| 20 |
foreach ($patterns as $pattern) { |
| 21 |
$prefix = substr($pattern, 0, strcspn($pattern, '*?[{')); |
| 22 |
$separator = strrpos($prefix, '/'); |
| 23 |
$directory = $separator === \false ? $root : $root . '/' . substr($prefix, 0, $separator); |
| 24 |
if (is_dir($directory)) { |
| 25 |
$roots[$directory] = $directory; |
| 26 |
} |
| 27 |
} |
| 28 |
sort($roots, \SORT_STRING); |
| 29 |
$narrowed_roots = []; |
| 30 |
foreach ($roots as $directory) { |
| 31 |
foreach ($narrowed_roots as $parent) { |
| 32 |
if (self::within($directory, $parent)) { |
| 33 |
continue 2; |
| 34 |
} |
| 35 |
} |
| 36 |
$narrowed_roots[] = $directory; |
| 37 |
} |
| 38 |
return ['kind' => 'local', 'root' => $root, 'roots' => array_reverse($narrowed_roots), 'allowed_roots' => $allowed_roots, 'patterns' => array_map([self::class, 'pattern_regex'], $patterns), 'exclude_patterns' => array_map([self::class, 'pattern_regex'], $exclude_patterns), 'stack' => [], 'seen_directories' => [], 'seen_files' => [], 'done' => \false]; |
| 39 |
}); |
| 40 |
} |
| 41 |
public static function local_scope(string $root, array $patterns, array $exclude_patterns = []): string |
| 42 |
{ |
| 43 |
return self::configuration_scope(self::local_configuration($root, $patterns, $exclude_patterns)); |
| 44 |
} |
| 45 |
/** |
| 46 |
* Snapshot provider-selected paths once, preserving Finder extension hooks. |
| 47 |
* The factory returns records with a path and any source metadata to retain. |
| 48 |
*/ |
| 49 |
public static function scan_files(string $scope, callable $factory, $cursor = \false, array $limits = []): array |
| 50 |
{ |
| 51 |
return self::scan('provider:' . $scope, $cursor, \true, $limits, static function () use ($factory): array { |
| 52 |
$files = []; |
| 53 |
foreach ($factory() as $file) { |
| 54 |
if (!is_array($file) || !isset($file['path']) || !is_string($file['path'])) { |
| 55 |
throw new \InvalidArgumentException(__('A scanner returned an invalid file path.', 'windpress')); |
| 56 |
} |
| 57 |
$real_path = realpath($file['path']); |
| 58 |
if ($real_path === \false || !is_file($real_path)) { |
| 59 |
throw new \RuntimeException(__('A source file no longer exists: ', 'windpress') . $file['path']); |
| 60 |
} |
| 61 |
$file['path'] = $real_path; |
| 62 |
$file['name'] = $file['name'] ?? basename($real_path); |
| 63 |
if (!isset($files[$real_path])) { |
| 64 |
$files[$real_path] = $file; |
| 65 |
} |
| 66 |
} |
| 67 |
ksort($files, \SORT_STRING); |
| 68 |
return ['kind' => 'files', 'files' => array_values($files), 'offset' => 0, 'done' => \false]; |
| 69 |
}); |
| 70 |
} |
| 71 |
private static function local_configuration(string $root, array $patterns, array $exclude_patterns): array |
| 72 |
{ |
| 73 |
$root = realpath($root); |
| 74 |
if ($root === \false || !is_dir($root)) { |
| 75 |
throw new \InvalidArgumentException(__('The source directory does not exist.', 'windpress')); |
| 76 |
} |
| 77 |
$patterns = self::normalize_patterns($patterns, $root); |
| 78 |
$exclude_patterns = self::normalize_patterns($exclude_patterns, $root); |
| 79 |
$allowed_roots = apply_filters('f!windpress/core/scanner/file:allowed_roots', [$root], $root); |
| 80 |
if (!is_array($allowed_roots)) { |
| 81 |
throw new \InvalidArgumentException(__('Allowed source directories must be an array.', 'windpress')); |
| 82 |
} |
| 83 |
foreach ($allowed_roots as $allowed_root) { |
| 84 |
if (!is_string($allowed_root)) { |
| 85 |
throw new \InvalidArgumentException(__('Allowed source directories must contain paths.', 'windpress')); |
| 86 |
} |
| 87 |
} |
| 88 |
$allowed_roots = array_values(array_unique(array_filter(array_map('realpath', $allowed_roots)))); |
| 89 |
sort($allowed_roots, \SORT_STRING); |
| 90 |
return [$root, $patterns, $exclude_patterns, $allowed_roots]; |
| 91 |
} |
| 92 |
private static function configuration_scope(array $configuration): string |
| 93 |
{ |
| 94 |
return hash('sha256', serialize(['files-v1', $configuration])); |
| 95 |
} |
| 96 |
private static function scan(string $scope, $cursor, bool $batched, array $limits, callable $factory): array |
| 97 |
{ |
| 98 |
$limits = array_replace(['files' => 100, 'bytes' => 2 * 1024 * 1024, 'seconds' => 1.0], $limits); |
| 99 |
$limits = apply_filters('f!windpress/core/scanner/file:limits', $limits, $scope); |
| 100 |
if (!is_array($limits) || !isset($limits['files'], $limits['bytes'], $limits['seconds']) || !is_int($limits['files']) || !is_int($limits['bytes']) || !is_numeric($limits['seconds']) || !is_finite((float) $limits['seconds']) || $limits['files'] < 1 || $limits['bytes'] < 1 || $limits['bytes'] === \PHP_INT_MAX || $limits['seconds'] <= 0) { |
| 101 |
throw new \InvalidArgumentException(__('File scan limits must be positive.', 'windpress')); |
| 102 |
} |
| 103 |
$page = 0; |
| 104 |
$key = ''; |
| 105 |
if ($cursor !== \false && $cursor !== null) { |
| 106 |
if (!$batched || !is_string($cursor) || !preg_match('/^fs_([a-f0-9]{32})_([0-9]+)$/D', $cursor, $matches)) { |
| 107 |
throw new \InvalidArgumentException(__('The file scan cursor is invalid.', 'windpress')); |
| 108 |
} |
| 109 |
$key = 'windpress_scan_' . get_current_user_id() . '_' . $matches[1]; |
| 110 |
$page = (int) $matches[2]; |
| 111 |
} elseif ($batched) { |
| 112 |
$key = 'windpress_scan_' . get_current_user_id() . '_' . bin2hex(random_bytes(16)); |
| 113 |
} |
| 114 |
$lease = $batched ? \WindPress\WindPress\Core\Scanner\ScanLock::acquire($key) : null; |
| 115 |
if ($batched && $lease === null) { |
| 116 |
throw new \RuntimeException(__('This file scan is already processing a batch. Retry the same cursor.', 'windpress'), 409); |
| 117 |
} |
| 118 |
try { |
| 119 |
return self::scan_page($scope, $cursor, $batched, $limits, $factory, $key, $page, $lease); |
| 120 |
} finally { |
| 121 |
if ($lease !== null) { |
| 122 |
\WindPress\WindPress\Core\Scanner\ScanLock::release($key, $lease); |
| 123 |
} |
| 124 |
} |
| 125 |
} |
| 126 |
private static function scan_page(string $scope, $cursor, bool $batched, array $limits, callable $factory, string $key, int $page, ?string $lease): array |
| 127 |
{ |
| 128 |
if ($cursor !== \false && $cursor !== null) { |
| 129 |
$job = get_transient($key); |
| 130 |
if (!is_array($job) || $job['scope'] !== $scope) { |
| 131 |
throw new \InvalidArgumentException(__('The file scan expired or its sources changed. Start a new scan.', 'windpress')); |
| 132 |
} |
| 133 |
if ($page === $job['previous_page']) { |
| 134 |
return $job['previous_response']; |
| 135 |
} |
| 136 |
if ($page !== $job['page']) { |
| 137 |
throw new \InvalidArgumentException(__('The file scan cursor is out of sequence. Start a new scan.', 'windpress')); |
| 138 |
} |
| 139 |
$state = $job['state']; |
| 140 |
} else { |
| 141 |
$state = $factory(); |
| 142 |
} |
| 143 |
$contents = []; |
| 144 |
$bytes = 0; |
| 145 |
$deadline = $batched ? microtime(\true) + $limits['seconds'] : \PHP_FLOAT_MAX; |
| 146 |
while (!$state['done'] && (!$batched || count($contents) < $limits['files'] && microtime(\true) < $deadline)) { |
| 147 |
if (!isset($state['pending'])) { |
| 148 |
$state['pending'] = self::next_file($state, $deadline); |
| 149 |
} |
| 150 |
if ($state['pending'] === null) { |
| 151 |
unset($state['pending']); |
| 152 |
break; |
| 153 |
} |
| 154 |
$file = $state['pending']; |
| 155 |
$path = $file['path']; |
| 156 |
if ($state['kind'] === 'local') { |
| 157 |
$path = self::allowed_path($state, $path); |
| 158 |
} |
| 159 |
clearstatcache(\true, $path); |
| 160 |
if (!is_file($path) || !is_readable($path)) { |
| 161 |
throw new \RuntimeException(__('A source file cannot be read: ', 'windpress') . $file['name']); |
| 162 |
} |
| 163 |
$size = @filesize($path); |
| 164 |
if ($size === \false) { |
| 165 |
throw new \RuntimeException(__('The source file size could not be read: ', 'windpress') . $file['name']); |
| 166 |
} |
| 167 |
if ($batched && $size > $limits['bytes']) { |
| 168 |
throw new \RuntimeException(__('A source file exceeds the scan byte limit: ', 'windpress') . $file['name']); |
| 169 |
} |
| 170 |
if ($batched && $bytes + $size > $limits['bytes']) { |
| 171 |
break; |
| 172 |
} |
| 173 |
$content = $batched ? @file_get_contents($path, \false, null, 0, $limits['bytes'] + 1) : @file_get_contents($path); |
| 174 |
if ($content === \false) { |
| 175 |
throw new \RuntimeException(__('A source file could not be read: ', 'windpress') . $file['name']); |
| 176 |
} |
| 177 |
if ($batched && strlen($content) > $limits['bytes']) { |
| 178 |
throw new \RuntimeException(__('A source file exceeds the scan byte limit: ', 'windpress') . $file['name']); |
| 179 |
} |
| 180 |
if ($batched && $bytes + strlen($content) > $limits['bytes']) { |
| 181 |
break; |
| 182 |
} |
| 183 |
$bytes += strlen($content); |
| 184 |
unset($file['path'], $state['pending']); |
| 185 |
$file['source_id'] = 'file:' . hash('sha256', $path); |
| 186 |
$file['content'] = $content; |
| 187 |
$contents[] = $file; |
| 188 |
} |
| 189 |
$next_batch = \false; |
| 190 |
if ($batched && !$state['done']) { |
| 191 |
$next_batch = 'fs_' . substr($key, -32) . '_' . ($page + 1); |
| 192 |
} |
| 193 |
$response = ['contents' => $contents, 'metadata' => ['next_batch' => $next_batch, 'scanned_files' => count($contents), 'scanned_bytes' => $bytes]]; |
| 194 |
if ($batched && ($next_batch !== \false || $cursor !== \false && $cursor !== null)) { |
| 195 |
if ($lease === null || !\WindPress\WindPress\Core\Scanner\ScanLock::is_owner($key, $lease)) { |
| 196 |
throw new \RuntimeException(__('The file scan lease expired. Retry the same cursor.', 'windpress'), 409); |
| 197 |
} |
| 198 |
$job = ['scope' => $scope, 'page' => $page + 1, 'state' => $state['done'] ? [] : $state, 'previous_page' => $page, 'previous_response' => $response]; |
| 199 |
if (!set_transient($key, $job, self::CURSOR_TTL)) { |
| 200 |
throw new \RuntimeException(__('The file scan continuation could not be saved.', 'windpress')); |
| 201 |
} |
| 202 |
} |
| 203 |
return $response; |
| 204 |
} |
| 205 |
private static function next_file(array &$state, float $deadline): ?array |
| 206 |
{ |
| 207 |
if ($state['kind'] === 'files') { |
| 208 |
if ($state['offset'] >= count($state['files'])) { |
| 209 |
$state['done'] = \true; |
| 210 |
return null; |
| 211 |
} |
| 212 |
return $state['files'][$state['offset']++]; |
| 213 |
} |
| 214 |
while (microtime(\true) < $deadline) { |
| 215 |
if ($state['stack'] === []) { |
| 216 |
if ($state['roots'] === []) { |
| 217 |
$state['done'] = \true; |
| 218 |
return null; |
| 219 |
} |
| 220 |
self::open_directory($state, array_pop($state['roots'])); |
| 221 |
continue; |
| 222 |
} |
| 223 |
$index = count($state['stack']) - 1; |
| 224 |
$frame =& $state['stack'][$index]; |
| 225 |
if ($frame['offset'] >= count($frame['entries'])) { |
| 226 |
unset($frame); |
| 227 |
array_pop($state['stack']); |
| 228 |
continue; |
| 229 |
} |
| 230 |
$name = $frame['entries'][$frame['offset']++]; |
| 231 |
$path = $frame['path'] . '/' . $name; |
| 232 |
unset($frame); |
| 233 |
// Preserve Finder's default hidden-file and VCS-directory exclusions. |
| 234 |
if ($name[0] === '.' || in_array($name, ['CVS', '_darcs', '_svn'], \true)) { |
| 235 |
continue; |
| 236 |
} |
| 237 |
$relative_path = substr($path, strlen($state['root']) + 1); |
| 238 |
if (self::matches($relative_path, $state['exclude_patterns']) || is_dir($path) && self::matches($relative_path . '/', $state['exclude_patterns'])) { |
| 239 |
continue; |
| 240 |
} |
| 241 |
if (is_dir($path)) { |
| 242 |
self::open_directory($state, $path); |
| 243 |
continue; |
| 244 |
} |
| 245 |
if (!self::matches($relative_path, $state['patterns'])) { |
| 246 |
continue; |
| 247 |
} |
| 248 |
$real_path = self::allowed_path($state, $path); |
| 249 |
if (isset($state['seen_files'][$real_path])) { |
| 250 |
continue; |
| 251 |
} |
| 252 |
$state['seen_files'][$real_path] = \true; |
| 253 |
return ['path' => $real_path, 'name' => $name, 'relative_path' => $relative_path]; |
| 254 |
} |
| 255 |
return null; |
| 256 |
} |
| 257 |
private static function open_directory(array &$state, string $path): void |
| 258 |
{ |
| 259 |
$relative_path = substr($path, strlen($state['root']) + 1); |
| 260 |
if (self::matches($relative_path, $state['exclude_patterns']) || self::matches($relative_path . '/', $state['exclude_patterns'])) { |
| 261 |
return; |
| 262 |
} |
| 263 |
$real_path = self::allowed_path($state, $path); |
| 264 |
if (isset($state['seen_directories'][$path])) { |
| 265 |
return; |
| 266 |
} |
| 267 |
foreach ($state['stack'] as $frame) { |
| 268 |
if ($frame['real_path'] === $real_path) { |
| 269 |
return; |
| 270 |
} |
| 271 |
} |
| 272 |
$entries = @scandir($path); |
| 273 |
if ($entries === \false) { |
| 274 |
throw new \RuntimeException(__('A source directory could not be read: ', 'windpress') . $relative_path); |
| 275 |
} |
| 276 |
$state['seen_directories'][$path] = \true; |
| 277 |
$state['stack'][] = ['path' => $path, 'real_path' => $real_path, 'entries' => $entries, 'offset' => 0]; |
| 278 |
} |
| 279 |
private static function allowed_path(array $state, string $path): string |
| 280 |
{ |
| 281 |
$real_path = realpath($path); |
| 282 |
if ($real_path === \false) { |
| 283 |
throw new \RuntimeException(__('A source path no longer exists: ', 'windpress') . substr($path, strlen($state['root']) + 1)); |
| 284 |
} |
| 285 |
foreach ($state['allowed_roots'] as $root) { |
| 286 |
if (self::within($real_path, $root)) { |
| 287 |
return $real_path; |
| 288 |
} |
| 289 |
} |
| 290 |
throw new \RuntimeException(__('A source symlink points outside the allowed directories: ', 'windpress') . substr($path, strlen($state['root']) + 1)); |
| 291 |
} |
| 292 |
private static function within(string $path, string $root): bool |
| 293 |
{ |
| 294 |
return $path === $root || strpos($path, rtrim($root, '/') . '/') === 0; |
| 295 |
} |
| 296 |
private static function normalize_patterns(array $patterns, string $root): array |
| 297 |
{ |
| 298 |
$normalized = []; |
| 299 |
foreach ($patterns as $pattern) { |
| 300 |
if (!is_string($pattern) || $pattern === '' || strlen($pattern) > 4096 || strpos($pattern, "\x00") !== \false) { |
| 301 |
throw new \InvalidArgumentException(__('Source patterns must be nonempty strings up to 4096 bytes.', 'windpress')); |
| 302 |
} |
| 303 |
$pattern = str_replace('\\', '/', $pattern); |
| 304 |
if ($pattern[0] === '/' || preg_match('#(^|/)\.\.(/|$)|^[a-zA-Z]+:#', $pattern)) { |
| 305 |
throw new \InvalidArgumentException(__('Source patterns must stay within the source directory.', 'windpress')); |
| 306 |
} |
| 307 |
while (strpos($pattern, './') === 0) { |
| 308 |
$pattern = substr($pattern, 2); |
| 309 |
} |
| 310 |
$pattern = rtrim($pattern, '/'); |
| 311 |
if ($pattern === '') { |
| 312 |
throw new \InvalidArgumentException(__('Source patterns must not be empty.', 'windpress')); |
| 313 |
} |
| 314 |
if (is_dir($root . '/' . $pattern)) { |
| 315 |
$pattern .= '/**'; |
| 316 |
} |
| 317 |
self::pattern_regex($pattern); |
| 318 |
$normalized[$pattern] = $pattern; |
| 319 |
} |
| 320 |
sort($normalized, \SORT_STRING); |
| 321 |
return $normalized; |
| 322 |
} |
| 323 |
private static function pattern_regex(string $pattern): string |
| 324 |
{ |
| 325 |
// Symfony recognizes recursive ** after a slash, including at pattern start here. |
| 326 |
$regex = Glob::toRegex('windpress/' . $pattern); |
| 327 |
if (@preg_match($regex, '') === \false) { |
| 328 |
throw new \InvalidArgumentException(__('A source pattern is invalid.', 'windpress')); |
| 329 |
} |
| 330 |
return $regex; |
| 331 |
} |
| 332 |
private static function matches(string $path, array $patterns): bool |
| 333 |
{ |
| 334 |
foreach ($patterns as $pattern) { |
| 335 |
if (preg_match($pattern, 'windpress/' . $path) === 1) { |
| 336 |
return \true; |
| 337 |
} |
| 338 |
} |
| 339 |
return \false; |
| 340 |
} |
| 341 |
} |
| 342 |
|
