| 1 |
<?php |
| 2 |
|
| 3 |
namespace Dudlewebs\WPMCS\s3\Aws\S3; |
| 4 |
|
| 5 |
use Dudlewebs\WPMCS\s3\Aws; |
| 6 |
use Dudlewebs\WPMCS\s3\Aws\CommandInterface; |
| 7 |
use Dudlewebs\WPMCS\s3\Aws\Exception\AwsException; |
| 8 |
use Dudlewebs\WPMCS\s3\Aws\MetricsBuilder; |
| 9 |
use Dudlewebs\WPMCS\s3\GuzzleHttp\Promise; |
| 10 |
use Dudlewebs\WPMCS\s3\GuzzleHttp\Promise\PromiseInterface; |
| 11 |
use Dudlewebs\WPMCS\s3\GuzzleHttp\Promise\PromisorInterface; |
| 12 |
use Iterator; |
| 13 |
/** |
| 14 |
* Transfers files from the local filesystem to S3 or from S3 to the local |
| 15 |
* filesystem. |
| 16 |
* |
| 17 |
* This class does not support copying from the local filesystem to somewhere |
| 18 |
* else on the local filesystem or from one S3 bucket to another. |
| 19 |
*/ |
| 20 |
class Transfer implements PromisorInterface |
| 21 |
{ |
| 22 |
private $client; |
| 23 |
private $promise; |
| 24 |
private $source; |
| 25 |
private $sourceMetadata; |
| 26 |
private $destination; |
| 27 |
private $concurrency; |
| 28 |
private $mupThreshold; |
| 29 |
private $before; |
| 30 |
private $after; |
| 31 |
private $s3Args = []; |
| 32 |
private $addContentMD5; |
| 33 |
/** |
| 34 |
* When providing the $source argument, you may provide a string referencing |
| 35 |
* the path to a directory on disk to upload, an s3 scheme URI that contains |
| 36 |
* the bucket and key (e.g., "s3://bucket/key"), or an \Iterator object |
| 37 |
* that yields strings containing filenames that are the path to a file on |
| 38 |
* disk or an s3 scheme URI. The bucket portion of the s3 URI may be an S3 |
| 39 |
* access point ARN. The "/key" portion of an s3 URI is optional. |
| 40 |
* |
| 41 |
* When providing an iterator for the $source argument, you must also |
| 42 |
* provide a 'base_dir' key value pair in the $options argument. |
| 43 |
* |
| 44 |
* The $dest argument can be the path to a directory on disk or an s3 |
| 45 |
* scheme URI (e.g., "s3://bucket/key"). |
| 46 |
* |
| 47 |
* The options array can contain the following key value pairs: |
| 48 |
* |
| 49 |
* - base_dir: (string) Base directory of the source, if $source is an |
| 50 |
* iterator. If the $source option is not an array, then this option is |
| 51 |
* ignored. |
| 52 |
* - before: (callable) A callback to invoke before each transfer. The |
| 53 |
* callback accepts a single argument: Aws\CommandInterface $command. |
| 54 |
* The provided command will be either a GetObject, PutObject, |
| 55 |
* InitiateMultipartUpload, or UploadPart command. |
| 56 |
* - after: (callable) A callback to invoke after each transfer promise is fulfilled. |
| 57 |
* The function is invoked with three arguments: the fulfillment value, the index |
| 58 |
* position from the iterable list of the promise, and the aggregate |
| 59 |
* promise that manages all the promises. The aggregate promise may |
| 60 |
* be resolved from within the callback to short-circuit the promise. |
| 61 |
* - mup_threshold: (int) Size in bytes in which a multipart upload should |
| 62 |
* be used instead of PutObject. Defaults to 20971520 (20 MB). |
| 63 |
* - concurrency: (int, default=5) Number of files to upload concurrently. |
| 64 |
* The ideal concurrency value will vary based on the number of files |
| 65 |
* being uploaded and the average size of each file. Generally speaking, |
| 66 |
* smaller files benefit from a higher concurrency while larger files |
| 67 |
* will not. |
| 68 |
* - debug: (bool) Set to true to print out debug information for |
| 69 |
* transfers. Set to an fopen() resource to write to a specific stream |
| 70 |
* rather than writing to STDOUT. |
| 71 |
* |
| 72 |
* @param S3ClientInterface $client Client used for transfers. |
| 73 |
* @param string|Iterator $source Where the files are transferred from. |
| 74 |
* @param string $dest Where the files are transferred to. |
| 75 |
* @param array $options Hash of options. |
| 76 |
*/ |
| 77 |
public function __construct(S3ClientInterface $client, $source, $dest, array $options = []) |
| 78 |
{ |
| 79 |
$this->client = $client; |
| 80 |
// Prepare the destination. |
| 81 |
$this->destination = $this->prepareTarget($dest); |
| 82 |
if ($this->destination['scheme'] === 's3') { |
| 83 |
$this->s3Args = $this->getS3Args($this->destination['path']); |
| 84 |
} |
| 85 |
// Prepare the source. |
| 86 |
if (\is_string($source)) { |
| 87 |
$this->sourceMetadata = $this->prepareTarget($source); |
| 88 |
$this->source = $source; |
| 89 |
} elseif ($source instanceof Iterator) { |
| 90 |
if (empty($options['base_dir'])) { |
| 91 |
throw new \InvalidArgumentException('You must provide the source' . ' argument as a string or provide the "base_dir" option.'); |
| 92 |
} |
| 93 |
$this->sourceMetadata = $this->prepareTarget($options['base_dir']); |
| 94 |
$this->source = $source; |
| 95 |
} else { |
| 96 |
throw new \InvalidArgumentException('source must be the path to a ' . 'directory or an iterator that yields file names.'); |
| 97 |
} |
| 98 |
// Validate schemes. |
| 99 |
if ($this->sourceMetadata['scheme'] === $this->destination['scheme']) { |
| 100 |
throw new \InvalidArgumentException("You cannot copy from" . " {$this->sourceMetadata['scheme']} to" . " {$this->destination['scheme']}."); |
| 101 |
} |
| 102 |
// Handle multipart-related options. |
| 103 |
$this->concurrency = isset($options['concurrency']) ? $options['concurrency'] : MultipartUploader::DEFAULT_CONCURRENCY; |
| 104 |
$this->mupThreshold = isset($options['mup_threshold']) ? $options['mup_threshold'] : 16777216; |
| 105 |
if ($this->mupThreshold < MultipartUploader::PART_MIN_SIZE) { |
| 106 |
throw new \InvalidArgumentException('mup_threshold must be >= 5MB'); |
| 107 |
} |
| 108 |
// Handle "before" callback option. |
| 109 |
if (isset($options['before'])) { |
| 110 |
$this->before = $options['before']; |
| 111 |
if (!\is_callable($this->before)) { |
| 112 |
throw new \InvalidArgumentException('before must be a callable.'); |
| 113 |
} |
| 114 |
} |
| 115 |
// Handle "after" callback option. |
| 116 |
if (isset($options['after'])) { |
| 117 |
$this->after = $options['after']; |
| 118 |
if (!\is_callable($this->after)) { |
| 119 |
throw new \InvalidArgumentException('after must be a callable.'); |
| 120 |
} |
| 121 |
} |
| 122 |
// Handle "debug" option. |
| 123 |
if (isset($options['debug'])) { |
| 124 |
if ($options['debug'] === \true) { |
| 125 |
$options['debug'] = \fopen('php://output', 'w'); |
| 126 |
} |
| 127 |
if (\is_resource($options['debug'])) { |
| 128 |
$this->addDebugToBefore($options['debug']); |
| 129 |
} |
| 130 |
} |
| 131 |
// Handle "add_content_md5" option. |
| 132 |
$this->addContentMD5 = isset($options['add_content_md5']) && $options['add_content_md5'] === \true; |
| 133 |
MetricsBuilder::appendMetricsCaptureMiddleware($this->client->getHandlerList(), MetricsBuilder::S3_TRANSFER); |
| 134 |
} |
| 135 |
/** |
| 136 |
* Transfers the files. |
| 137 |
* |
| 138 |
* @return PromiseInterface |
| 139 |
*/ |
| 140 |
public function promise() : PromiseInterface |
| 141 |
{ |
| 142 |
// If the promise has been created, just return it. |
| 143 |
if (!$this->promise) { |
| 144 |
// Create an upload/download promise for the transfer. |
| 145 |
$this->promise = $this->sourceMetadata['scheme'] === 'file' ? $this->createUploadPromise() : $this->createDownloadPromise(); |
| 146 |
} |
| 147 |
return $this->promise; |
| 148 |
} |
| 149 |
/** |
| 150 |
* Transfers the files synchronously. |
| 151 |
*/ |
| 152 |
public function transfer() |
| 153 |
{ |
| 154 |
$this->promise()->wait(); |
| 155 |
} |
| 156 |
private function prepareTarget($targetPath) |
| 157 |
{ |
| 158 |
$target = ['path' => $this->normalizePath($targetPath), 'scheme' => $this->determineScheme($targetPath)]; |
| 159 |
if ($target['scheme'] !== 's3' && $target['scheme'] !== 'file') { |
| 160 |
throw new \InvalidArgumentException('Scheme must be "s3" or "file".'); |
| 161 |
} |
| 162 |
return $target; |
| 163 |
} |
| 164 |
/** |
| 165 |
* Creates an array that contains Bucket and Key by parsing the filename. |
| 166 |
* |
| 167 |
* @param string $path Path to parse. |
| 168 |
* |
| 169 |
* @return array |
| 170 |
*/ |
| 171 |
private function getS3Args($path) |
| 172 |
{ |
| 173 |
$parts = \explode('/', \str_replace('s3://', '', $path), 2); |
| 174 |
$args = ['Bucket' => $parts[0]]; |
| 175 |
if (isset($parts[1])) { |
| 176 |
$args['Key'] = $parts[1]; |
| 177 |
} |
| 178 |
return $args; |
| 179 |
} |
| 180 |
/** |
| 181 |
* Parses the scheme from a filename. |
| 182 |
* |
| 183 |
* @param string $path Path to parse. |
| 184 |
* |
| 185 |
* @return string |
| 186 |
*/ |
| 187 |
private function determineScheme($path) |
| 188 |
{ |
| 189 |
return !\strpos($path, '://') ? 'file' : \explode('://', $path)[0]; |
| 190 |
} |
| 191 |
/** |
| 192 |
* Normalize a path so that it has UNIX-style directory separators and no trailing / |
| 193 |
* |
| 194 |
* @param string $path |
| 195 |
* |
| 196 |
* @return string |
| 197 |
*/ |
| 198 |
private function normalizePath($path) |
| 199 |
{ |
| 200 |
return \rtrim(\str_replace('\\', '/', $path), '/'); |
| 201 |
} |
| 202 |
private function resolvesOutsideTargetDirectory($sink, $objectKey) |
| 203 |
{ |
| 204 |
$resolved = []; |
| 205 |
$sections = \explode('/', $sink); |
| 206 |
$targetSectionsLength = \count(\explode('/', $objectKey)); |
| 207 |
$targetSections = \array_slice($sections, -($targetSectionsLength + 1)); |
| 208 |
$targetDirectory = $targetSections[0]; |
| 209 |
foreach ($targetSections as $section) { |
| 210 |
if ($section === '.' || $section === '') { |
| 211 |
continue; |
| 212 |
} |
| 213 |
if ($section === '..') { |
| 214 |
\array_pop($resolved); |
| 215 |
if (empty($resolved) || $resolved[0] !== $targetDirectory) { |
| 216 |
return \true; |
| 217 |
} |
| 218 |
} else { |
| 219 |
$resolved[] = $section; |
| 220 |
} |
| 221 |
} |
| 222 |
return \false; |
| 223 |
} |
| 224 |
private function createDownloadPromise() |
| 225 |
{ |
| 226 |
$parts = $this->getS3Args($this->sourceMetadata['path']); |
| 227 |
$prefix = "s3://{$parts['Bucket']}/" . (isset($parts['Key']) ? $parts['Key'] . '/' : ''); |
| 228 |
$commands = []; |
| 229 |
foreach ($this->getDownloadsIterator() as $object) { |
| 230 |
// Prepare the sink. |
| 231 |
$objectKey = \preg_replace('/^' . \preg_quote($prefix, '/') . '/', '', $object); |
| 232 |
$sink = $this->destination['path'] . '/' . $objectKey; |
| 233 |
$command = $this->client->getCommand('GetObject', $this->getS3Args($object) + ['@http' => ['sink' => $sink]]); |
| 234 |
if ($this->resolvesOutsideTargetDirectory($sink, $objectKey)) { |
| 235 |
throw new AwsException('Cannot download key ' . $objectKey . ', its relative path resolves outside the' . ' parent directory', $command); |
| 236 |
} |
| 237 |
// Create the directory if needed. |
| 238 |
$dir = \dirname($sink); |
| 239 |
if (!\is_dir($dir) && !\mkdir($dir, 0777, \true)) { |
| 240 |
throw new \RuntimeException("Could not create dir: {$dir}"); |
| 241 |
} |
| 242 |
// Create the command. |
| 243 |
$commands[] = $command; |
| 244 |
} |
| 245 |
// Create a GetObject command pool and return the promise. |
| 246 |
return (new Aws\CommandPool($this->client, $commands, ['concurrency' => $this->concurrency, 'before' => $this->before, 'fulfill' => $this->after, 'rejected' => function ($reason, $idx, Promise\PromiseInterface $p) { |
| 247 |
$p->reject($reason); |
| 248 |
}]))->promise(); |
| 249 |
} |
| 250 |
private function createUploadPromise() |
| 251 |
{ |
| 252 |
// Map each file into a promise that performs the actual transfer. |
| 253 |
$files = \Dudlewebs\WPMCS\s3\Aws\map($this->getUploadsIterator(), function ($file) { |
| 254 |
return \filesize($file) >= $this->mupThreshold ? $this->uploadMultipart($file) : $this->upload($file); |
| 255 |
}); |
| 256 |
// Create an EachPromise, that will concurrently handle the upload |
| 257 |
// operations' yielded promises from the iterator. |
| 258 |
return Promise\Each::ofLimitAll($files, $this->concurrency, $this->after); |
| 259 |
} |
| 260 |
/** @return Iterator */ |
| 261 |
private function getUploadsIterator() |
| 262 |
{ |
| 263 |
if (\is_string($this->source)) { |
| 264 |
return Aws\filter(Aws\recursive_dir_iterator($this->sourceMetadata['path']), function ($file) { |
| 265 |
return !\is_dir($file); |
| 266 |
}); |
| 267 |
} |
| 268 |
return $this->source; |
| 269 |
} |
| 270 |
/** @return Iterator */ |
| 271 |
private function getDownloadsIterator() |
| 272 |
{ |
| 273 |
if (\is_string($this->source)) { |
| 274 |
$listArgs = $this->getS3Args($this->sourceMetadata['path']); |
| 275 |
if (isset($listArgs['Key'])) { |
| 276 |
$listArgs['Prefix'] = $listArgs['Key'] . '/'; |
| 277 |
unset($listArgs['Key']); |
| 278 |
} |
| 279 |
$files = $this->client->getPaginator('ListObjects', $listArgs)->search('Contents[].Key'); |
| 280 |
$files = Aws\map($files, function ($key) use($listArgs) { |
| 281 |
return "s3://{$listArgs['Bucket']}/{$key}"; |
| 282 |
}); |
| 283 |
return Aws\filter($files, function ($key) { |
| 284 |
return \substr($key, -1, 1) !== '/'; |
| 285 |
}); |
| 286 |
} |
| 287 |
return $this->source; |
| 288 |
} |
| 289 |
private function upload($filename) |
| 290 |
{ |
| 291 |
$args = $this->s3Args; |
| 292 |
$args['SourceFile'] = $filename; |
| 293 |
$args['Key'] = $this->createS3Key($filename); |
| 294 |
$args['AddContentMD5'] = $this->addContentMD5; |
| 295 |
$command = $this->client->getCommand('PutObject', $args); |
| 296 |
$this->before and \call_user_func($this->before, $command); |
| 297 |
return $this->client->executeAsync($command); |
| 298 |
} |
| 299 |
private function uploadMultipart($filename) |
| 300 |
{ |
| 301 |
$args = $this->s3Args; |
| 302 |
$args['Key'] = $this->createS3Key($filename); |
| 303 |
$filename = $filename instanceof \SplFileInfo ? $filename->getPathname() : $filename; |
| 304 |
return (new MultipartUploader($this->client, $filename, ['bucket' => $args['Bucket'], 'key' => $args['Key'], 'before_initiate' => $this->before, 'before_upload' => $this->before, 'before_complete' => $this->before, 'concurrency' => $this->concurrency, 'add_content_md5' => $this->addContentMD5]))->promise(); |
| 305 |
} |
| 306 |
private function createS3Key($filename) |
| 307 |
{ |
| 308 |
$filename = $this->normalizePath($filename); |
| 309 |
$relative_file_path = \ltrim(\preg_replace('#^' . \preg_quote($this->sourceMetadata['path']) . '#', '', $filename), '/\\'); |
| 310 |
if (isset($this->s3Args['Key'])) { |
| 311 |
return \rtrim($this->s3Args['Key'], '/') . '/' . $relative_file_path; |
| 312 |
} |
| 313 |
return $relative_file_path; |
| 314 |
} |
| 315 |
private function addDebugToBefore($debug) |
| 316 |
{ |
| 317 |
$before = $this->before; |
| 318 |
$sourcePath = $this->sourceMetadata['path']; |
| 319 |
$s3Args = $this->s3Args; |
| 320 |
$this->before = static function (CommandInterface $command) use($before, $debug, $sourcePath, $s3Args) { |
| 321 |
// Call the composed before function. |
| 322 |
$before and $before($command); |
| 323 |
// Determine the source and dest values based on operation. |
| 324 |
switch ($operation = $command->getName()) { |
| 325 |
case 'GetObject': |
| 326 |
$source = "s3://{$command['Bucket']}/{$command['Key']}"; |
| 327 |
$dest = $command['@http']['sink']; |
| 328 |
break; |
| 329 |
case 'PutObject': |
| 330 |
$source = $command['SourceFile']; |
| 331 |
$dest = "s3://{$command['Bucket']}/{$command['Key']}"; |
| 332 |
break; |
| 333 |
case 'UploadPart': |
| 334 |
$part = $command['PartNumber']; |
| 335 |
case 'CreateMultipartUpload': |
| 336 |
case 'CompleteMultipartUpload': |
| 337 |
$sourceKey = $command['Key']; |
| 338 |
if (isset($s3Args['Key']) && \strpos($sourceKey, $s3Args['Key']) === 0) { |
| 339 |
$sourceKey = \substr($sourceKey, \strlen($s3Args['Key']) + 1); |
| 340 |
} |
| 341 |
$source = "{$sourcePath}/{$sourceKey}"; |
| 342 |
$dest = "s3://{$command['Bucket']}/{$command['Key']}"; |
| 343 |
break; |
| 344 |
default: |
| 345 |
throw new \UnexpectedValueException("Transfer encountered an unexpected operation: {$operation}."); |
| 346 |
} |
| 347 |
// Print the debugging message. |
| 348 |
$context = \sprintf('%s -> %s (%s)', $source, $dest, $operation); |
| 349 |
if (isset($part)) { |
| 350 |
$context .= " : Part={$part}"; |
| 351 |
} |
| 352 |
\fwrite($debug, "Transferring {$context}\n"); |
| 353 |
}; |
| 354 |
} |
| 355 |
} |
| 356 |
|
