Cli.php
4 days ago
Import.php
4 days ago
MailChimp.php
4 months ago
MailChimpDataMapper.php
4 months ago
index.php
3 years ago
Import.php
847 lines
| 1 | <?php // phpcs:ignore SlevomatCodingStandard.TypeHints.DeclareStrictTypes.DeclareStrictTypesMissing |
| 2 | |
| 3 | namespace MailPoet\Subscribers\ImportExport\Import; |
| 4 | |
| 5 | if (!defined('ABSPATH')) exit; |
| 6 | |
| 7 | |
| 8 | use MailPoet\CustomFields\CustomFieldsRepository; |
| 9 | use MailPoet\Entities\CustomFieldEntity; |
| 10 | use MailPoet\Entities\SubscriberCustomFieldEntity; |
| 11 | use MailPoet\Entities\SubscriberEntity; |
| 12 | use MailPoet\Entities\SubscriberSegmentEntity; |
| 13 | use MailPoet\Entities\SubscriberTagEntity; |
| 14 | use MailPoet\Newsletter\Options\NewsletterOptionsRepository; |
| 15 | use MailPoet\Segments\WP; |
| 16 | use MailPoet\Services\Validator; |
| 17 | use MailPoet\Subscribers\ImportExport\ImportExportFactory; |
| 18 | use MailPoet\Subscribers\ImportExport\ImportExportRepository; |
| 19 | use MailPoet\Subscribers\Source; |
| 20 | use MailPoet\Subscribers\SubscribersRepository; |
| 21 | use MailPoet\Tags\TagRepository; |
| 22 | use MailPoet\Util\DateConverter; |
| 23 | use MailPoet\Util\Helpers; |
| 24 | use MailPoet\Util\Security; |
| 25 | use MailPoet\WP\Functions as WPFunctions; |
| 26 | use MailPoetVendor\Carbon\Carbon; |
| 27 | |
| 28 | class Import { |
| 29 | /** @var array */ |
| 30 | public $subscribersData; |
| 31 | /** @var array */ |
| 32 | public $segmentsIds; |
| 33 | /** @var string[] */ |
| 34 | public $tags; |
| 35 | /** @var string */ |
| 36 | public $newSubscribersStatus; |
| 37 | /** @var string */ |
| 38 | public $existingSubscribersStatus; |
| 39 | /** @var bool */ |
| 40 | public $updateSubscribers; |
| 41 | /** @var array */ |
| 42 | public $subscribersFields; |
| 43 | /** @var array */ |
| 44 | public $subscribersCustomFields; |
| 45 | /** @var int */ |
| 46 | public $subscribersCount; |
| 47 | /** @var Carbon */ |
| 48 | public $createdAt; |
| 49 | /** @var Carbon */ |
| 50 | public $updatedAt; |
| 51 | /** @var array<string, mixed> */ |
| 52 | public $requiredSubscribersFields; |
| 53 | const DB_QUERY_CHUNK_SIZE = 100; |
| 54 | // Matches the subscribers.tracking_consent_method column width. |
| 55 | const TRACKING_CONSENT_METHOD_MAX_LENGTH = 40; |
| 56 | const STATUS_DONT_UPDATE = 'dont_update'; |
| 57 | |
| 58 | public const ACTION_CREATE = 'create'; |
| 59 | public const ACTION_UPDATE = 'update'; |
| 60 | |
| 61 | /** @var WP */ |
| 62 | private $wpSegment; |
| 63 | |
| 64 | /** @var CustomFieldsRepository */ |
| 65 | private $customFieldsRepository; |
| 66 | |
| 67 | /** @var ImportExportRepository */ |
| 68 | private $importExportRepository; |
| 69 | |
| 70 | /** @var NewsletterOptionsRepository */ |
| 71 | private $newsletterOptionsRepository; |
| 72 | |
| 73 | /** @var SubscribersRepository */ |
| 74 | private $subscriberRepository; |
| 75 | |
| 76 | /** @var TagRepository */ |
| 77 | private $tagRepository; |
| 78 | |
| 79 | /** @var Validator */ |
| 80 | private $validator; |
| 81 | |
| 82 | public function __construct( |
| 83 | WP $wpSegment, |
| 84 | CustomFieldsRepository $customFieldsRepository, |
| 85 | ImportExportRepository $importExportRepository, |
| 86 | NewsletterOptionsRepository $newsletterOptionsRepository, |
| 87 | SubscribersRepository $subscriberRepository, |
| 88 | TagRepository $tagRepository, |
| 89 | Validator $validator, |
| 90 | array $data |
| 91 | ) { |
| 92 | $this->wpSegment = $wpSegment; |
| 93 | $this->customFieldsRepository = $customFieldsRepository; |
| 94 | $this->importExportRepository = $importExportRepository; |
| 95 | $this->newsletterOptionsRepository = $newsletterOptionsRepository; |
| 96 | $this->subscriberRepository = $subscriberRepository; |
| 97 | $this->tagRepository = $tagRepository; |
| 98 | $this->validator = $validator; |
| 99 | $this->validateImportData($data); |
| 100 | $this->subscribersData = $this->transformSubscribersData( |
| 101 | $data['subscribers'], |
| 102 | $data['columns'] |
| 103 | ); |
| 104 | $this->segmentsIds = $data['segments']; |
| 105 | $this->tags = $data['tags']; |
| 106 | $this->newSubscribersStatus = $data['newSubscribersStatus']; |
| 107 | $this->existingSubscribersStatus = $data['existingSubscribersStatus']; |
| 108 | $this->updateSubscribers = $data['updateSubscribers']; |
| 109 | $this->subscribersFields = $this->getSubscribersFields( |
| 110 | array_keys($data['columns']) |
| 111 | ); |
| 112 | $this->subscribersCustomFields = $this->getCustomSubscribersFields( |
| 113 | array_keys($data['columns']) |
| 114 | ); |
| 115 | $this->subscribersCount = (reset($this->subscribersData) === false) ? 0 : count(reset($this->subscribersData)); |
| 116 | $this->createdAt = Carbon::now()->millisecond(0); |
| 117 | $this->updatedAt = Carbon::createFromTimestamp(WPFunctions::get()->currentTime('timestamp', true) + 1); |
| 118 | $this->requiredSubscribersFields = [ |
| 119 | 'status' => SubscriberEntity::STATUS_SUBSCRIBED, |
| 120 | 'first_name' => '', |
| 121 | 'last_name' => '', |
| 122 | 'created_at' => $this->createdAt, |
| 123 | ]; |
| 124 | } |
| 125 | |
| 126 | public function validateImportData(array $data): void { |
| 127 | $requiredDataFields = [ |
| 128 | 'subscribers', |
| 129 | 'columns', |
| 130 | 'segments', |
| 131 | 'timestamp', |
| 132 | 'newSubscribersStatus', |
| 133 | 'existingSubscribersStatus', |
| 134 | 'updateSubscribers', |
| 135 | 'tags', |
| 136 | ]; |
| 137 | // 1. data should contain all required fields |
| 138 | // 2. column names should only contain alphanumeric & underscore characters |
| 139 | if ( |
| 140 | count(array_intersect_key(array_flip($requiredDataFields), $data)) !== count($requiredDataFields) || |
| 141 | preg_grep('/[^a-zA-Z0-9_]/', array_keys($data['columns'])) |
| 142 | ) { |
| 143 | throw new \Exception(__('Missing or invalid import data.', 'mailpoet')); |
| 144 | } |
| 145 | } |
| 146 | |
| 147 | /** |
| 148 | * @return array{created: int, updated:int, segments: array, added_to_segment_with_welcome_notification:bool} |
| 149 | * @throws \Exception |
| 150 | */ |
| 151 | public function process(): array { |
| 152 | // validate data based on field validation rules |
| 153 | $subscribersData = $this->validateSubscribersData($this->subscribersData); |
| 154 | if (!$subscribersData) { |
| 155 | throw new \Exception(__('No valid subscribers were found.', 'mailpoet')); |
| 156 | } |
| 157 | // permanently trash deleted subscribers |
| 158 | $this->deleteExistingTrashedSubscribers($subscribersData); |
| 159 | |
| 160 | // split subscribers into "existing" and "new" and free up memory |
| 161 | $existingSubscribers = $newSubscribers = [ |
| 162 | 'data' => [], |
| 163 | 'fields' => $this->subscribersFields, |
| 164 | ]; |
| 165 | list($existingSubscribers['data'], $newSubscribers['data'], $wpUsers) = |
| 166 | $this->splitSubscribersData($subscribersData); |
| 167 | $subscribersData = null; |
| 168 | |
| 169 | // create or update subscribers |
| 170 | $createdSubscribers = $updatedSubscribers = []; |
| 171 | try { |
| 172 | if ($newSubscribers['data']) { |
| 173 | // add, if required, missing required fields to new subscribers |
| 174 | $newSubscribers = $this->addMissingRequiredFields($newSubscribers); |
| 175 | $newSubscribers = $this->setSubscriptionStatusToDefault($newSubscribers, $this->newSubscribersStatus); |
| 176 | $newSubscribers = $this->setSource($newSubscribers); |
| 177 | $newSubscribers = $this->setLinkToken($newSubscribers); |
| 178 | $newSubscribers = $this->applyTrackingConsentToNewSubscribers($newSubscribers); |
| 179 | $createdSubscribers = |
| 180 | $this->createOrUpdateSubscribers( |
| 181 | self::ACTION_CREATE, |
| 182 | $newSubscribers, |
| 183 | $this->subscribersCustomFields |
| 184 | ); |
| 185 | } |
| 186 | |
| 187 | $updateExistingSubscribersStatus = false; |
| 188 | |
| 189 | if ($existingSubscribers['data']) { |
| 190 | $allowedStatuses = [ |
| 191 | SubscriberEntity::STATUS_SUBSCRIBED, |
| 192 | SubscriberEntity::STATUS_UNSUBSCRIBED, |
| 193 | SubscriberEntity::STATUS_INACTIVE, |
| 194 | ]; |
| 195 | if (in_array($this->existingSubscribersStatus, $allowedStatuses, true)) { |
| 196 | $updateExistingSubscribersStatus = true; |
| 197 | $existingSubscribers = $this->addField($existingSubscribers, 'status', $this->existingSubscribersStatus); |
| 198 | } |
| 199 | if ($this->updateSubscribers) { |
| 200 | // Update existing subscribers' info (first_name, last_name etc.) |
| 201 | // as well as status (optionally) if the status column was added above |
| 202 | $updatedSubscribers = |
| 203 | $this->createOrUpdateSubscribers( |
| 204 | self::ACTION_UPDATE, |
| 205 | $this->stripTrackingConsentColumns($existingSubscribers), |
| 206 | $this->subscribersCustomFields |
| 207 | ); |
| 208 | $this->updateTrackingConsent($existingSubscribers); |
| 209 | if ($wpUsers) { |
| 210 | $this->synchronizeWPUsers($wpUsers); |
| 211 | } |
| 212 | } elseif ($updateExistingSubscribersStatus) { |
| 213 | // Only update existing subscribers' status |
| 214 | // For this we need to remove all other fields except email and status |
| 215 | $existingSubscribers['fields'] = array_intersect($existingSubscribers['fields'], ['email', 'status']); |
| 216 | $existingSubscribers['data'] = array_intersect_key($existingSubscribers['data'], array_flip(['email', 'status'])); |
| 217 | $updatedSubscribers = |
| 218 | $this->createOrUpdateSubscribers( |
| 219 | self::ACTION_UPDATE, |
| 220 | $existingSubscribers |
| 221 | ); |
| 222 | } |
| 223 | } |
| 224 | } catch (\Exception $e) { |
| 225 | throw new \Exception(__('Unable to save imported subscribers.', 'mailpoet')); |
| 226 | } |
| 227 | |
| 228 | // check if any subscribers were added to segments that have welcome notifications configured |
| 229 | $importFactory = new ImportExportFactory('import'); |
| 230 | $segments = $importFactory->getSegments(); |
| 231 | $welcomeNotificationsInSegments = |
| 232 | ($createdSubscribers || $updatedSubscribers) ? |
| 233 | $this->newsletterOptionsRepository->findWelcomeNotificationsForSegments($this->segmentsIds) : |
| 234 | false; |
| 235 | |
| 236 | return [ |
| 237 | 'created' => is_array($createdSubscribers) ? count($createdSubscribers) : 0, |
| 238 | 'updated' => is_array($updatedSubscribers) ? count($updatedSubscribers) : 0, |
| 239 | 'segments' => $segments, |
| 240 | 'added_to_segment_with_welcome_notification' => |
| 241 | ($welcomeNotificationsInSegments) ? true : false, |
| 242 | ]; |
| 243 | } |
| 244 | |
| 245 | /** |
| 246 | * @param array $subscribersData |
| 247 | * @return false|array |
| 248 | */ |
| 249 | public function validateSubscribersData(array $subscribersData) { |
| 250 | $invalidRecords = []; |
| 251 | foreach ($subscribersData as $column => &$data) { |
| 252 | if ($column === 'email') { |
| 253 | $data = array_map( |
| 254 | function($index, $email) use(&$invalidRecords) { |
| 255 | if (!$this->validator->validateNonRoleEmail($email)) { |
| 256 | $invalidRecords[] = $index; |
| 257 | } |
| 258 | return strtolower($email); |
| 259 | }, |
| 260 | array_keys($data), |
| 261 | $data |
| 262 | ); |
| 263 | } |
| 264 | if (in_array($column, ['created_at', 'confirmed_at'], true)) { |
| 265 | $data = $this->validateDateTime($data, $invalidRecords); |
| 266 | } |
| 267 | if (in_array($column, ['confirmed_ip', 'subscribed_ip'], true)) { |
| 268 | $data = array_map( |
| 269 | function($index, $ip) { |
| 270 | if (!filter_var($ip, FILTER_VALIDATE_IP)) { |
| 271 | // if invalid or empty, we allow the import but remove the IP |
| 272 | return null; |
| 273 | } |
| 274 | return $ip; |
| 275 | }, |
| 276 | array_keys($data), |
| 277 | $data |
| 278 | ); |
| 279 | } |
| 280 | if ($column === 'tracking_consent') { |
| 281 | $validStates = [ |
| 282 | SubscriberEntity::TRACKING_CONSENT_GRANTED, |
| 283 | SubscriberEntity::TRACKING_CONSENT_DENIED, |
| 284 | SubscriberEntity::TRACKING_CONSENT_UNKNOWN, |
| 285 | ]; |
| 286 | $data = array_map(function($value) use ($validStates) { |
| 287 | $value = trim((string)$value); |
| 288 | // A blank cell is left blank on purpose: blank means "leave the stored value alone", |
| 289 | // which is a different thing from an invalid value, and only the caller can act on it. |
| 290 | if ($value === '') { |
| 291 | return ''; |
| 292 | } |
| 293 | return in_array($value, $validStates, true) ? $value : SubscriberEntity::TRACKING_CONSENT_UNKNOWN; |
| 294 | }, $data); |
| 295 | } |
| 296 | // if this is a custom column |
| 297 | if (in_array($column, $this->subscribersCustomFields)) { |
| 298 | $customField = $this->customFieldsRepository->findOneById($column); |
| 299 | if (!$customField instanceof CustomFieldEntity) { |
| 300 | continue; |
| 301 | } |
| 302 | // validate date type |
| 303 | if ($customField->getType() === CustomFieldEntity::TYPE_DATE) { |
| 304 | $data = $this->validateDateTime($data, $invalidRecords); |
| 305 | } |
| 306 | } |
| 307 | } |
| 308 | if ($invalidRecords) { |
| 309 | foreach ($subscribersData as $column => &$data) { |
| 310 | $data = array_diff_key($data, array_flip($invalidRecords)); |
| 311 | $data = array_values($data); |
| 312 | } |
| 313 | } |
| 314 | if (empty($subscribersData['email'])) return false; |
| 315 | return $subscribersData; |
| 316 | } |
| 317 | |
| 318 | private function validateDateTime(array $data, array &$invalidRecords): array { |
| 319 | $siteUsesCustomFormat = WPFunctions::get()->getOption('date_format') === 'd/m/Y'; |
| 320 | if ($siteUsesCustomFormat) { |
| 321 | return $this->validateDateTimeAttemptCustomFormat($data, $invalidRecords); |
| 322 | } |
| 323 | |
| 324 | $validationRule = 'datetime'; |
| 325 | return array_map( |
| 326 | function ($index, $date) use ($validationRule, &$invalidRecords) { |
| 327 | if (empty($date)) return $date; |
| 328 | $date = (new DateConverter())->convertDateToDatetime($date, $validationRule); |
| 329 | if (!$date) { |
| 330 | $invalidRecords[] = $index; |
| 331 | } |
| 332 | return $date; |
| 333 | }, |
| 334 | array_keys($data), |
| 335 | $data |
| 336 | ); |
| 337 | } |
| 338 | |
| 339 | private function validateDateTimeAttemptCustomFormat(array $data, array &$invalidRecords): array { |
| 340 | $validationRule = 'datetime'; |
| 341 | $dateTimeDates = $data; |
| 342 | $dateTimeInvalidRecords = $invalidRecords; |
| 343 | $datetimeErrorCount = 0; |
| 344 | |
| 345 | $validationRuleCustom = 'd/m/Y'; |
| 346 | $customFormatDates = $data; |
| 347 | $customFormatInvalidRecords = $invalidRecords; |
| 348 | $customFormatErrorCount = 0; |
| 349 | |
| 350 | // We attempt converting with both date formats |
| 351 | foreach ($data as $index => $date) { |
| 352 | if (empty($date)) { |
| 353 | $dateTimeDates[$index] = $date; |
| 354 | $customFormatDates[$index] = $date; |
| 355 | continue; |
| 356 | }; |
| 357 | $dateTimeDates[$index] = (new DateConverter())->convertDateToDatetime($date, $validationRule); |
| 358 | if (!$dateTimeDates[$index]) { |
| 359 | $datetimeErrorCount ++; |
| 360 | $dateTimeInvalidRecords[] = $index; |
| 361 | } |
| 362 | $customFormatDates[$index] = (new DateConverter())->convertDateToDatetime($date, $validationRuleCustom); |
| 363 | if (!$customFormatDates[$index]) { |
| 364 | $customFormatErrorCount ++; |
| 365 | $customFormatInvalidRecords[] = $index; |
| 366 | } |
| 367 | } |
| 368 | |
| 369 | if ($customFormatErrorCount < $datetimeErrorCount) { |
| 370 | $invalidRecords = $customFormatInvalidRecords; |
| 371 | return $customFormatDates; |
| 372 | } |
| 373 | |
| 374 | $invalidRecords = $dateTimeInvalidRecords; |
| 375 | return $dateTimeDates; |
| 376 | } |
| 377 | |
| 378 | public function transformSubscribersData(array $subscribers, array $columns): array { |
| 379 | $transformedSubscribers = []; |
| 380 | foreach ($columns as $column => $data) { |
| 381 | $transformedSubscribers[$column] = array_column($subscribers, $data['index']); |
| 382 | } |
| 383 | return $transformedSubscribers; |
| 384 | } |
| 385 | |
| 386 | /** |
| 387 | * @param array $subscribersData |
| 388 | * @return array{array|false,array,array|false} |
| 389 | */ |
| 390 | public function splitSubscribersData(array $subscribersData): array { |
| 391 | // $subscribers_data is an two-dimensional associative array |
| 392 | // of all subscribers being imported: [field => [value1, value2], field => [value1, value2], ...] |
| 393 | $tempExistingSubscribers = []; |
| 394 | foreach (array_chunk($subscribersData['email'], self::DB_QUERY_CHUNK_SIZE) as $subscribersEmails) { |
| 395 | // create a two-dimensional indexed array of all existing subscribers |
| 396 | // with just wp_user_id and email fields: [[wp_user_id, email], [wp_user_id, email], ...] |
| 397 | $tempExistingSubscribers = array_merge( |
| 398 | $tempExistingSubscribers, |
| 399 | $this->subscriberRepository->findWpUserIdAndEmailByEmails($subscribersEmails) |
| 400 | ); |
| 401 | } |
| 402 | if (!$tempExistingSubscribers) { |
| 403 | return [ |
| 404 | false, // existing subscribers |
| 405 | $subscribersData, // new subscribers |
| 406 | false, // WP users |
| 407 | ]; |
| 408 | } |
| 409 | // extract WP users ids into a simple indexed array: [wp_user_id_1, wp_user_id_2, ...] |
| 410 | $wpUsers = array_filter(array_column($tempExistingSubscribers, 'wp_user_id')); |
| 411 | // create a new two-dimensional associative array with existing subscribers ($existing_subscribers) |
| 412 | // and reduce $subscribers_data to only new subscribers by removing existing subscribers |
| 413 | $existingSubscribers = []; |
| 414 | $subscribersEmails = array_flip($subscribersData['email']); |
| 415 | foreach ($tempExistingSubscribers as $tempExistingSubscriber) { |
| 416 | $existingSubscriberKey = $subscribersEmails[$tempExistingSubscriber['email']]; |
| 417 | foreach ($subscribersData as $field => &$value) { |
| 418 | $existingSubscribers[$field][] = $value[$existingSubscriberKey]; |
| 419 | unset($value[$existingSubscriberKey]); |
| 420 | } |
| 421 | } |
| 422 | $newSubscribers = $subscribersData; |
| 423 | // reindex array after unsetting elements |
| 424 | $newSubscribers = array_map('array_values', $newSubscribers); |
| 425 | // remove empty values |
| 426 | $newSubscribers = array_filter($newSubscribers); |
| 427 | return [ |
| 428 | $existingSubscribers, |
| 429 | $newSubscribers, |
| 430 | $wpUsers, |
| 431 | ]; |
| 432 | } |
| 433 | |
| 434 | public function deleteExistingTrashedSubscribers(array $subscribersData): void { |
| 435 | $existingTrashedRecords = array_filter( |
| 436 | array_map(function($subscriberEmails) { |
| 437 | return $this->subscriberRepository->findIdsOfDeletedByEmails($subscriberEmails); |
| 438 | }, array_chunk($subscribersData['email'], self::DB_QUERY_CHUNK_SIZE)) |
| 439 | ); |
| 440 | $existingTrashedRecords = Helpers::flattenArray($existingTrashedRecords); |
| 441 | if (!$existingTrashedRecords) { |
| 442 | return; |
| 443 | } |
| 444 | foreach (array_chunk($existingTrashedRecords, self::DB_QUERY_CHUNK_SIZE) as $subscriberIds) { |
| 445 | $this->subscriberRepository->bulkDelete($subscriberIds); |
| 446 | } |
| 447 | } |
| 448 | |
| 449 | public function addMissingRequiredFields(array $subscribers): array { |
| 450 | foreach (array_keys($this->requiredSubscribersFields) as $requiredField) { |
| 451 | $subscribers = $this->addField($subscribers, $requiredField, $this->requiredSubscribersFields[$requiredField]); |
| 452 | } |
| 453 | return $subscribers; |
| 454 | } |
| 455 | |
| 456 | /** |
| 457 | * @param array $subscribers |
| 458 | * @param string $fieldName |
| 459 | * @param mixed $fieldValue |
| 460 | * @return array |
| 461 | */ |
| 462 | private function addField(array $subscribers, string $fieldName, $fieldValue): array { |
| 463 | if (in_array($fieldName, $subscribers['fields'])) return $subscribers; |
| 464 | |
| 465 | $subscribersCount = count($subscribers['data'][key($subscribers['data'])]); |
| 466 | $subscribers['data'][$fieldName] = array_fill( |
| 467 | 0, |
| 468 | $subscribersCount, |
| 469 | $fieldValue |
| 470 | ); |
| 471 | $subscribers['fields'][] = $fieldName; |
| 472 | |
| 473 | return $subscribers; |
| 474 | } |
| 475 | |
| 476 | private function setSubscriptionStatusToDefault(array $subscribersData, string $defaultStatus): array { |
| 477 | if (!in_array('status', $subscribersData['fields'])) return $subscribersData; |
| 478 | $subscribersData['data']['status'] = array_map(function() use ($defaultStatus) { |
| 479 | return $defaultStatus; |
| 480 | }, $subscribersData['data']['status']); |
| 481 | |
| 482 | if ($defaultStatus === SubscriberEntity::STATUS_SUBSCRIBED) { |
| 483 | if (!in_array('last_subscribed_at', $subscribersData['fields'])) { |
| 484 | $subscribersData['fields'][] = 'last_subscribed_at'; |
| 485 | } |
| 486 | $subscribersData['data']['last_subscribed_at'] = array_map(function() { |
| 487 | return $this->createdAt; |
| 488 | }, $subscribersData['data']['status']); |
| 489 | } |
| 490 | return $subscribersData; |
| 491 | } |
| 492 | |
| 493 | private function setSource(array $subscribersData): array { |
| 494 | $subscribersCount = count($subscribersData['data'][key($subscribersData['data'])]); |
| 495 | $subscribersData['fields'][] = 'source'; |
| 496 | $subscribersData['data']['source'] = array_fill( |
| 497 | 0, |
| 498 | $subscribersCount, |
| 499 | Source::IMPORTED |
| 500 | ); |
| 501 | return $subscribersData; |
| 502 | } |
| 503 | |
| 504 | private function setLinkToken(array $subscribersData): array { |
| 505 | $subscribersCount = count($subscribersData['data'][key($subscribersData['data'])]); |
| 506 | $subscribersData['fields'][] = 'link_token'; |
| 507 | $subscribersData['data']['link_token'] = array_map( |
| 508 | function () { |
| 509 | return Security::generateRandomString(SubscriberEntity::LINK_TOKEN_LENGTH); |
| 510 | }, |
| 511 | array_fill(0, $subscribersCount, null) |
| 512 | ); |
| 513 | return $subscribersData; |
| 514 | } |
| 515 | |
| 516 | /** |
| 517 | * Stamps consent evidence for newly created subscribers. A blank cell needs no |
| 518 | * work: the column defaults (unknown/NULL/NULL/NULL) already mean "untouched" |
| 519 | * for a row that did not exist. A non-blank cell is the collection event, so |
| 520 | * the timestamp is stamped now and never read from the CSV — it drives the |
| 521 | * tracked-at-send predicate, and a backdated value could push a rate over 100%. |
| 522 | */ |
| 523 | private function applyTrackingConsentToNewSubscribers(array $subscribersData): array { |
| 524 | if (!in_array('tracking_consent', $subscribersData['fields'], true)) { |
| 525 | // Evidence with no consent behind it is not a record of anything, so a CSV that |
| 526 | // maps only the method or wording column is dropped rather than stored against |
| 527 | // the default `unknown` with no timestamp. Same rule the public API applies. |
| 528 | return $this->stripTrackingConsentEvidenceColumns($subscribersData); |
| 529 | } |
| 530 | $states = $subscribersData['data']['tracking_consent']; |
| 531 | $methods = $subscribersData['data']['tracking_consent_method'] ?? []; |
| 532 | $copies = $subscribersData['data']['tracking_consent_copy'] ?? []; |
| 533 | |
| 534 | $stateValues = $updatedAtValues = $methodValues = $copyValues = []; |
| 535 | foreach ($states as $index => $state) { |
| 536 | if (trim((string)$state) === '') { |
| 537 | // The column is NOT NULL, so a blank cell has to be written as the default |
| 538 | // rather than as an empty string. |
| 539 | $stateValues[] = SubscriberEntity::TRACKING_CONSENT_UNKNOWN; |
| 540 | $updatedAtValues[] = null; |
| 541 | $methodValues[] = null; |
| 542 | $copyValues[] = null; |
| 543 | continue; |
| 544 | } |
| 545 | $stateValues[] = trim((string)$state); |
| 546 | $updatedAtValues[] = $this->createdAt; |
| 547 | $method = trim((string)($methods[$index] ?? '')); |
| 548 | $methodValues[] = $method !== '' ? mb_substr($method, 0, self::TRACKING_CONSENT_METHOD_MAX_LENGTH) : SubscriberEntity::TRACKING_CONSENT_METHOD_IMPORT; |
| 549 | $copy = trim((string)($copies[$index] ?? '')); |
| 550 | $copyValues[] = $copy !== '' ? $copy : null; |
| 551 | } |
| 552 | |
| 553 | $evidence = [ |
| 554 | 'tracking_consent' => $stateValues, |
| 555 | 'tracking_consent_updated_at' => $updatedAtValues, |
| 556 | 'tracking_consent_method' => $methodValues, |
| 557 | 'tracking_consent_copy' => $copyValues, |
| 558 | ]; |
| 559 | foreach ($evidence as $field => $values) { |
| 560 | if (!in_array($field, $subscribersData['fields'], true)) { |
| 561 | $subscribersData['fields'][] = $field; |
| 562 | } |
| 563 | $subscribersData['data'][$field] = $values; |
| 564 | } |
| 565 | return $subscribersData; |
| 566 | } |
| 567 | |
| 568 | /** Drops the evidence columns, used when no consent state came with them. */ |
| 569 | private function stripTrackingConsentEvidenceColumns(array $subscribersData): array { |
| 570 | $evidenceFields = ['tracking_consent_method', 'tracking_consent_copy']; |
| 571 | foreach ($evidenceFields as $field) { |
| 572 | unset($subscribersData['data'][$field]); |
| 573 | } |
| 574 | $subscribersData['fields'] = array_values(array_diff($subscribersData['fields'], $evidenceFields)); |
| 575 | return $subscribersData; |
| 576 | } |
| 577 | |
| 578 | /** |
| 579 | * Removes the consent columns from the standard update write. updateMultiple() |
| 580 | * writes one uniform set of columns per call, so leaving them in would let a |
| 581 | * blank CSV cell overwrite a stored value. updateTrackingConsent() below does |
| 582 | * the real write, scoped to the rows that actually supplied a state. |
| 583 | */ |
| 584 | private function stripTrackingConsentColumns(array $subscribersData): array { |
| 585 | $consentFields = ['tracking_consent', 'tracking_consent_method', 'tracking_consent_copy']; |
| 586 | foreach ($consentFields as $field) { |
| 587 | unset($subscribersData['data'][$field]); |
| 588 | } |
| 589 | $subscribersData['fields'] = array_values(array_diff($subscribersData['fields'], $consentFields)); |
| 590 | return $subscribersData; |
| 591 | } |
| 592 | |
| 593 | /** |
| 594 | * Writes consent for existing subscribers, skipping every row whose CSV cell |
| 595 | * was blank so their stored value is left alone. |
| 596 | */ |
| 597 | private function updateTrackingConsent(array $subscribersData): void { |
| 598 | if (!in_array('tracking_consent', $subscribersData['fields'], true)) { |
| 599 | return; |
| 600 | } |
| 601 | $emails = $subscribersData['data']['email']; |
| 602 | $states = $subscribersData['data']['tracking_consent']; |
| 603 | $methods = $subscribersData['data']['tracking_consent_method'] ?? []; |
| 604 | $copies = $subscribersData['data']['tracking_consent_copy'] ?? []; |
| 605 | |
| 606 | $rows = []; |
| 607 | foreach ($states as $index => $state) { |
| 608 | $state = trim((string)$state); |
| 609 | if ($state === '') { |
| 610 | continue; |
| 611 | } |
| 612 | $method = trim((string)($methods[$index] ?? '')); |
| 613 | $copy = trim((string)($copies[$index] ?? '')); |
| 614 | $rows[] = [ |
| 615 | $emails[$index], |
| 616 | $state, |
| 617 | $this->updatedAt, |
| 618 | $method !== '' ? mb_substr($method, 0, self::TRACKING_CONSENT_METHOD_MAX_LENGTH) : SubscriberEntity::TRACKING_CONSENT_METHOD_IMPORT, |
| 619 | $copy !== '' ? $copy : null, |
| 620 | ]; |
| 621 | } |
| 622 | if (!$rows) { |
| 623 | return; |
| 624 | } |
| 625 | foreach (array_chunk($rows, self::DB_QUERY_CHUNK_SIZE) as $chunk) { |
| 626 | $this->importExportRepository->updateMultiple( |
| 627 | SubscriberEntity::class, |
| 628 | ['email', 'tracking_consent', 'tracking_consent_updated_at', 'tracking_consent_method', 'tracking_consent_copy'], |
| 629 | $chunk |
| 630 | ); |
| 631 | } |
| 632 | } |
| 633 | |
| 634 | public function getSubscribersFields(array $subscribersFields): array { |
| 635 | return array_values( |
| 636 | array_filter( |
| 637 | array_map(function($field) { |
| 638 | if (!is_int($field)) return $field; |
| 639 | }, $subscribersFields) |
| 640 | ) |
| 641 | ); |
| 642 | } |
| 643 | |
| 644 | /** |
| 645 | * @param array $subscribersFields |
| 646 | * @return int[] |
| 647 | */ |
| 648 | public function getCustomSubscribersFields(array $subscribersFields): array { |
| 649 | return array_values( |
| 650 | array_filter( |
| 651 | array_map(function($field) { |
| 652 | if (is_int($field)) return $field; |
| 653 | }, $subscribersFields) |
| 654 | ) |
| 655 | ); |
| 656 | } |
| 657 | |
| 658 | public function createOrUpdateSubscribers( |
| 659 | string $action, |
| 660 | array $subscribersData, |
| 661 | array $subscribersCustomFields = [] |
| 662 | ): ?array { |
| 663 | $subscribersCount = count($subscribersData['data'][key($subscribersData['data'])]); |
| 664 | $subscribers = array_map(function($index) use ($subscribersData) { |
| 665 | return array_map(function($field) use ($index, $subscribersData) { |
| 666 | return $subscribersData['data'][$field][$index]; |
| 667 | }, $subscribersData['fields']); |
| 668 | }, range(0, $subscribersCount - 1)); |
| 669 | foreach (array_chunk($subscribers, self::DB_QUERY_CHUNK_SIZE) as $data) { |
| 670 | if ($action === self::ACTION_CREATE) { |
| 671 | $this->importExportRepository->insertMultiple( |
| 672 | SubscriberEntity::class, |
| 673 | $subscribersData['fields'], |
| 674 | $data |
| 675 | ); |
| 676 | } elseif ($action === self::ACTION_UPDATE) { |
| 677 | $this->importExportRepository->updateMultiple( |
| 678 | SubscriberEntity::class, |
| 679 | $subscribersData['fields'], |
| 680 | $data, |
| 681 | $this->updatedAt |
| 682 | ); |
| 683 | } |
| 684 | } |
| 685 | $createdOrUpdatedSubscribers = []; |
| 686 | foreach (array_chunk($subscribersData['data']['email'], self::DB_QUERY_CHUNK_SIZE) as $emails) { |
| 687 | foreach ($this->subscriberRepository->findIdAndEmailByEmails($emails) as $createdOrUpdatedSubscriber) { |
| 688 | // ensure emails loaded from the DB are lowercased (imported emails are lowercased as well) |
| 689 | $createdOrUpdatedSubscriber['email'] = mb_strtolower($createdOrUpdatedSubscriber['email']); |
| 690 | $createdOrUpdatedSubscribers[] = $createdOrUpdatedSubscriber; |
| 691 | } |
| 692 | } |
| 693 | if (empty($createdOrUpdatedSubscribers)) return null; |
| 694 | |
| 695 | $this->subscriberRepository->invalidateTotalSubscribersCache(); |
| 696 | $createdOrUpdatedSubscribersIds = array_column($createdOrUpdatedSubscribers, 'id'); |
| 697 | if ($subscribersCustomFields) { |
| 698 | $this->createOrUpdateCustomFields( |
| 699 | $action, |
| 700 | $createdOrUpdatedSubscribers, |
| 701 | $subscribersData, |
| 702 | $subscribersCustomFields |
| 703 | ); |
| 704 | } |
| 705 | $this->addSubscribersToSegments( |
| 706 | $createdOrUpdatedSubscribersIds, |
| 707 | $this->segmentsIds |
| 708 | ); |
| 709 | $this->addTagsToSubscribers( |
| 710 | $createdOrUpdatedSubscribersIds, |
| 711 | $this->tags |
| 712 | ); |
| 713 | return $createdOrUpdatedSubscribers; |
| 714 | } |
| 715 | |
| 716 | public function createOrUpdateCustomFields( |
| 717 | string $action, |
| 718 | array $createdOrUpdatedSubscribers, |
| 719 | array $subscribersData, |
| 720 | array $subscribersCustomFieldsIds |
| 721 | ): void { |
| 722 | // check if custom fields exist in the database |
| 723 | $subscribersCustomFieldsIds = array_map(function(CustomFieldEntity $customField): int { |
| 724 | return (int)$customField->getId(); |
| 725 | }, $this->customFieldsRepository->findBy(['id' => $subscribersCustomFieldsIds, 'deletedAt' => null])); |
| 726 | if (!$subscribersCustomFieldsIds) { |
| 727 | return; |
| 728 | } |
| 729 | // assemble a two-dimensional array: [[custom_field_id, subscriber_id, value], [custom_field_id, subscriber_id, value], ...] |
| 730 | $subscribersCustomFieldsData = []; |
| 731 | $subscribersEmails = array_flip($subscribersData['data']['email']); |
| 732 | foreach ($createdOrUpdatedSubscribers as $createdOrUpdatedSubscriber) { |
| 733 | $subscriberIndex = $subscribersEmails[$createdOrUpdatedSubscriber['email']]; |
| 734 | foreach ($subscribersData['data'] as $field => $values) { |
| 735 | // exclude non-custom fields |
| 736 | if (!is_int($field)) continue; |
| 737 | $subscribersCustomFieldsData[] = [ |
| 738 | (int)$field, |
| 739 | $createdOrUpdatedSubscriber['id'], |
| 740 | $values[$subscriberIndex], |
| 741 | $this->createdAt, |
| 742 | ]; |
| 743 | } |
| 744 | } |
| 745 | $columns = [ |
| 746 | 'custom_field_id', |
| 747 | 'subscriber_id', |
| 748 | 'value', |
| 749 | 'created_at', |
| 750 | ]; |
| 751 | $customFieldCount = count($subscribersCustomFieldsIds); |
| 752 | $customFieldBatchSize = (int)(round(self::DB_QUERY_CHUNK_SIZE / $customFieldCount) * $customFieldCount); |
| 753 | $customFieldBatchSize = ($customFieldBatchSize > 0) ? $customFieldBatchSize : 1; |
| 754 | foreach (array_chunk($subscribersCustomFieldsData, $customFieldBatchSize) as $subscribersCustomFieldsDataChunk) { |
| 755 | $this->importExportRepository->insertMultiple( |
| 756 | SubscriberCustomFieldEntity::class, |
| 757 | $columns, |
| 758 | $subscribersCustomFieldsDataChunk |
| 759 | ); |
| 760 | if ($action === self::ACTION_UPDATE) { |
| 761 | $this->importExportRepository->updateMultiple( |
| 762 | SubscriberCustomFieldEntity::class, |
| 763 | $columns, |
| 764 | $subscribersCustomFieldsDataChunk, |
| 765 | $this->updatedAt |
| 766 | ); |
| 767 | } |
| 768 | } |
| 769 | } |
| 770 | |
| 771 | /** |
| 772 | * @param int[] $wpUsers |
| 773 | * @return array |
| 774 | */ |
| 775 | public function synchronizeWPUsers(array $wpUsers): array { |
| 776 | $users = array_map([$this->wpSegment, 'synchronizeUser'], $wpUsers); |
| 777 | $this->subscriberRepository->invalidateTotalSubscribersCache(); |
| 778 | return $users; |
| 779 | } |
| 780 | |
| 781 | public function addSubscribersToSegments(array $subscribersIds, array $segmentsIds): void { |
| 782 | $columns = [ |
| 783 | 'subscriber_id', |
| 784 | 'segment_id', |
| 785 | 'created_at', |
| 786 | ]; |
| 787 | foreach ($segmentsIds as $segmentId) { |
| 788 | foreach (array_chunk($subscribersIds, self::DB_QUERY_CHUNK_SIZE) as $subscriberIdsChunk) { |
| 789 | $data = []; |
| 790 | $data = array_merge($data, array_map(function ($subscriberId) use ($segmentId): array { |
| 791 | return [ |
| 792 | $subscriberId, |
| 793 | $segmentId, |
| 794 | $this->createdAt, |
| 795 | ]; |
| 796 | }, $subscriberIdsChunk)); |
| 797 | |
| 798 | $this->importExportRepository->insertMultiple( |
| 799 | SubscriberSegmentEntity::class, |
| 800 | $columns, |
| 801 | $data |
| 802 | ); |
| 803 | } |
| 804 | } |
| 805 | $this->subscriberRepository->recalculateSegmentsCount($subscribersIds); |
| 806 | } |
| 807 | |
| 808 | /** |
| 809 | * @param int[] $subscribersIds |
| 810 | * @param string[] $tagNames |
| 811 | */ |
| 812 | public function addTagsToSubscribers(array $subscribersIds, array $tagNames): void { |
| 813 | $tagIds = []; |
| 814 | foreach ($tagNames as $tagName) { |
| 815 | $tag = $this->tagRepository->findOneBy(['name' => $tagName]); |
| 816 | if (!$tag) { |
| 817 | $tag = $this->tagRepository->createOrUpdate(['name' => $tagName]); |
| 818 | } |
| 819 | $tagIds[] = $tag->getId(); |
| 820 | } |
| 821 | |
| 822 | $columns = [ |
| 823 | 'subscriber_id', |
| 824 | 'tag_id', |
| 825 | 'created_at', |
| 826 | ]; |
| 827 | foreach ($tagIds as $tagId) { |
| 828 | foreach (array_chunk($subscribersIds, self::DB_QUERY_CHUNK_SIZE) as $subscriberIdsChunk) { |
| 829 | $data = []; |
| 830 | $data = array_merge($data, array_map(function ($subscriberId) use ($tagId): array { |
| 831 | return [ |
| 832 | $subscriberId, |
| 833 | $tagId, |
| 834 | $this->createdAt, |
| 835 | ]; |
| 836 | }, $subscriberIdsChunk)); |
| 837 | |
| 838 | $this->importExportRepository->insertMultiple( |
| 839 | SubscriberTagEntity::class, |
| 840 | $columns, |
| 841 | $data |
| 842 | ); |
| 843 | } |
| 844 | } |
| 845 | } |
| 846 | } |
| 847 |