DataConverters
1 year ago
Exceptions
4 years ago
Importers
1 year ago
Logger
4 years ago
Config.php
4 years ago
Geoip2.php
2 years ago
Importer.php
1 year ago
RecordInserter.php
4 years ago
Importer.php
300 lines
| 1 | <?php |
| 2 | |
| 3 | namespace WpMatomo\WpStatistics; |
| 4 | |
| 5 | use Piwik\ArchiveProcessor\Parameters; |
| 6 | use Piwik\Common; |
| 7 | use Piwik\Container\StaticContainer; |
| 8 | use Piwik\DataAccess\ArchiveWriter; |
| 9 | use Piwik\Date; |
| 10 | use Piwik\Option; |
| 11 | use Piwik\Period\Factory; |
| 12 | use Piwik\Plugin\Manager; |
| 13 | use Piwik\Segment; |
| 14 | use Piwik\Site; |
| 15 | use Psr\Log\LoggerInterface; |
| 16 | use Piwik\Archive\ArchiveInvalidator; |
| 17 | use WP_STATISTICS\DB; |
| 18 | use WpMatomo\Db\Settings; |
| 19 | use WpMatomo\ScheduledTasks; |
| 20 | use WpMatomo\WpStatistics\Exceptions\MaxEndDateReachedException; |
| 21 | use WpMatomo\WpStatistics\Importers\Actions\RecordImporter; |
| 22 | /** |
| 23 | * @package WpMatomo |
| 24 | * @subpackage WpStatisticsImport |
| 25 | * |
| 26 | * phpcs:disable WordPress.DB |
| 27 | */ |
| 28 | class Importer { |
| 29 | |
| 30 | const IS_IMPORTED_FROM_WPS_NUMERIC = 'WpStatisticsImporter_isImportedFromWpStatistics'; |
| 31 | |
| 32 | /** |
| 33 | * @var LoggerInterface |
| 34 | */ |
| 35 | private $logger; |
| 36 | |
| 37 | /** |
| 38 | * @var array|null |
| 39 | */ |
| 40 | private $record_importers; |
| 41 | |
| 42 | /** |
| 43 | * @var string |
| 44 | */ |
| 45 | private $no_data_message_removed = false; |
| 46 | |
| 47 | /** |
| 48 | * @var Date |
| 49 | */ |
| 50 | private $end_date = null; |
| 51 | |
| 52 | private $should_rethrow = false; |
| 53 | |
| 54 | public function __construct( LoggerInterface $logger ) { |
| 55 | $this->logger = $logger; |
| 56 | $this->end_date = $this->get_ending_date(); |
| 57 | } |
| 58 | |
| 59 | public function set_should_rethrow( $should_rethrow ) { |
| 60 | $this->should_rethrow = $should_rethrow; |
| 61 | } |
| 62 | |
| 63 | /** |
| 64 | * Returns the first date in the matomo records |
| 65 | * |
| 66 | * @return Date |
| 67 | */ |
| 68 | protected function get_ending_date() { |
| 69 | global $wpdb; |
| 70 | $db_settings = new Settings(); |
| 71 | $table = $db_settings->prefix_table_name( 'log_visit' ); |
| 72 | $sql = <<<SQL |
| 73 | SELECT min(visit_last_action_time) from $table |
| 74 | SQL; |
| 75 | try { |
| 76 | $row = $wpdb->get_row( $sql, ARRAY_N ); |
| 77 | if ( ! empty( $row[0] ) ) { |
| 78 | return Date::factory( $row[0] ); |
| 79 | } else { |
| 80 | return Date::yesterday(); |
| 81 | } |
| 82 | } catch ( \Exception $e ) { |
| 83 | return Date::yesterday(); |
| 84 | } |
| 85 | } |
| 86 | |
| 87 | |
| 88 | /** |
| 89 | * Returns the first date in the wpStatistics data |
| 90 | * |
| 91 | * @return \Piwik\Date |
| 92 | */ |
| 93 | protected function get_started() { |
| 94 | global $wpdb; |
| 95 | $table = DB::table( 'visit' ); |
| 96 | if ( ! empty( $table ) ) { |
| 97 | $sql = <<<SQL |
| 98 | SELECT min(last_visit) from $table |
| 99 | SQL; |
| 100 | } else { |
| 101 | $table = DB::table( 'visitor' ); |
| 102 | |
| 103 | $sql = <<<SQL |
| 104 | SELECT min(last_view) from $table |
| 105 | SQL; |
| 106 | } |
| 107 | |
| 108 | $row = $wpdb->get_row( $sql, ARRAY_N ); |
| 109 | $date = $row[0]; |
| 110 | if ( empty( $date ) ) { |
| 111 | return Date::yesterday(); |
| 112 | } |
| 113 | |
| 114 | return Date::factory( $date ); |
| 115 | } |
| 116 | |
| 117 | /** |
| 118 | * Update the first date in the configuration. |
| 119 | * Otherwise records are here but the date picker does not allow to select these dates |
| 120 | * |
| 121 | * @param int $id_site |
| 122 | * @param Date $date |
| 123 | * |
| 124 | * @return void |
| 125 | */ |
| 126 | private function adjust_matomo_date( $id_site, Date $date ) { |
| 127 | global $wpdb; |
| 128 | $db_settings = new Settings(); |
| 129 | $prefix_table = $db_settings->prefix_table_name( 'site' ); |
| 130 | $wpdb->update( $prefix_table, [ 'ts_created' => $date->toString( 'Y-m-d h:i:s' ) ], [ 'idsite' => $id_site ] ); |
| 131 | } |
| 132 | |
| 133 | public function import( $id_site, $archive = true ) { |
| 134 | $end = $this->end_date; |
| 135 | $start = $this->get_started(); |
| 136 | |
| 137 | $this->adjust_matomo_date( $id_site, $start ); |
| 138 | try { |
| 139 | $this->no_data_message_removed = false; |
| 140 | |
| 141 | $end_plus_one = $end->addDay( 1 ); |
| 142 | |
| 143 | if ( $start->getTimestamp() >= $end_plus_one->getTimestamp() ) { |
| 144 | throw new \InvalidArgumentException( "Invalid date range, start date is later than end date: {$start},{$end}" ); |
| 145 | } |
| 146 | $record_importers = $this->get_record_importers(); |
| 147 | $site = new Site( $id_site ); |
| 148 | // phpcs:ignore Generic.CodeAnalysis.ForLoopWithTestFunctionCall.NotAllowed |
| 149 | for ( $date = $start; $date->getTimestamp() < $end_plus_one->getTimestamp(); $date = $date->addDay( 1 ) ) { |
| 150 | $this->logger->notice( |
| 151 | 'Importing data for date {date}...', |
| 152 | [ |
| 153 | 'date' => $date->toString(), |
| 154 | ] |
| 155 | ); |
| 156 | |
| 157 | try { |
| 158 | $this->import_day( $site, $date, $record_importers ); |
| 159 | } finally { |
| 160 | // force delete all tables in case they aren't all freed |
| 161 | \Piwik\DataTable\Manager::getInstance()->deleteAll(); |
| 162 | } |
| 163 | } |
| 164 | unset( $record_importers ); |
| 165 | } catch ( MaxEndDateReachedException $ex ) { |
| 166 | $this->logger->info( 'Max end date reached. This occurs in Matomo for WordPress installs when the importer tries to import days on or after the day Matomo for WordPress installed.' ); |
| 167 | |
| 168 | if ( true === $archive ) { |
| 169 | // by launching the archiver now the weekly, monthly and yearly archives should be generated right away and it won't |
| 170 | // take up to an hour. Also by running it on the cli we have less risk that this long running archiving process times out |
| 171 | $this->logger->info( 'Matomo Analytics starting the report generation of weekly, monthly and yearly reports. This may take a while.' ); |
| 172 | $scheduled_tasks = new ScheduledTasks( \WpMatomo::$settings ); |
| 173 | $scheduled_tasks->archive(); |
| 174 | } |
| 175 | $this->logger->info( 'Matomo Analytics report generation finished' ); |
| 176 | |
| 177 | return true; |
| 178 | } catch ( \Exception $ex ) { |
| 179 | $this->on_error( $ex ); |
| 180 | return true; |
| 181 | } |
| 182 | |
| 183 | return false; |
| 184 | } |
| 185 | |
| 186 | /** |
| 187 | * For use in record_importers that need to archive data for segments. |
| 188 | * |
| 189 | * @var RecordImporter[] $record_importers |
| 190 | * @throws MaxEndDateReachedException In case we have reach the end date to proceed. |
| 191 | */ |
| 192 | public function import_day( Site $site, Date $date, $record_importers ) { |
| 193 | if ( $this->end_date && $this->end_date->isEarlier( $date ) ) { |
| 194 | throw new MaxEndDateReachedException(); |
| 195 | } |
| 196 | $archive_writer = $this->make_archive_writer( $site, $date ); |
| 197 | $archive_writer->initNewArchive(); |
| 198 | |
| 199 | $record_inserter = new RecordInserter( $archive_writer ); |
| 200 | |
| 201 | foreach ( $record_importers as $plugin => $record_importer ) { |
| 202 | if ( ! $record_importer->supports_site() ) { |
| 203 | continue; |
| 204 | } |
| 205 | |
| 206 | $this->logger->info( |
| 207 | 'Importing data for the {plugin} plugin.', |
| 208 | [ |
| 209 | 'plugin' => $plugin, |
| 210 | ] |
| 211 | ); |
| 212 | |
| 213 | $record_importer->set_record_inserter( $record_inserter ); |
| 214 | |
| 215 | $record_importer->import_records( $date ); |
| 216 | |
| 217 | // since we recorded some data, at some time, remove the no data message |
| 218 | if ( ! $this->no_data_message_removed ) { |
| 219 | $this->remove_no_data_message( $site->getId() ); |
| 220 | $this->no_data_message_removed = true; |
| 221 | } |
| 222 | } |
| 223 | |
| 224 | $archive_writer->insertRecord( self::IS_IMPORTED_FROM_WPS_NUMERIC, 1 ); |
| 225 | $archive_writer->finalizeArchive(); |
| 226 | |
| 227 | $invalidator = StaticContainer::get( ArchiveInvalidator::class ); |
| 228 | $invalidator->markArchivesAsInvalidated( |
| 229 | [ $site->getId() ], |
| 230 | [ $date ], |
| 231 | 'week', |
| 232 | null, |
| 233 | false, |
| 234 | false, |
| 235 | null, |
| 236 | $ignore_purge_log_data_date = true |
| 237 | ); |
| 238 | |
| 239 | Common::destroy( $archive_writer ); |
| 240 | } |
| 241 | |
| 242 | private function make_archive_writer( Site $site, Date $date, $segment = '' ) { |
| 243 | $period = Factory::build( 'day', $date ); |
| 244 | $segment = new Segment( $segment, [ $site->getId() ] ); |
| 245 | |
| 246 | $params = new Parameters( $site, $period, $segment ); |
| 247 | return new ArchiveWriter( $params ); |
| 248 | } |
| 249 | |
| 250 | /** |
| 251 | * @return RecordImporter[] |
| 252 | * @throws \Exception In case importer has no plugin name. |
| 253 | */ |
| 254 | private function get_record_importers() { |
| 255 | if ( empty( $this->record_importers ) ) { |
| 256 | $record_importers = Config::get_importers(); |
| 257 | |
| 258 | $this->record_importers = []; |
| 259 | foreach ( $record_importers as $record_importer_class ) { |
| 260 | if ( ! defined( $record_importer_class . '::PLUGIN_NAME' ) ) { |
| 261 | throw new \Exception( "The $record_importer_class record importer is missing the PLUGIN_NAME constant." ); |
| 262 | } |
| 263 | |
| 264 | $namespace = explode( '\\', $record_importer_class ); |
| 265 | $plugin_name = array_pop( $namespace ); |
| 266 | if ( $this->is_plugin_unavailable( $record_importer_class::PLUGIN_NAME ) ) { |
| 267 | continue; |
| 268 | } |
| 269 | |
| 270 | $this->record_importers[ $plugin_name ] = $record_importer_class; |
| 271 | } |
| 272 | } |
| 273 | |
| 274 | $instances = []; |
| 275 | foreach ( $this->record_importers as $plugin_name => $class_name ) { |
| 276 | $instances[ $plugin_name ] = new $class_name( $this->logger ); |
| 277 | } |
| 278 | return $instances; |
| 279 | } |
| 280 | |
| 281 | private function remove_no_data_message( $id_site ) { |
| 282 | $had_traffic_key = 'SitesManagerHadTrafficInPast_' . (int) $id_site; |
| 283 | Option::set( $had_traffic_key, 1 ); |
| 284 | } |
| 285 | |
| 286 | private function is_plugin_unavailable( $plugin_name ) { |
| 287 | return ! Manager::getInstance()->isPluginActivated( $plugin_name ) |
| 288 | || ! Manager::getInstance()->isPluginLoaded( $plugin_name ) |
| 289 | || ! Manager::getInstance()->isPluginInFilesystem( $plugin_name ); |
| 290 | } |
| 291 | |
| 292 | private function on_error( \Exception $ex ) { |
| 293 | $this->logger->info( 'Unexpected Error: {ex}', [ 'ex' => $ex ] ); |
| 294 | |
| 295 | if ( $this->should_rethrow ) { |
| 296 | throw $ex; |
| 297 | } |
| 298 | } |
| 299 | } |
| 300 |