DataConverters
2 years ago
Exceptions
4 years ago
Importers
2 years ago
Logger
4 years ago
Config.php
4 years ago
Geoip2.php
2 years ago
Importer.php
4 years ago
RecordInserter.php
4 years ago
Importer.php
276 lines
| 1 | <?php |
| 2 | |
| 3 | namespace WpMatomo\WpStatistics; |
| 4 | |
| 5 | use Piwik\ArchiveProcessor\Parameters; |
| 6 | use Piwik\Common; |
| 7 | use Piwik\Container\StaticContainer; |
| 8 | use Piwik\DataAccess\ArchiveWriter; |
| 9 | use Piwik\Date; |
| 10 | use Piwik\Option; |
| 11 | use Piwik\Period\Factory; |
| 12 | use Piwik\Plugin\Manager; |
| 13 | use Piwik\Segment; |
| 14 | use Piwik\Site; |
| 15 | use Psr\Log\LoggerInterface; |
| 16 | use Piwik\Archive\ArchiveInvalidator; |
| 17 | use WP_STATISTICS\DB; |
| 18 | use WpMatomo\Db\Settings; |
| 19 | use WpMatomo\ScheduledTasks; |
| 20 | use WpMatomo\WpStatistics\Exceptions\MaxEndDateReachedException; |
| 21 | use WpMatomo\WpStatistics\Importers\Actions\RecordImporter; |
| 22 | /** |
| 23 | * @package WpMatomo |
| 24 | * @subpackage WpStatisticsImport |
| 25 | * |
| 26 | * phpcs:disable WordPress.DB |
| 27 | */ |
| 28 | class Importer { |
| 29 | |
| 30 | const IS_IMPORTED_FROM_WPS_NUMERIC = 'WpStatisticsImporter_isImportedFromWpStatistics'; |
| 31 | |
| 32 | /** |
| 33 | * @var LoggerInterface |
| 34 | */ |
| 35 | private $logger; |
| 36 | |
| 37 | /** |
| 38 | * @var array|null |
| 39 | */ |
| 40 | private $record_importers; |
| 41 | |
| 42 | /** |
| 43 | * @var string |
| 44 | */ |
| 45 | private $no_data_message_removed = false; |
| 46 | |
| 47 | /** |
| 48 | * @var Date |
| 49 | */ |
| 50 | private $end_date = null; |
| 51 | |
| 52 | public function __construct( LoggerInterface $logger ) { |
| 53 | $this->logger = $logger; |
| 54 | $this->end_date = $this->get_ending_date(); |
| 55 | } |
| 56 | |
| 57 | /** |
| 58 | * Returns the first date in the matomo records |
| 59 | * |
| 60 | * @return Date |
| 61 | */ |
| 62 | protected function get_ending_date() { |
| 63 | global $wpdb; |
| 64 | $db_settings = new Settings(); |
| 65 | $table = $db_settings->prefix_table_name( 'log_visit' ); |
| 66 | $sql = <<<SQL |
| 67 | SELECT min(visit_last_action_time) from $table |
| 68 | SQL; |
| 69 | try { |
| 70 | $row = $wpdb->get_row( $sql, ARRAY_N ); |
| 71 | if ( ! empty( $row[0] ) ) { |
| 72 | return Date::factory( $row[0] ); |
| 73 | } else { |
| 74 | return Date::yesterday(); |
| 75 | } |
| 76 | } catch ( \Exception $e ) { |
| 77 | return Date::yesterday(); |
| 78 | } |
| 79 | } |
| 80 | |
| 81 | |
| 82 | /** |
| 83 | * Returns the first date in the wpStatistics data |
| 84 | * |
| 85 | * @return \Piwik\Date |
| 86 | */ |
| 87 | protected function get_started() { |
| 88 | global $wpdb; |
| 89 | $table = DB::table( 'visit' ); |
| 90 | $sql = <<<SQL |
| 91 | SELECT min(last_visit) from $table |
| 92 | SQL; |
| 93 | $row = $wpdb->get_row( $sql, ARRAY_N ); |
| 94 | return Date::factory( $row[0] ); |
| 95 | } |
| 96 | |
| 97 | /** |
| 98 | * Update the first date in the configuration. |
| 99 | * Otherwise records are here but the date picker does not allow to select these dates |
| 100 | * |
| 101 | * @param int $id_site |
| 102 | * @param Date $date |
| 103 | * |
| 104 | * @return void |
| 105 | */ |
| 106 | private function adjust_matomo_date( $id_site, Date $date ) { |
| 107 | global $wpdb; |
| 108 | $db_settings = new Settings(); |
| 109 | $prefix_table = $db_settings->prefix_table_name( 'site' ); |
| 110 | $wpdb->update( $prefix_table, [ 'ts_created' => $date->toString( 'Y-m-d h:i:s' ) ], [ 'idsite' => $id_site ] ); |
| 111 | } |
| 112 | |
| 113 | public function import( $id_site, $archive = true ) { |
| 114 | $end = $this->end_date; |
| 115 | $start = $this->get_started(); |
| 116 | |
| 117 | $this->adjust_matomo_date( $id_site, $start ); |
| 118 | try { |
| 119 | $this->no_data_message_removed = false; |
| 120 | |
| 121 | $end_plus_one = $end->addDay( 1 ); |
| 122 | |
| 123 | if ( $start->getTimestamp() >= $end_plus_one->getTimestamp() ) { |
| 124 | throw new \InvalidArgumentException( "Invalid date range, start date is later than end date: {$start},{$end}" ); |
| 125 | } |
| 126 | $record_importers = $this->get_record_importers(); |
| 127 | $site = new Site( $id_site ); |
| 128 | // phpcs:ignore Generic.CodeAnalysis.ForLoopWithTestFunctionCall.NotAllowed |
| 129 | for ( $date = $start; $date->getTimestamp() < $end_plus_one->getTimestamp(); $date = $date->addDay( 1 ) ) { |
| 130 | $this->logger->notice( |
| 131 | 'Importing data for date {date}...', |
| 132 | [ |
| 133 | 'date' => $date->toString(), |
| 134 | ] |
| 135 | ); |
| 136 | |
| 137 | try { |
| 138 | $this->import_day( $site, $date, $record_importers ); |
| 139 | } finally { |
| 140 | // force delete all tables in case they aren't all freed |
| 141 | \Piwik\DataTable\Manager::getInstance()->deleteAll(); |
| 142 | } |
| 143 | } |
| 144 | unset( $record_importers ); |
| 145 | } catch ( MaxEndDateReachedException $ex ) { |
| 146 | $this->logger->info( 'Max end date reached. This occurs in Matomo for WordPress installs when the importer tries to import days on or after the day Matomo for WordPress installed.' ); |
| 147 | |
| 148 | if ( true === $archive ) { |
| 149 | // by launching the archiver now the weekly, monthly and yearly archives should be generated right away and it won't |
| 150 | // take up to an hour. Also by running it on the cli we have less risk that this long running archiving process times out |
| 151 | $this->logger->info( 'Matomo Analytics starting the report generation of weekly, monthly and yearly reports. This may take a while.' ); |
| 152 | $scheduled_tasks = new ScheduledTasks( \WpMatomo::$settings ); |
| 153 | $scheduled_tasks->archive(); |
| 154 | } |
| 155 | $this->logger->info( 'Matomo Analytics report generation finished' ); |
| 156 | |
| 157 | return true; |
| 158 | } catch ( \Exception $ex ) { |
| 159 | $this->on_error( $ex ); |
| 160 | return true; |
| 161 | } |
| 162 | |
| 163 | return false; |
| 164 | } |
| 165 | |
| 166 | /** |
| 167 | * For use in record_importers that need to archive data for segments. |
| 168 | * |
| 169 | * @var RecordImporter[] $record_importers |
| 170 | * @throws MaxEndDateReachedException In case we have reach the end date to proceed. |
| 171 | */ |
| 172 | public function import_day( Site $site, Date $date, $record_importers ) { |
| 173 | if ( $this->end_date && $this->end_date->isEarlier( $date ) ) { |
| 174 | throw new MaxEndDateReachedException(); |
| 175 | } |
| 176 | $archive_writer = $this->make_archive_writer( $site, $date ); |
| 177 | $archive_writer->initNewArchive(); |
| 178 | |
| 179 | $record_inserter = new RecordInserter( $archive_writer ); |
| 180 | |
| 181 | foreach ( $record_importers as $plugin => $record_importer ) { |
| 182 | if ( ! $record_importer->supports_site() ) { |
| 183 | continue; |
| 184 | } |
| 185 | |
| 186 | $this->logger->info( |
| 187 | 'Importing data for the {plugin} plugin.', |
| 188 | [ |
| 189 | 'plugin' => $plugin, |
| 190 | ] |
| 191 | ); |
| 192 | |
| 193 | $record_importer->set_record_inserter( $record_inserter ); |
| 194 | |
| 195 | $record_importer->import_records( $date ); |
| 196 | |
| 197 | // since we recorded some data, at some time, remove the no data message |
| 198 | if ( ! $this->no_data_message_removed ) { |
| 199 | $this->remove_no_data_message( $site->getId() ); |
| 200 | $this->no_data_message_removed = true; |
| 201 | } |
| 202 | } |
| 203 | |
| 204 | $archive_writer->insertRecord( self::IS_IMPORTED_FROM_WPS_NUMERIC, 1 ); |
| 205 | $archive_writer->finalizeArchive(); |
| 206 | |
| 207 | $invalidator = StaticContainer::get( ArchiveInvalidator::class ); |
| 208 | $invalidator->markArchivesAsInvalidated( |
| 209 | [ $site->getId() ], |
| 210 | [ $date ], |
| 211 | 'week', |
| 212 | null, |
| 213 | false, |
| 214 | false, |
| 215 | null, |
| 216 | $ignore_purge_log_data_date = true |
| 217 | ); |
| 218 | |
| 219 | Common::destroy( $archive_writer ); |
| 220 | } |
| 221 | |
| 222 | private function make_archive_writer( Site $site, Date $date, $segment = '' ) { |
| 223 | $period = Factory::build( 'day', $date ); |
| 224 | $segment = new Segment( $segment, [ $site->getId() ] ); |
| 225 | |
| 226 | $params = new Parameters( $site, $period, $segment ); |
| 227 | return new ArchiveWriter( $params ); |
| 228 | } |
| 229 | |
| 230 | /** |
| 231 | * @return RecordImporter[] |
| 232 | * @throws \Exception In case importer has no plugin name. |
| 233 | */ |
| 234 | private function get_record_importers() { |
| 235 | if ( empty( $this->record_importers ) ) { |
| 236 | $record_importers = Config::get_importers(); |
| 237 | |
| 238 | $this->record_importers = []; |
| 239 | foreach ( $record_importers as $record_importer_class ) { |
| 240 | if ( ! defined( $record_importer_class . '::PLUGIN_NAME' ) ) { |
| 241 | throw new \Exception( "The $record_importer_class record importer is missing the PLUGIN_NAME constant." ); |
| 242 | } |
| 243 | |
| 244 | $namespace = explode( '\\', $record_importer_class ); |
| 245 | $plugin_name = array_pop( $namespace ); |
| 246 | if ( $this->is_plugin_unavailable( $record_importer_class::PLUGIN_NAME ) ) { |
| 247 | continue; |
| 248 | } |
| 249 | |
| 250 | $this->record_importers[ $plugin_name ] = $record_importer_class; |
| 251 | } |
| 252 | } |
| 253 | |
| 254 | $instances = []; |
| 255 | foreach ( $this->record_importers as $plugin_name => $class_name ) { |
| 256 | $instances[ $plugin_name ] = new $class_name( $this->logger ); |
| 257 | } |
| 258 | return $instances; |
| 259 | } |
| 260 | |
| 261 | private function remove_no_data_message( $id_site ) { |
| 262 | $had_traffic_key = 'SitesManagerHadTrafficInPast_' . (int) $id_site; |
| 263 | Option::set( $had_traffic_key, 1 ); |
| 264 | } |
| 265 | |
| 266 | private function is_plugin_unavailable( $plugin_name ) { |
| 267 | return ! Manager::getInstance()->isPluginActivated( $plugin_name ) |
| 268 | || ! Manager::getInstance()->isPluginLoaded( $plugin_name ) |
| 269 | || ! Manager::getInstance()->isPluginInFilesystem( $plugin_name ); |
| 270 | } |
| 271 | |
| 272 | private function on_error( \Exception $ex ) { |
| 273 | $this->logger->info( 'Unexpected Error: {ex}', [ 'ex' => $ex ] ); |
| 274 | } |
| 275 | } |
| 276 |