surecookie / inc / modules / services / services-source.php

services-source.php in SureCookie – GDPR Cookie Consent Banner, Cookie Scanner & Script Blocking trunk, at inc/modules/services/services-source.php

579 lines 18.1 KB

Raw Download Zip

1	<?php
2	/**
3	* Services Source (unified catalog).
4	*
5	* Single fetch/cache authority for the unified `dataset/services.json` catalog
6	* (1.3.0+), carrying each service's blocking patterns AND declared cookies under
7	* one slug. Replaces the two near-identical pipelines that fetched
8	* `blocking-scripts.json` and `service-cookies.json` separately: `Known_Scripts`
9	* and `Service_Cookies_Source` now project their views from this one source - one
10	* HTTP fetch, one transient, one file cache, one bundled floor.
11	*
12	* Resolution order (hot path first): request transient -> file cache (merged over
13	* the bundled floor, remote wins per slug) -> bundled floor. The remote is never
14	* fetched inline; it is warmed off-request by the script-blocking cron.
15	*
16	* @package SureCookie\Inc\Modules\Services
17	* @since 1.3.0
18	*/
19
20	namespace SureCookie\Inc\Modules\Services;
21
22	use SureCookie\Inc\Functions\Cache;
23	use SureCookie\Inc\Functions\Cookie_Identity;
24	use SureCookie\Inc\Functions\Helper;
25	use SureCookie\Inc\Traits\GetInstance;
26	use SureCookie\Inc\Utils\Logger;
27
28	if ( ! defined( 'ABSPATH' ) ) {
29	exit; // Exit if accessed directly.
30	}
31
32	/**
33	* Services_Source
34	*
35	* Loads, caches and refreshes the unified service catalog and projects it into
36	* the blocking view (category => slug => patterns) and the declared-cookie view
37	* (slug => cookies).
38	*
39	* @since 1.3.0
40	*/
41	class Services_Source {
42	use GetInstance;
43
44	/**
45	* Remote dataset path (appended to the agent app URL).
46	*/
47	protected const REMOTE_FILE_PATH = 'dataset/services.json';
48
49	/**
50	* Transient key for the merged unified catalog.
51	*/
52	protected const CACHE_KEY = 'surecookie_services';
53
54	/**
55	* File cache path (relative to the uploads cache dir).
56	*/
57	protected const CACHE_FILE = 'services/services.json';
58
59	/**
60	* Bundled baseline catalog shipped with the plugin (full core embed set:
61	* patterns + declared cookies). Guarantees blocking + declared cookies work
62	* offline / before the first remote warm; the remote catalog wins per slug.
63	*/
64	protected const BUNDLED_FILE = 'inc/modules/services/data/services.json';
65
66	/**
67	* Transient duration in seconds (24 hours).
68	*/
69	protected const CACHE_DURATION = DAY_IN_SECONDS;
70
71	/**
72	* File cache duration in seconds (7 days).
73	*/
74	protected const FILE_CACHE_DURATION = WEEK_IN_SECONDS;
75
76	/**
77	* Per-request cache of the resolved unified catalog (slug => entry).
78	*
79	* @var array<string, array<string, mixed>>\|null
80	*/
81	private ?array $catalog = null;
82
83	/**
84	* Resolve the unified catalog: slug => {label, category, gcm_compatible?,
85	* patterns keyed by Pattern_Kinds bucket, cookies:[...]}. `_meta` is stripped.
86	*
87	* First-party placeholder domains are resolved here, on the way out of the
88	* caches, so every consumer (Known Services REST, install(), declared-cookie
89	* seeding, the provider index) sees this site's host. See
90	* {@see self::resolve_first_party()} for why here and not one layer down.
91	*
92	* @since 1.3.0
93	* @return array<string, array<string, mixed>>
94	*/
95	public function get_catalog(): array {
96	if ( $this->catalog !== null ) {
97	return $this->catalog;
98	}
99
100	$this->catalog = $this->resolve_first_party( $this->load_catalog() );
101
102	return $this->catalog;
103	}
104
105	/**
106	* Project the catalog into the blocking view consumed by Known_Scripts /
107	* Blocker: category => slug => {label, one array per Pattern_Kinds bucket,
108	* gcm_compatible?}. Only services with at least one pattern are emitted.
109	*
110	* @since 1.3.0
111	* @return array<string, array<string, array<string, mixed>>>
112	*/
113	public function get_blocking_view(): array {
114	return $this->blocking_view( $this->get_catalog() );
115	}
116
117	/**
118	* Blocking view built from the bundled floor only (offline baseline). Used as
119	* the guaranteed floor and by the bundled-fallback tests.
120	*
121	* @since 1.3.0
122	* @return array<string, array<string, array<string, mixed>>>
123	*/
124	public function get_bundled_blocking_view(): array {
125	return $this->blocking_view( $this->get_bundled_data() );
126	}
127
128	/**
129	* Project the catalog into the declared-cookie view consumed by
130	* Service_Cookies_Source / Declared_Cookies: slug => [cookie rows]. Only
131	* services that declare at least one cookie are emitted.
132	*
133	* @since 1.3.0
134	* @return array<string, array<int, array<string, mixed>>>
135	*/
136	public function get_cookies_view(): array {
137	$view = [];
138
139	foreach ( $this->get_catalog() as $slug => $service ) {
140	if ( ! is_array( $service ) ) {
141	continue;
142	}
143
144	$cookies = $service['cookies'] ?? [];
145	if ( is_array( $cookies ) && $cookies !== [] ) {
146	$view[ $slug ] = array_values( $cookies );
147	}
148	}
149
150	return $view;
151	}
152
153	/**
154	* Fetch the remote catalog, validate it, merge over the bundled floor and
155	* persist it to the file cache. Called off the request path by the cron.
156	*
157	* On any failure the existing cache/floor is left untouched so neither
158	* blocking nor declared cookies regress. Busts the downstream transients so
159	* the projected views and the presets REST response rebuild.
160	*
161	* @since 1.3.0
162	* @return void
163	*/
164	public function refresh_from_remote(): void {
165	$api_data = $this->fetch_from_api();
166
167	if ( $api_data === null \|\| empty( $api_data ) ) {
168	return;
169	}
170
171	$merged = $this->merge_catalogs( $this->get_bundled_data(), $api_data );
172
173	$this->set_file_cache( $merged );
174
175	delete_transient( self::CACHE_KEY );
176
177	$this->catalog = null;
178	}
179
180	/**
181	* Clear the cached catalog, falling back to the bundled floor so blocking and
182	* declared cookies keep running until the next request repopulates.
183	*
184	* @since 1.3.0
185	* @return void
186	*/
187	public function clear_cache(): void {
188	delete_transient( self::CACHE_KEY );
189	Cache::delete_file( self::CACHE_FILE );
190	$this->catalog = null;
191	}
192
193	/**
194	* Load the plugin-bundled unified catalog (with `_meta` stripped).
195	*
196	* @since 1.3.0
197	* @return array<string, array<string, mixed>>
198	*/
199	public function get_bundled_data(): array {
200	$path = SURECOOKIE_DIR . self::BUNDLED_FILE;
201
202	if ( ! file_exists( $path ) ) {
203	return [];
204	}
205
206	$data = wp_json_file_decode( $path, [ 'associative' => true ] );
207	$data = is_array( $data ) ? $data : [];
208	unset( $data['_meta'] );
209
210	return $data;
211	}
212
213	/**
214	* One catalog entry's patterns, counted the way the admin may state them.
215	*
216	* `blockable` is true only when a pass can act on something the service
217	* declares. A font or image host is a real pattern and still belongs in the
218	* listing, but calling it blocked would promise gating the engine cannot do,
219	* so it is counted separately as `unblockableCount`.
220	*
221	* @param array<string, mixed> $service Catalog entry.
222	* @since 1.5.0
223	* @return array{scriptCount: int, iframeCount: int, styleCount: int, unblockableCount: int, blockable: bool}
224	*/
225	public static function pattern_summary( array $service ): array {
226	$resources = self::pattern_lists( $service );
227	$blockable = 0;
228	$unblockable = 0;
229
230	foreach ( array_keys( Pattern_Kinds::enforced() ) as $bucket ) {
231	$blockable += count( $resources[ $bucket ] );
232	}
233
234	foreach ( array_keys( Pattern_Kinds::unenforced() ) as $bucket ) {
235	$unblockable += count( $resources[ $bucket ] );
236	}
237
238	return [
239	'scriptCount' => count( $resources['scripts'] ),
240	'iframeCount' => count( $resources['iframes'] ),
241	'styleCount' => count( $resources['styles'] ),
242	'unblockableCount' => $unblockable,
243	'blockable' => $blockable > 0,
244	];
245	}
246
247	/**
248	* One catalog entry's patterns, by bucket, with every bucket present.
249	*
250	* @param array<string, mixed> $service Catalog entry.
251	* @since 1.5.0
252	* @return array<string, array<int, string>>
253	*/
254	public static function pattern_lists( array $service ): array {
255	$resources = [];
256
257	foreach ( Pattern_Kinds::buckets() as $bucket ) {
258	$resources[ $bucket ] = array_values( (array) ( $service['patterns'][ $bucket ] ?? [] ) );
259	}
260
261	return $resources;
262	}
263
264	/**
265	* Load the catalog as authored, from the transient, the file cache or the
266	* bundled floor.
267	*
268	* @since 1.3.0
269	* @return array<string, array<string, mixed>>
270	*/
271	private function load_catalog(): array {
272	// 1. Transient already holds the merged catalog - the hot path.
273	$cached = get_transient( self::CACHE_KEY );
274	if ( $cached !== false && is_array( $cached ) && ! empty( $cached ) ) {
275	return $cached;
276	}
277
278	$bundled = $this->get_bundled_data();
279
280	// 2. File cache, merged over the bundled floor (remote wins per slug).
281	$file_cached = $this->get_file_cache();
282	if ( $file_cached !== null ) {
283	$merged = $this->merge_catalogs( $bundled, $file_cached );
284	set_transient( self::CACHE_KEY, $merged, self::CACHE_DURATION );
285	return $merged;
286	}
287
288	// 3. Both caches cold: serve the bundled floor now and warm the remote
289	// off-request (a near-immediate one-off, so a fresh install does not wait
290	// for the daily cron) - the page load is never blocked on the network.
291	set_transient( self::CACHE_KEY, $bundled, self::CACHE_DURATION );
292	wp_schedule_single_event( time() + MINUTE_IN_SECONDS, Cron::REFRESH_HOOK );
293
294	return $bundled;
295	}
296
297	/**
298	* Substitute this site's host for the catalog's first-party placeholder.
299	*
300	* Applied AFTER the caches are populated, never before: the transient and
301	* 7-day file cache are shared across every blog of a multisite network, so
302	* baking one blog's host in would leak it to the others and survive a domain
303	* change for a week. Resolving on the way out means a moved or cloned site is
304	* correct on its next request.
305	*
306	* Touched rows are flagged {@see Cookie_Identity::FIRST_PARTY_FLAG}: the
307	* substituted host is still not necessarily where the tag wrote the cookie
308	* (Analytics scopes `_ga` to the registrable domain, so `shop.example.com`
309	* observes `.example.com`), so the dedup paths must know which domains were
310	* inferred rather than authored.
311	*
312	* @param array<string, array<string, mixed>> $catalog Catalog as authored.
313	* @since 1.3.0
314	* @return array<string, array<string, mixed>>
315	*/
316	private function resolve_first_party( array $catalog ): array {
317	foreach ( $catalog as $slug => $service ) {
318	if ( ! is_array( $service ) \|\| ! is_array( $service['cookies'] ?? null ) ) {
319	continue;
320	}
321
322	foreach ( $service['cookies'] as $index => $cookie ) {
323	if ( ! is_array( $cookie ) \|\| ! Cookie_Identity::is_placeholder( (string) ( $cookie['domain'] ?? '' ) ) ) {
324	continue;
325	}
326
327	$catalog[ $slug ]['cookies'][ $index ]['domain'] = Cookie_Identity::resolve( (string) $cookie['domain'] );
328	$catalog[ $slug ]['cookies'][ $index ][ Cookie_Identity::FIRST_PARTY_FLAG ] = true;
329	}
330	}
331
332	return $catalog;
333	}
334
335	/**
336	* Project a unified catalog into the blocking view (category => slug =>
337	* {label, one array per Pattern_Kinds bucket, gcm_compatible?}); services
338	* without patterns are omitted.
339	*
340	* Every bucket is emitted, including the ones no pass reads: a consumer that
341	* only cares what gets rewritten filters on `Pattern_Kinds::enforced()`,
342	* while the admin surfaces need the rest to say what is NOT blocked.
343	*
344	* @param array<string, array<string, mixed>> $catalog Unified catalog.
345	* @since 1.3.0
346	* @return array<string, array<string, array<string, mixed>>>
347	*/
348	private function blocking_view( array $catalog ): array {
349	$view = [];
350
351	foreach ( $catalog as $slug => $service ) {
352	if ( ! is_array( $service ) ) {
353	continue;
354	}
355
356	$entry = [ 'label' => (string) ( $service['label'] ?? $slug ) ];
357	$empty = true;
358
359	foreach ( Pattern_Kinds::buckets() as $bucket ) {
360	$entry[ $bucket ] = array_values( (array) ( $service['patterns'][ $bucket ] ?? [] ) );
361	$empty = $empty && $entry[ $bucket ] === [];
362	}
363
364	if ( $empty ) {
365	continue;
366	}
367
368	$category = is_string( $service['category'] ?? null ) ? $service['category'] : 'uncategorized';
369
370	if ( isset( $service['gcm_compatible'] ) ) {
371	$entry['gcm_compatible'] = (bool) $service['gcm_compatible'];
372	}
373
374	$view[ $category ][ $slug ] = $entry;
375	}
376
377	return $view;
378	}
379
380	/**
381	* Merge two unified catalogs at the slug level, `$override` winning per slug.
382	*
383	* Deliberately a replace and not a deep merge: the remote has to stay able to
384	* retire a cookie row or a pattern that has gone wrong. The one exception is
385	* blocking patterns, where the bundled entry is a floor - the validator admits
386	* a remote row on valid cookies alone, so a cookies-only row would otherwise
387	* silently stop blocking a service that every bundled entry has patterns for.
388	* Emptiness is treated as an incomplete row, never as "stop blocking this".
389	*
390	* @param array<string, array<string, mixed>> $base Bundled floor.
391	* @param array<string, array<string, mixed>> $override Remote/cached catalog.
392	* @since 1.3.0
393	* @return array<string, array<string, mixed>>
394	*/
395	private function merge_catalogs( array $base, array $override ): array {
396	unset( $override['_meta'] );
397
398	foreach ( $override as $slug => $entry ) {
399	if ( is_array( $entry ) && isset( $base[ $slug ] ) && ! empty( $base[ $slug ]['patterns'] ) ) {
400	$patterns = is_array( $entry['patterns'] ?? null ) ? $entry['patterns'] : [];
401	$declared = false;
402
403	// Only the ENFORCED buckets count, as they did before `styles`
404	// and `media` existed. A remote row carrying nothing but a
405	// media classification is still an incomplete row, and letting
406	// it satisfy this guard would drop the floor's blocking
407	// patterns for that service entirely.
408	foreach ( array_keys( Pattern_Kinds::enforced() ) as $bucket ) {
409	$declared = $declared \|\| ! empty( $patterns[ $bucket ] );
410	}
411
412	$entry['patterns'] = $declared
413	? self::reclassify_patterns( $patterns, (array) $base[ $slug ]['patterns'] )
414	: $base[ $slug ]['patterns'];
415	}
416
417	$base[ $slug ] = $entry;
418	}
419
420	return $base;
421	}
422
423	/**
424	* Move a remote pattern into the bucket the bundled floor files it under.
425	*
426	* The remote catalog still publishes every pattern as a script, so a
427	* stylesheet or image host would come back as one and the admin would again
428	* report it as blocked by a pass that cannot see a `<link>` or an `<img>`.
429	* The remote stays free to add and retire patterns; only where it repeats a
430	* pattern this plugin has already classified does the floor win.
431	*
432	* A pattern is moved only when the floor files it in exactly ONE bucket and
433	* the remote put it somewhere else. Nine bundled services deliberately
434	* declare the same host under both `scripts` and `iframes` (reCAPTCHA,
435	* Wistia, Stripe and the video players): with a single-valued home the
436	* later bucket won and every remote copy was routed to it, emptying the
437	* other array. Pooling means blocking survived that, but the counts the
438	* admin screens read did not.
439	*
440	* @param array<string, mixed> $remote Remote patterns, by bucket.
441	* @param array<string, mixed> $bundled Bundled patterns, by bucket.
442	* @since 1.5.0
443	* @return array<string, array<int, string>>
444	*/
445	private static function reclassify_patterns( array $remote, array $bundled ): array {
446	$home = [];
447
448	foreach ( Pattern_Kinds::buckets() as $bucket ) {
449	foreach ( (array) ( $bundled[ $bucket ] ?? [] ) as $pattern ) {
450	$home[ strtolower( trim( (string) $pattern ) ) ][ $bucket ] = true;
451	}
452	}
453
454	$merged = array_fill_keys( Pattern_Kinds::buckets(), [] );
455
456	foreach ( Pattern_Kinds::buckets() as $bucket ) {
457	foreach ( (array) ( $remote[ $bucket ] ?? [] ) as $pattern ) {
458	$pattern = (string) $pattern;
459	$buckets = $home[ strtolower( trim( $pattern ) ) ] ?? [];
460
461	// Unknown to the floor, or filed there under this same bucket:
462	// leave it where the remote put it. Only an unambiguous
463	// disagreement relocates, so a floor entry that spans buckets
464	// never collapses the remote's placement into one of them.
465	$target = count( $buckets ) === 1 && ! isset( $buckets[ $bucket ] )
466	? (string) array_key_first( $buckets )
467	: $bucket;
468
469	$merged[ $target ][] = $pattern;
470	}
471	}
472
473	return array_map(
474	static fn( array $patterns ): array => array_values( array_unique( $patterns ) ),
475	$merged
476	);
477	}
478
479	/**
480	* Retrieve the catalog from the file cache.
481	*
482	* @since 1.3.0
483	* @return array<string, array<string, mixed>>\|null
484	*/
485	private function get_file_cache(): ?array {
486	$cache_raw = Cache::get_file( self::CACHE_FILE );
487
488	if ( ! is_string( $cache_raw ) \|\| $cache_raw === '' ) {
489	return null;
490	}
491
492	$decoded = json_decode( $cache_raw, true );
493
494	if ( ! is_array( $decoded ) \|\| ! isset( $decoded['data'] ) \|\| ! is_array( $decoded['data'] ) ) {
495	return null;
496	}
497
498	if ( ! isset( $decoded['timestamp'] ) \|\| ! is_int( $decoded['timestamp'] ) ) {
499	return null;
500	}
501
502	if ( time() - $decoded['timestamp'] > self::FILE_CACHE_DURATION ) {
503	return null;
504	}
505
506	return $decoded['data'];
507	}
508
509	/**
510	* Store the catalog in the file cache.
511	*
512	* @param array<string, array<string, mixed>> $catalog Catalog data.
513	* @since 1.3.0
514	* @return void
515	*/
516	private function set_file_cache( array $catalog ): void {
517	$payload = wp_json_encode(
518	[
519	'timestamp' => time(),
520	'data' => $catalog,
521	]
522	);
523
524	if ( ! is_string( $payload ) ) {
525	return;
526	}
527
528	if ( ! Cache::store_file( self::CACHE_FILE, $payload ) ) {
529	Logger::get_instance()->log( 'Unable to write services file cache.' );
530	}
531	}
532
533	/**
534	* Fetch and validate the remote unified catalog.
535	*
536	* @since 1.3.0
537	* @return array<string, array<string, mixed>>\|null Validated catalog, or null on failure.
538	*/
539	private function fetch_from_api(): ?array {
540	$url = Helper::get_agent_app_url() . self::REMOTE_FILE_PATH;
541
542	if ( ! Dataset_Validator::is_allowed_url( $url ) ) {
543	Logger::get_instance()->log( 'Services API URL not allowed: ' . $url );
544	return null;
545	}
546
547	$response = wp_remote_get(
548	$url,
549	[
550	'timeout' => 10,
551	'headers' => [
552	'Accept' => 'application/json',
553	],
554	]
555	);
556
557	if ( is_wp_error( $response ) ) {
558	Logger::get_instance()->log( 'Services API request failed: ' . $response->get_error_message() );
559	return null;
560	}
561
562	$response_code = wp_remote_retrieve_response_code( $response );
563
564	if ( $response_code !== 200 ) {
565	Logger::get_instance()->log( 'Services API returned status: ' . $response_code );
566	return null;
567	}
568
569	$data = json_decode( wp_remote_retrieve_body( $response ), true );
570
571	if ( ! is_array( $data ) ) {
572	Logger::get_instance()->log( 'Services API returned invalid JSON.' );
573	return null;
574	}
575
576	return Dataset_Validator::validate_services( $data );
577	}
578	}
579