| 1 |
<?php |
| 2 |
/** |
| 3 |
* BotWriter — Cross-task duplicate detection for RSS / WordPress sources. |
| 4 |
* |
| 5 |
* Public API: |
| 6 |
* - botwriter_dedup_is_duplicate( array $candidate ): array |
| 7 |
* |
| 8 |
* Settings (whitelisted in includes/settings.php): |
| 9 |
* - botwriter_dedup_enabled (UI) "1" / "0" default "1" |
| 10 |
* - botwriter_dedup_title_threshold (UI) int 0-100 default 70 |
| 11 |
* - botwriter_dedup_window_days (hidden) int 1-30 default 7 |
| 12 |
* - botwriter_dedup_url_normalize (hidden) "1" / "0" default "1" |
| 13 |
* - botwriter_dedup_content_threshold (hidden) int 0-100 default 80 |
| 14 |
* - botwriter_dedup_max_history (hidden) int 50-1000 default 200 |
| 15 |
* - botwriter_dedup_scope (hidden) "task" / "site" default "site" |
| 16 |
* - botwriter_dedup_action (hidden) "skip"|"log_only" default "skip" |
| 17 |
*/ |
| 18 |
|
| 19 |
if (!defined('ABSPATH')) { |
| 20 |
exit; |
| 21 |
} |
| 22 |
|
| 23 |
/** |
| 24 |
* Return all dedup options merged with their defaults. |
| 25 |
* |
| 26 |
* @return array |
| 27 |
*/ |
| 28 |
function botwriter_dedup_get_options() { |
| 29 |
return array( |
| 30 |
'enabled' => get_option('botwriter_dedup_enabled', '1') === '1', |
| 31 |
'title_threshold' => max(0, min(100, intval(get_option('botwriter_dedup_title_threshold', 70)))), |
| 32 |
'content_threshold' => max(0, min(100, intval(get_option('botwriter_dedup_content_threshold', 80)))), |
| 33 |
'window_days' => max(1, min(30, intval(get_option('botwriter_dedup_window_days', 7)))), |
| 34 |
'max_history' => max(50, min(1000, intval(get_option('botwriter_dedup_max_history', 200)))), |
| 35 |
'url_normalize' => get_option('botwriter_dedup_url_normalize', '1') === '1', |
| 36 |
'scope' => get_option('botwriter_dedup_scope', 'site') === 'task' ? 'task' : 'site', |
| 37 |
'action' => get_option('botwriter_dedup_action', 'skip') === 'log_only' ? 'log_only' : 'skip', |
| 38 |
); |
| 39 |
} |
| 40 |
|
| 41 |
/** |
| 42 |
* Normalize a URL so equivalent links compare equal. |
| 43 |
* |
| 44 |
* - Lowercase scheme + host |
| 45 |
* - Strip "www." |
| 46 |
* - Drop fragment |
| 47 |
* - Drop common tracking query params (utm_*, fbclid, gclid, intcmp, ref, source, ...) |
| 48 |
* - Trim trailing slash from path |
| 49 |
* |
| 50 |
* @param string $url |
| 51 |
* @return string Normalized URL (or original on parse failure). |
| 52 |
*/ |
| 53 |
function botwriter_dedup_normalize_url($url) { |
| 54 |
if (!is_string($url) || $url === '') { |
| 55 |
return ''; |
| 56 |
} |
| 57 |
|
| 58 |
$url = trim($url); |
| 59 |
$parts = wp_parse_url($url); |
| 60 |
if (!is_array($parts) || empty($parts['host'])) { |
| 61 |
return strtolower($url); |
| 62 |
} |
| 63 |
|
| 64 |
$scheme = isset($parts['scheme']) ? strtolower($parts['scheme']) : 'https'; |
| 65 |
$host = strtolower($parts['host']); |
| 66 |
if (strpos($host, 'www.') === 0) { |
| 67 |
$host = substr($host, 4); |
| 68 |
} |
| 69 |
|
| 70 |
$path = isset($parts['path']) ? $parts['path'] : '/'; |
| 71 |
if ($path !== '/' && substr($path, -1) === '/') { |
| 72 |
$path = rtrim($path, '/'); |
| 73 |
} |
| 74 |
|
| 75 |
$query = ''; |
| 76 |
if (!empty($parts['query'])) { |
| 77 |
$pairs = array(); |
| 78 |
parse_str($parts['query'], $pairs); |
| 79 |
|
| 80 |
// Drop tracking params |
| 81 |
$drop_exact = array('fbclid', 'gclid', 'intcmp', 'ref', 'source', 'mc_cid', 'mc_eid', '_hsenc', '_hsmi'); |
| 82 |
foreach ($pairs as $k => $v) { |
| 83 |
$kl = strtolower($k); |
| 84 |
if (strpos($kl, 'utm_') === 0 || in_array($kl, $drop_exact, true)) { |
| 85 |
unset($pairs[$k]); |
| 86 |
} |
| 87 |
} |
| 88 |
|
| 89 |
if (!empty($pairs)) { |
| 90 |
ksort($pairs); |
| 91 |
$query = '?' . http_build_query($pairs); |
| 92 |
} |
| 93 |
} |
| 94 |
|
| 95 |
return $scheme . '://' . $host . $path . $query; |
| 96 |
} |
| 97 |
|
| 98 |
/** |
| 99 |
* Reduce a string to a comparable form: lowercase, strip tags, collapse whitespace. |
| 100 |
* |
| 101 |
* @param string $text |
| 102 |
* @return string |
| 103 |
*/ |
| 104 |
function botwriter_dedup_normalize_text($text) { |
| 105 |
if (!is_string($text)) { |
| 106 |
return ''; |
| 107 |
} |
| 108 |
$text = wp_strip_all_tags($text); |
| 109 |
$text = html_entity_decode($text, ENT_QUOTES | ENT_HTML5, 'UTF-8'); |
| 110 |
if (function_exists('mb_strtolower')) { |
| 111 |
$text = mb_strtolower($text, 'UTF-8'); |
| 112 |
} else { |
| 113 |
$text = strtolower($text); |
| 114 |
} |
| 115 |
$text = preg_replace('/\s+/u', ' ', $text); |
| 116 |
return trim((string) $text); |
| 117 |
} |
| 118 |
|
| 119 |
/** |
| 120 |
* Compute a similarity percentage (0-100) between two strings. |
| 121 |
* |
| 122 |
* Uses PHP's similar_text(); inputs are normalized first. For very long strings |
| 123 |
* we sample the first 2000 chars to keep the cost bounded. |
| 124 |
* |
| 125 |
* @param string $a |
| 126 |
* @param string $b |
| 127 |
* @return float |
| 128 |
*/ |
| 129 |
function botwriter_dedup_similarity($a, $b) { |
| 130 |
$a = botwriter_dedup_normalize_text($a); |
| 131 |
$b = botwriter_dedup_normalize_text($b); |
| 132 |
if ($a === '' || $b === '') { |
| 133 |
return 0.0; |
| 134 |
} |
| 135 |
if ($a === $b) { |
| 136 |
return 100.0; |
| 137 |
} |
| 138 |
if (strlen($a) > 2000) { $a = substr($a, 0, 2000); } |
| 139 |
if (strlen($b) > 2000) { $b = substr($b, 0, 2000); } |
| 140 |
|
| 141 |
$percent = 0.0; |
| 142 |
similar_text($a, $b, $percent); |
| 143 |
return (float) $percent; |
| 144 |
} |
| 145 |
|
| 146 |
/** |
| 147 |
* Fetch recent log rows used for dedup comparison. |
| 148 |
* |
| 149 |
* @param array $opts Options from botwriter_dedup_get_options(). |
| 150 |
* @param int|null $id_task Restrict to this task when scope === 'task'. |
| 151 |
* @return array Rows with link_post_original, aigenerated_title, aigenerated_content. |
| 152 |
*/ |
| 153 |
function botwriter_dedup_get_recent_rows($opts, $id_task = null) { |
| 154 |
global $wpdb; |
| 155 |
$table = $wpdb->prefix . 'botwriter_logs'; |
| 156 |
$table_sql = preg_replace('/[^A-Za-z0-9_]/', '', (string) $table); |
| 157 |
if ($table_sql === '') { |
| 158 |
return array(); |
| 159 |
} |
| 160 |
|
| 161 |
$since = gmdate('Y-m-d H:i:s', time() - ($opts['window_days'] * DAY_IN_SECONDS)); |
| 162 |
$limit = (int) $opts['max_history']; |
| 163 |
|
| 164 |
if ($opts['scope'] === 'task' && $id_task) { |
| 165 |
$rows = $wpdb->get_results( |
| 166 |
$wpdb->prepare( |
| 167 |
"SELECT link_post_original, aigenerated_title, aigenerated_content |
| 168 |
FROM `" . $table_sql . "` |
| 169 |
WHERE id_task = %d |
| 170 |
AND task_status = 'completed' |
| 171 |
AND created_at >= %s |
| 172 |
ORDER BY id DESC |
| 173 |
LIMIT %d", |
| 174 |
(int) $id_task, |
| 175 |
$since, |
| 176 |
$limit |
| 177 |
), |
| 178 |
ARRAY_A |
| 179 |
); |
| 180 |
} else { |
| 181 |
$rows = $wpdb->get_results( |
| 182 |
$wpdb->prepare( |
| 183 |
"SELECT link_post_original, aigenerated_title, aigenerated_content |
| 184 |
FROM `" . $table_sql . "` |
| 185 |
WHERE task_status = 'completed' |
| 186 |
AND created_at >= %s |
| 187 |
ORDER BY id DESC |
| 188 |
LIMIT %d", |
| 189 |
$since, |
| 190 |
$limit |
| 191 |
), |
| 192 |
ARRAY_A |
| 193 |
); |
| 194 |
} |
| 195 |
|
| 196 |
return is_array($rows) ? $rows : array(); |
| 197 |
} |
| 198 |
|
| 199 |
/** |
| 200 |
* Decide whether a candidate article (URL + title + optional content) should be |
| 201 |
* considered a duplicate of something already published recently. |
| 202 |
* |
| 203 |
* @param array $candidate { |
| 204 |
* @type string $url Source URL (link_post_original). |
| 205 |
* @type string $title Source article title. |
| 206 |
* @type string $content Source article content (optional). |
| 207 |
* @type int $id_task Current task id (optional, used when scope = 'task'). |
| 208 |
* } |
| 209 |
* @return array { |
| 210 |
* @type bool $is_duplicate |
| 211 |
* @type string $reason 'disabled' | 'url' | 'title' | 'content' | 'none' |
| 212 |
* @type float $score 0-100 similarity (only when is_duplicate) |
| 213 |
* @type string $matched_url URL of the matched previous post |
| 214 |
* @type string $action 'skip' | 'log_only' |
| 215 |
* } |
| 216 |
*/ |
| 217 |
function botwriter_dedup_is_duplicate(array $candidate) { |
| 218 |
$opts = botwriter_dedup_get_options(); |
| 219 |
|
| 220 |
$result = array( |
| 221 |
'is_duplicate' => false, |
| 222 |
'reason' => 'none', |
| 223 |
'score' => 0.0, |
| 224 |
'matched_url' => '', |
| 225 |
'action' => $opts['action'], |
| 226 |
); |
| 227 |
|
| 228 |
if (!$opts['enabled']) { |
| 229 |
$result['reason'] = 'disabled'; |
| 230 |
return $result; |
| 231 |
} |
| 232 |
|
| 233 |
$cand_url = isset($candidate['url']) ? (string) $candidate['url'] : ''; |
| 234 |
$cand_title = isset($candidate['title']) ? (string) $candidate['title'] : ''; |
| 235 |
$cand_content = isset($candidate['content']) ? (string) $candidate['content'] : ''; |
| 236 |
$id_task = isset($candidate['id_task']) ? (int) $candidate['id_task'] : null; |
| 237 |
|
| 238 |
$cand_url_norm = $opts['url_normalize'] ? botwriter_dedup_normalize_url($cand_url) : strtolower(trim($cand_url)); |
| 239 |
|
| 240 |
$rows = botwriter_dedup_get_recent_rows($opts, $id_task); |
| 241 |
if (empty($rows)) { |
| 242 |
return $result; |
| 243 |
} |
| 244 |
|
| 245 |
foreach ($rows as $row) { |
| 246 |
$row_url = isset($row['link_post_original']) ? (string) $row['link_post_original'] : ''; |
| 247 |
if ($row_url !== '' && $cand_url_norm !== '') { |
| 248 |
$row_url_norm = $opts['url_normalize'] ? botwriter_dedup_normalize_url($row_url) : strtolower(trim($row_url)); |
| 249 |
if ($row_url_norm !== '' && $row_url_norm === $cand_url_norm) { |
| 250 |
$result['is_duplicate'] = true; |
| 251 |
$result['reason'] = 'url'; |
| 252 |
$result['score'] = 100.0; |
| 253 |
$result['matched_url'] = $row_url; |
| 254 |
return $result; |
| 255 |
} |
| 256 |
} |
| 257 |
} |
| 258 |
|
| 259 |
if ($cand_title !== '' && $opts['title_threshold'] > 0) { |
| 260 |
foreach ($rows as $row) { |
| 261 |
$row_title = isset($row['aigenerated_title']) ? (string) $row['aigenerated_title'] : ''; |
| 262 |
if ($row_title === '') { |
| 263 |
continue; |
| 264 |
} |
| 265 |
$score = botwriter_dedup_similarity($cand_title, $row_title); |
| 266 |
if ($score >= $opts['title_threshold']) { |
| 267 |
$result['is_duplicate'] = true; |
| 268 |
$result['reason'] = 'title'; |
| 269 |
$result['score'] = $score; |
| 270 |
$result['matched_url'] = isset($row['link_post_original']) ? (string) $row['link_post_original'] : ''; |
| 271 |
return $result; |
| 272 |
} |
| 273 |
} |
| 274 |
} |
| 275 |
|
| 276 |
if ($cand_content !== '' && $opts['content_threshold'] > 0) { |
| 277 |
foreach ($rows as $row) { |
| 278 |
$row_content = isset($row['aigenerated_content']) ? (string) $row['aigenerated_content'] : ''; |
| 279 |
if ($row_content === '') { |
| 280 |
continue; |
| 281 |
} |
| 282 |
$score = botwriter_dedup_similarity($cand_content, $row_content); |
| 283 |
if ($score >= $opts['content_threshold']) { |
| 284 |
$result['is_duplicate'] = true; |
| 285 |
$result['reason'] = 'content'; |
| 286 |
$result['score'] = $score; |
| 287 |
$result['matched_url'] = isset($row['link_post_original']) ? (string) $row['link_post_original'] : ''; |
| 288 |
return $result; |
| 289 |
} |
| 290 |
} |
| 291 |
} |
| 292 |
|
| 293 |
return $result; |
| 294 |
} |
| 295 |
|
