| 1 |
<?php |
| 2 |
/** |
| 3 |
* MxChat Chunker - Text chunking utility for RAG optimization |
| 4 |
* |
| 5 |
* Splits large content into chunks for improved semantic retrieval. |
| 6 |
* All chunks for a URL are reassembled before sending to AI, so no overlap is needed. |
| 7 |
* |
| 8 |
* @package MxChat |
| 9 |
* @since 2.6.3 |
| 10 |
*/ |
| 11 |
|
| 12 |
if (!defined('ABSPATH')) { |
| 13 |
exit; // Exit if accessed directly |
| 14 |
} |
| 15 |
|
| 16 |
class MxChat_Chunker { |
| 17 |
|
| 18 |
/** |
| 19 |
* Maximum characters per chunk |
| 20 |
* @var int |
| 21 |
*/ |
| 22 |
private $chunk_size; |
| 23 |
|
| 24 |
/** |
| 25 |
* Constructor |
| 26 |
* |
| 27 |
* @param int $chunk_size Characters per chunk (default 4000 ≈ 1000 tokens) |
| 28 |
*/ |
| 29 |
public function __construct($chunk_size = 4000) { |
| 30 |
$this->chunk_size = max(1000, min(10000, intval($chunk_size))); |
| 31 |
} |
| 32 |
|
| 33 |
/** |
| 34 |
* Get chunking settings from WordPress options |
| 35 |
* |
| 36 |
* @return array Array with chunk_size and chunking_enabled |
| 37 |
*/ |
| 38 |
public static function get_settings() { |
| 39 |
$options = get_option('mxchat_options', array()); |
| 40 |
|
| 41 |
return array( |
| 42 |
'chunk_size' => isset($options['chunk_size']) ? intval($options['chunk_size']) : 4000, |
| 43 |
'chunking_enabled' => isset($options['chunking_enabled']) ? (bool) $options['chunking_enabled'] : true |
| 44 |
); |
| 45 |
} |
| 46 |
|
| 47 |
/** |
| 48 |
* Create a chunker instance with settings from WordPress options |
| 49 |
* |
| 50 |
* @return MxChat_Chunker |
| 51 |
*/ |
| 52 |
public static function from_settings() { |
| 53 |
$settings = self::get_settings(); |
| 54 |
return new self($settings['chunk_size']); |
| 55 |
} |
| 56 |
|
| 57 |
/** |
| 58 |
* Check if content should be chunked |
| 59 |
* |
| 60 |
* @param string $text Content to evaluate |
| 61 |
* @return bool True if content should be chunked |
| 62 |
*/ |
| 63 |
public function should_chunk($text) { |
| 64 |
$settings = self::get_settings(); |
| 65 |
|
| 66 |
// Check if chunking is enabled globally |
| 67 |
if (!$settings['chunking_enabled']) { |
| 68 |
return false; |
| 69 |
} |
| 70 |
|
| 71 |
// Only chunk if content exceeds chunk size (characters, not bytes — |
| 72 |
// multibyte scripts would otherwise hit the limit 3x early) |
| 73 |
return mb_strlen($text) > $this->chunk_size; |
| 74 |
} |
| 75 |
|
| 76 |
/** |
| 77 |
* Split text into chunks |
| 78 |
* |
| 79 |
* Algorithm: |
| 80 |
* 1. Split content by paragraph boundaries |
| 81 |
* 2. Accumulate paragraphs until chunk size exceeded |
| 82 |
* 3. Start new chunk (no overlap needed since we reassemble all chunks) |
| 83 |
* |
| 84 |
* @param string $text Content to chunk |
| 85 |
* @return array Array of chunk strings |
| 86 |
*/ |
| 87 |
public function chunk_text($text) { |
| 88 |
// Handle empty content |
| 89 |
if (empty(trim($text))) { |
| 90 |
return array(); |
| 91 |
} |
| 92 |
|
| 93 |
// Handle content smaller than chunk size - return as single chunk |
| 94 |
if (mb_strlen($text) <= $this->chunk_size) { |
| 95 |
return array(trim($text)); |
| 96 |
} |
| 97 |
|
| 98 |
$chunks = array(); |
| 99 |
$paragraphs = preg_split('/\n\s*\n/', $text); // Split by paragraph boundaries |
| 100 |
$current_chunk = ''; |
| 101 |
|
| 102 |
foreach ($paragraphs as $paragraph) { |
| 103 |
$paragraph = trim($paragraph); |
| 104 |
|
| 105 |
// Skip empty paragraphs |
| 106 |
if (empty($paragraph)) { |
| 107 |
continue; |
| 108 |
} |
| 109 |
|
| 110 |
// Calculate size if we add this paragraph |
| 111 |
$separator = empty($current_chunk) ? '' : "\n\n"; |
| 112 |
$potential_size = mb_strlen($current_chunk) + mb_strlen($separator) + mb_strlen($paragraph); |
| 113 |
|
| 114 |
// If adding this paragraph exceeds chunk size |
| 115 |
if ($potential_size > $this->chunk_size && !empty($current_chunk)) { |
| 116 |
// Save current chunk and start fresh |
| 117 |
$chunks[] = trim($current_chunk); |
| 118 |
$current_chunk = $paragraph; |
| 119 |
} else { |
| 120 |
// Add paragraph to current chunk |
| 121 |
$current_chunk .= $separator . $paragraph; |
| 122 |
} |
| 123 |
|
| 124 |
// Handle very long paragraphs that exceed chunk size on their own |
| 125 |
if (mb_strlen($current_chunk) > $this->chunk_size) { |
| 126 |
$split_chunks = $this->split_long_paragraph($current_chunk); |
| 127 |
|
| 128 |
// Add all but the last split chunk |
| 129 |
for ($i = 0; $i < count($split_chunks) - 1; $i++) { |
| 130 |
$chunks[] = trim($split_chunks[$i]); |
| 131 |
} |
| 132 |
|
| 133 |
// Keep the last one as current chunk (may accumulate more) |
| 134 |
$current_chunk = $split_chunks[count($split_chunks) - 1]; |
| 135 |
} |
| 136 |
} |
| 137 |
|
| 138 |
// Add final chunk if not empty |
| 139 |
if (!empty(trim($current_chunk))) { |
| 140 |
$chunks[] = trim($current_chunk); |
| 141 |
} |
| 142 |
|
| 143 |
return $chunks; |
| 144 |
} |
| 145 |
|
| 146 |
/** |
| 147 |
* Split a very long paragraph into chunks |
| 148 |
* |
| 149 |
* Used when a single paragraph exceeds chunk size. |
| 150 |
* Splits by sentences, then by words if needed. |
| 151 |
* |
| 152 |
* @param string $paragraph Long paragraph to split |
| 153 |
* @return array Array of chunk strings |
| 154 |
*/ |
| 155 |
private function split_long_paragraph($paragraph) { |
| 156 |
$chunks = array(); |
| 157 |
|
| 158 |
// First try splitting by sentences |
| 159 |
$sentences = preg_split('/(?<=[.!?])\s+/', $paragraph); |
| 160 |
$current_chunk = ''; |
| 161 |
|
| 162 |
foreach ($sentences as $sentence) { |
| 163 |
$sentence = trim($sentence); |
| 164 |
if (empty($sentence)) { |
| 165 |
continue; |
| 166 |
} |
| 167 |
|
| 168 |
// If single sentence is too long, split by words |
| 169 |
if (mb_strlen($sentence) > $this->chunk_size) { |
| 170 |
if (!empty($current_chunk)) { |
| 171 |
$chunks[] = trim($current_chunk); |
| 172 |
$current_chunk = ''; |
| 173 |
} |
| 174 |
|
| 175 |
// Split long sentence by words |
| 176 |
$word_chunks = $this->split_by_words($sentence); |
| 177 |
foreach ($word_chunks as $word_chunk) { |
| 178 |
$chunks[] = $word_chunk; |
| 179 |
} |
| 180 |
continue; |
| 181 |
} |
| 182 |
|
| 183 |
$separator = empty($current_chunk) ? '' : ' '; |
| 184 |
$potential_size = mb_strlen($current_chunk) + mb_strlen($separator) + mb_strlen($sentence); |
| 185 |
|
| 186 |
if ($potential_size > $this->chunk_size && !empty($current_chunk)) { |
| 187 |
$chunks[] = trim($current_chunk); |
| 188 |
$current_chunk = $sentence; |
| 189 |
} else { |
| 190 |
$current_chunk .= $separator . $sentence; |
| 191 |
} |
| 192 |
} |
| 193 |
|
| 194 |
if (!empty(trim($current_chunk))) { |
| 195 |
$chunks[] = trim($current_chunk); |
| 196 |
} |
| 197 |
|
| 198 |
return $chunks; |
| 199 |
} |
| 200 |
|
| 201 |
/** |
| 202 |
* Split text by words when sentences are too long |
| 203 |
* |
| 204 |
* Last resort splitting method for very long unbroken text. |
| 205 |
* |
| 206 |
* @param string $text Text to split |
| 207 |
* @return array Array of chunk strings |
| 208 |
*/ |
| 209 |
private function split_by_words($text) { |
| 210 |
$chunks = array(); |
| 211 |
$words = preg_split('/\s+/', $text); |
| 212 |
$current_chunk = ''; |
| 213 |
|
| 214 |
foreach ($words as $word) { |
| 215 |
// Last-resort hard split: a single "word" larger than the chunk size |
| 216 |
// (space-free scripts like Japanese/Chinese/Thai are one token to \s+) |
| 217 |
// must be split by characters or it is emitted whole at any size. |
| 218 |
if (mb_strlen($word) > $this->chunk_size) { |
| 219 |
if (!empty(trim($current_chunk))) { |
| 220 |
$chunks[] = trim($current_chunk); |
| 221 |
$current_chunk = ''; |
| 222 |
} |
| 223 |
$pieces = $this->mb_hard_split($word, $this->chunk_size); |
| 224 |
// Emit all full pieces; the last may still accumulate following words |
| 225 |
$current_chunk = array_pop($pieces); |
| 226 |
foreach ($pieces as $piece) { |
| 227 |
$chunks[] = $piece; |
| 228 |
} |
| 229 |
continue; |
| 230 |
} |
| 231 |
|
| 232 |
$separator = empty($current_chunk) ? '' : ' '; |
| 233 |
$potential_size = mb_strlen($current_chunk) + mb_strlen($separator) + mb_strlen($word); |
| 234 |
|
| 235 |
if ($potential_size > $this->chunk_size && !empty($current_chunk)) { |
| 236 |
$chunks[] = trim($current_chunk); |
| 237 |
$current_chunk = $word; |
| 238 |
} else { |
| 239 |
$current_chunk .= $separator . $word; |
| 240 |
} |
| 241 |
} |
| 242 |
|
| 243 |
if (!empty(trim($current_chunk))) { |
| 244 |
$chunks[] = trim($current_chunk); |
| 245 |
} |
| 246 |
|
| 247 |
return $chunks; |
| 248 |
} |
| 249 |
|
| 250 |
/** |
| 251 |
* Split a string into fixed-size character pieces (multibyte-safe) |
| 252 |
* |
| 253 |
* @param string $text Text to split |
| 254 |
* @param int $size Characters per piece |
| 255 |
* @return array Array of string pieces |
| 256 |
*/ |
| 257 |
private function mb_hard_split($text, $size) { |
| 258 |
if (function_exists('mb_str_split')) { |
| 259 |
return mb_str_split($text, $size); |
| 260 |
} |
| 261 |
$pieces = array(); |
| 262 |
$len = mb_strlen($text); |
| 263 |
for ($i = 0; $i < $len; $i += $size) { |
| 264 |
$pieces[] = mb_substr($text, $i, $size); |
| 265 |
} |
| 266 |
return $pieces; |
| 267 |
} |
| 268 |
|
| 269 |
/** |
| 270 |
* Create chunk metadata for storage |
| 271 |
* |
| 272 |
* @param int $chunk_index 0-based index of this chunk |
| 273 |
* @param int $total_chunks Total number of chunks for this content |
| 274 |
* @param string $source_url Original source URL |
| 275 |
* @return array Metadata array |
| 276 |
*/ |
| 277 |
public static function create_chunk_metadata($chunk_index, $total_chunks, $source_url) { |
| 278 |
return array( |
| 279 |
'document_type' => 'chunked', |
| 280 |
'chunk_index' => intval($chunk_index), |
| 281 |
'total_chunks' => intval($total_chunks), |
| 282 |
'source_url' => $source_url, |
| 283 |
'parent_url_hash' => md5($source_url) |
| 284 |
); |
| 285 |
} |
| 286 |
|
| 287 |
/** |
| 288 |
* Format chunk content with metadata prefix (for WordPress DB storage) |
| 289 |
* |
| 290 |
* @param string $chunk_content The chunk text |
| 291 |
* @param array $metadata Chunk metadata |
| 292 |
* @return string Formatted content with JSON prefix |
| 293 |
*/ |
| 294 |
public static function format_chunk_for_storage($chunk_content, $metadata) { |
| 295 |
return wp_json_encode($metadata) . "\n---\n" . $chunk_content; |
| 296 |
} |
| 297 |
|
| 298 |
/** |
| 299 |
* Parse chunk content to extract metadata and text |
| 300 |
* |
| 301 |
* @param string $stored_content Content from database |
| 302 |
* @return array Array with 'metadata' and 'text' keys |
| 303 |
*/ |
| 304 |
public static function parse_stored_chunk($stored_content) { |
| 305 |
// Check if content has metadata prefix |
| 306 |
if (strpos($stored_content, '{"document_type"') === 0) { |
| 307 |
$parts = explode("\n---\n", $stored_content, 2); |
| 308 |
|
| 309 |
if (count($parts) === 2) { |
| 310 |
$metadata = json_decode($parts[0], true); |
| 311 |
return array( |
| 312 |
'metadata' => $metadata ?: array(), |
| 313 |
'text' => $parts[1], |
| 314 |
'is_chunked' => isset($metadata['document_type']) && $metadata['document_type'] === 'chunked' |
| 315 |
); |
| 316 |
} |
| 317 |
} |
| 318 |
|
| 319 |
// Non-chunked content |
| 320 |
return array( |
| 321 |
'metadata' => array(), |
| 322 |
'text' => $stored_content, |
| 323 |
'is_chunked' => false |
| 324 |
); |
| 325 |
} |
| 326 |
|
| 327 |
/** |
| 328 |
* Generate vector ID for a chunk |
| 329 |
* |
| 330 |
* @param string $source_url Original source URL |
| 331 |
* @param int $chunk_index 0-based chunk index |
| 332 |
* @return string Vector ID in format: {md5(url)}_chunk_{index} |
| 333 |
*/ |
| 334 |
public static function generate_chunk_vector_id($source_url, $chunk_index) { |
| 335 |
$base_id = md5($source_url); |
| 336 |
return $base_id . '_chunk_' . intval($chunk_index); |
| 337 |
} |
| 338 |
|
| 339 |
/** |
| 340 |
* Extract base URL hash from a chunk vector ID |
| 341 |
* |
| 342 |
* @param string $vector_id Vector ID to parse |
| 343 |
* @return string|null Base URL hash or null if not a chunk ID |
| 344 |
*/ |
| 345 |
public static function get_base_hash_from_vector_id($vector_id) { |
| 346 |
if (preg_match('/^([a-f0-9]{32})_chunk_\d+$/', $vector_id, $matches)) { |
| 347 |
return $matches[1]; |
| 348 |
} |
| 349 |
return null; |
| 350 |
} |
| 351 |
|
| 352 |
/** |
| 353 |
* Check if a vector ID is a chunk ID |
| 354 |
* |
| 355 |
* @param string $vector_id Vector ID to check |
| 356 |
* @return bool True if this is a chunk vector ID |
| 357 |
*/ |
| 358 |
public static function is_chunk_vector_id($vector_id) { |
| 359 |
return (bool) preg_match('/^[a-f0-9]{32}_chunk_\d+$/', $vector_id); |
| 360 |
} |
| 361 |
} |
| 362 |
|
