| @@ -67,10 +67,11 @@ | ||
| 67 | 67 | if (!$settings['chunking_enabled']) { |
| 68 | 68 | return false; |
| 69 | 69 | } |
| 70 | 70 | |
| 71 | - // Only chunk if content exceeds chunk size | |
| 72 | - return strlen($text) > $this->chunk_size; | |
| 71 | + // Only chunk if content exceeds chunk size (characters, not bytes — | |
| 72 | + // multibyte scripts would otherwise hit the limit 3x early) | |
| 73 | + return mb_strlen($text) > $this->chunk_size; | |
| 73 | 74 | } |
| 74 | 75 | |
| 75 | 76 | /** |
| 76 | 77 | * Split text into chunks |
| @@ -89,9 +90,9 @@ | ||
| 89 | 90 | return array(); |
| 90 | 91 | } |
| 91 | 92 | |
| 92 | 93 | // Handle content smaller than chunk size - return as single chunk |
| 93 | - if (strlen($text) <= $this->chunk_size) { | |
| 94 | + if (mb_strlen($text) <= $this->chunk_size) { | |
| 94 | 95 | return array(trim($text)); |
| 95 | 96 | } |
| 96 | 97 | |
| 97 | 98 | $chunks = array(); |
| @@ -107,9 +108,9 @@ | ||
| 107 | 108 | } |
| 108 | 109 | |
| 109 | 110 | // Calculate size if we add this paragraph |
| 110 | 111 | $separator = empty($current_chunk) ? '' : "\n\n"; |
| 111 | - $potential_size = strlen($current_chunk) + strlen($separator) + strlen($paragraph); | |
| 112 | + $potential_size = mb_strlen($current_chunk) + mb_strlen($separator) + mb_strlen($paragraph); | |
| 112 | 113 | |
| 113 | 114 | // If adding this paragraph exceeds chunk size |
| 114 | 115 | if ($potential_size > $this->chunk_size && !empty($current_chunk)) { |
| 115 | 116 | // Save current chunk and start fresh |
| @@ -120,9 +121,9 @@ | ||
| 120 | 121 | $current_chunk .= $separator . $paragraph; |
| 121 | 122 | } |
| 122 | 123 | |
| 123 | 124 | // Handle very long paragraphs that exceed chunk size on their own |
| 124 | - if (strlen($current_chunk) > $this->chunk_size) { | |
| 125 | + if (mb_strlen($current_chunk) > $this->chunk_size) { | |
| 125 | 126 | $split_chunks = $this->split_long_paragraph($current_chunk); |
| 126 | 127 | |
| 127 | 128 | // Add all but the last split chunk |
| 128 | 129 | for ($i = 0; $i < count($split_chunks) - 1; $i++) { |
| @@ -164,9 +165,9 @@ | ||
| 164 | 165 | continue; |
| 165 | 166 | } |
| 166 | 167 | |
| 167 | 168 | // If single sentence is too long, split by words |
| 168 | - if (strlen($sentence) > $this->chunk_size) { | |
| 169 | + if (mb_strlen($sentence) > $this->chunk_size) { | |
| 169 | 170 | if (!empty($current_chunk)) { |
| 170 | 171 | $chunks[] = trim($current_chunk); |
| 171 | 172 | $current_chunk = ''; |
| 172 | 173 | } |
| @@ -179,9 +180,9 @@ | ||
| 179 | 180 | continue; |
| 180 | 181 | } |
| 181 | 182 | |
| 182 | 183 | $separator = empty($current_chunk) ? '' : ' '; |
| 183 | - $potential_size = strlen($current_chunk) + strlen($separator) + strlen($sentence); | |
| 184 | + $potential_size = mb_strlen($current_chunk) + mb_strlen($separator) + mb_strlen($sentence); | |
| 184 | 185 | |
| 185 | 186 | if ($potential_size > $this->chunk_size && !empty($current_chunk)) { |
| 186 | 187 | $chunks[] = trim($current_chunk); |
| 187 | 188 | $current_chunk = $sentence; |
| @@ -210,10 +211,27 @@ | ||
| 210 | 211 | $words = preg_split('/\s+/', $text); |
| 211 | 212 | $current_chunk = ''; |
| 212 | 213 | |
| 213 | 214 | foreach ($words as $word) { |
| 215 | + // Last-resort hard split: a single "word" larger than the chunk size | |
| 216 | + // (space-free scripts like Japanese/Chinese/Thai are one token to \s+) | |
| 217 | + // must be split by characters or it is emitted whole at any size. | |
| 218 | + if (mb_strlen($word) > $this->chunk_size) { | |
| 219 | + if (!empty(trim($current_chunk))) { | |
| 220 | + $chunks[] = trim($current_chunk); | |
| 221 | + $current_chunk = ''; | |
| 222 | + } | |
| 223 | + $pieces = $this->mb_hard_split($word, $this->chunk_size); | |
| 224 | + // Emit all full pieces; the last may still accumulate following words | |
| 225 | + $current_chunk = array_pop($pieces); | |
| 226 | + foreach ($pieces as $piece) { | |
| 227 | + $chunks[] = $piece; | |
| 228 | + } | |
| 229 | + continue; | |
| 230 | + } | |
| 231 | + | |
| 214 | 232 | $separator = empty($current_chunk) ? '' : ' '; |
| 215 | - $potential_size = strlen($current_chunk) + strlen($separator) + strlen($word); | |
| 233 | + $potential_size = mb_strlen($current_chunk) + mb_strlen($separator) + mb_strlen($word); | |
| 216 | 234 | |
| 217 | 235 | if ($potential_size > $this->chunk_size && !empty($current_chunk)) { |
| 218 | 236 | $chunks[] = trim($current_chunk); |
| 219 | 237 | $current_chunk = $word; |
| @@ -226,8 +244,27 @@ | ||
| 226 | 244 | $chunks[] = trim($current_chunk); |
| 227 | 245 | } |
| 228 | 246 | |
| 229 | 247 | return $chunks; |
| 248 | + } | |
| 249 | + | |
| 250 | + /** | |
| 251 | + * Split a string into fixed-size character pieces (multibyte-safe) | |
| 252 | + * | |
| 253 | + * @param string $text Text to split | |
| 254 | + * @param int $size Characters per piece | |
| 255 | + * @return array Array of string pieces | |
| 256 | + */ | |
| 257 | + private function mb_hard_split($text, $size) { | |
| 258 | + if (function_exists('mb_str_split')) { | |
| 259 | + return mb_str_split($text, $size); | |
| 260 | + } | |
| 261 | + $pieces = array(); | |
| 262 | + $len = mb_strlen($text); | |
| 263 | + for ($i = 0; $i < $len; $i += $size) { | |
| 264 | + $pieces[] = mb_substr($text, $i, $size); | |
| 265 | + } | |
| 266 | + return $pieces; | |
| 230 | 267 | } |
| 231 | 268 | |
| 232 | 269 | /** |
| 233 | 270 | * Create chunk metadata for storage |
