| @@ -67,11 +67,10 @@ | ||
| 67 | 67 | if (!$settings['chunking_enabled']) { |
| 68 | 68 | return false; |
| 69 | 69 | } |
| 70 | 70 | |
| 71 | - // Only chunk if content exceeds chunk size (characters, not bytes — | |
| 72 | - // multibyte scripts would otherwise hit the limit 3x early) | |
| 73 | - return mb_strlen($text) > $this->chunk_size; | |
| 71 | + // Only chunk if content exceeds chunk size | |
| 72 | + return strlen($text) > $this->chunk_size; | |
| 74 | 73 | } |
| 75 | 74 | |
| 76 | 75 | /** |
| 77 | 76 | * Split text into chunks |
| @@ -90,9 +89,9 @@ | ||
| 90 | 89 | return array(); |
| 91 | 90 | } |
| 92 | 91 | |
| 93 | 92 | // Handle content smaller than chunk size - return as single chunk |
| 94 | - if (mb_strlen($text) <= $this->chunk_size) { | |
| 93 | + if (strlen($text) <= $this->chunk_size) { | |
| 95 | 94 | return array(trim($text)); |
| 96 | 95 | } |
| 97 | 96 | |
| 98 | 97 | $chunks = array(); |
| @@ -108,9 +107,9 @@ | ||
| 108 | 107 | } |
| 109 | 108 | |
| 110 | 109 | // Calculate size if we add this paragraph |
| 111 | 110 | $separator = empty($current_chunk) ? '' : "\n\n"; |
| 112 | - $potential_size = mb_strlen($current_chunk) + mb_strlen($separator) + mb_strlen($paragraph); | |
| 111 | + $potential_size = strlen($current_chunk) + strlen($separator) + strlen($paragraph); | |
| 113 | 112 | |
| 114 | 113 | // If adding this paragraph exceeds chunk size |
| 115 | 114 | if ($potential_size > $this->chunk_size && !empty($current_chunk)) { |
| 116 | 115 | // Save current chunk and start fresh |
| @@ -121,9 +120,9 @@ | ||
| 121 | 120 | $current_chunk .= $separator . $paragraph; |
| 122 | 121 | } |
| 123 | 122 | |
| 124 | 123 | // Handle very long paragraphs that exceed chunk size on their own |
| 125 | - if (mb_strlen($current_chunk) > $this->chunk_size) { | |
| 124 | + if (strlen($current_chunk) > $this->chunk_size) { | |
| 126 | 125 | $split_chunks = $this->split_long_paragraph($current_chunk); |
| 127 | 126 | |
| 128 | 127 | // Add all but the last split chunk |
| 129 | 128 | for ($i = 0; $i < count($split_chunks) - 1; $i++) { |
| @@ -165,9 +164,9 @@ | ||
| 165 | 164 | continue; |
| 166 | 165 | } |
| 167 | 166 | |
| 168 | 167 | // If single sentence is too long, split by words |
| 169 | - if (mb_strlen($sentence) > $this->chunk_size) { | |
| 168 | + if (strlen($sentence) > $this->chunk_size) { | |
| 170 | 169 | if (!empty($current_chunk)) { |
| 171 | 170 | $chunks[] = trim($current_chunk); |
| 172 | 171 | $current_chunk = ''; |
| 173 | 172 | } |
| @@ -180,9 +179,9 @@ | ||
| 180 | 179 | continue; |
| 181 | 180 | } |
| 182 | 181 | |
| 183 | 182 | $separator = empty($current_chunk) ? '' : ' '; |
| 184 | - $potential_size = mb_strlen($current_chunk) + mb_strlen($separator) + mb_strlen($sentence); | |
| 183 | + $potential_size = strlen($current_chunk) + strlen($separator) + strlen($sentence); | |
| 185 | 184 | |
| 186 | 185 | if ($potential_size > $this->chunk_size && !empty($current_chunk)) { |
| 187 | 186 | $chunks[] = trim($current_chunk); |
| 188 | 187 | $current_chunk = $sentence; |
| @@ -211,27 +210,10 @@ | ||
| 211 | 210 | $words = preg_split('/\s+/', $text); |
| 212 | 211 | $current_chunk = ''; |
| 213 | 212 | |
| 214 | 213 | foreach ($words as $word) { |
| 215 | - // Last-resort hard split: a single "word" larger than the chunk size | |
| 216 | - // (space-free scripts like Japanese/Chinese/Thai are one token to \s+) | |
| 217 | - // must be split by characters or it is emitted whole at any size. | |
| 218 | - if (mb_strlen($word) > $this->chunk_size) { | |
| 219 | - if (!empty(trim($current_chunk))) { | |
| 220 | - $chunks[] = trim($current_chunk); | |
| 221 | - $current_chunk = ''; | |
| 222 | - } | |
| 223 | - $pieces = $this->mb_hard_split($word, $this->chunk_size); | |
| 224 | - // Emit all full pieces; the last may still accumulate following words | |
| 225 | - $current_chunk = array_pop($pieces); | |
| 226 | - foreach ($pieces as $piece) { | |
| 227 | - $chunks[] = $piece; | |
| 228 | - } | |
| 229 | - continue; | |
| 230 | - } | |
| 231 | - | |
| 232 | 214 | $separator = empty($current_chunk) ? '' : ' '; |
| 233 | - $potential_size = mb_strlen($current_chunk) + mb_strlen($separator) + mb_strlen($word); | |
| 215 | + $potential_size = strlen($current_chunk) + strlen($separator) + strlen($word); | |
| 234 | 216 | |
| 235 | 217 | if ($potential_size > $this->chunk_size && !empty($current_chunk)) { |
| 236 | 218 | $chunks[] = trim($current_chunk); |
| 237 | 219 | $current_chunk = $word; |
| @@ -244,27 +226,8 @@ | ||
| 244 | 226 | $chunks[] = trim($current_chunk); |
| 245 | 227 | } |
| 246 | 228 | |
| 247 | 229 | return $chunks; |
| 248 | - } | |
| 249 | - | |
| 250 | - /** | |
| 251 | - * Split a string into fixed-size character pieces (multibyte-safe) | |
| 252 | - * | |
| 253 | - * @param string $text Text to split | |
| 254 | - * @param int $size Characters per piece | |
| 255 | - * @return array Array of string pieces | |
| 256 | - */ | |
| 257 | - private function mb_hard_split($text, $size) { | |
| 258 | - if (function_exists('mb_str_split')) { | |
| 259 | - return mb_str_split($text, $size); | |
| 260 | - } | |
| 261 | - $pieces = array(); | |
| 262 | - $len = mb_strlen($text); | |
| 263 | - for ($i = 0; $i < $len; $i += $size) { | |
| 264 | - $pieces[] = mb_substr($text, $i, $size); | |
| 265 | - } | |
| 266 | - return $pieces; | |
| 267 | 230 | } |
| 268 | 231 | |
| 269 | 232 | /** |
| 270 | 233 | * Create chunk metadata for storage |
