| @@ -75,9 +75,12 @@ | ||
| 75 | 75 | * |
| 76 | 76 | * @since 4.5.0 |
| 77 | 77 | * @var array |
| 78 | 78 | */ |
| 79 | - public $labels = []; | |
| 79 | + public $labels = [ | |
| 80 | + 'plural' => '', | |
| 81 | + 'singular' => '', | |
| 82 | + ]; | |
| 80 | 83 | |
| 81 | 84 | /** |
| 82 | 85 | * Get number of bulk items to index per page |
| 83 | 86 | * |
| @@ -259,12 +262,16 @@ | ||
| 259 | 262 | * |
| 260 | 263 | * @param int $object_id Object to index. |
| 261 | 264 | * @param boolean $blocking Blocking HTTP request or not. |
| 262 | 265 | * @since 3.0 |
| 263 | - * @return boolean | |
| 266 | + * @return object|boolean | |
| 264 | 267 | */ |
| 265 | 268 | public function index( $object_id, $blocking = false ) { |
| 266 | - $document = $this->prepare_document( $object_id ); | |
| 269 | + try { | |
| 270 | + $document = $this->prepare_document( $object_id ); | |
| 271 | + } catch ( \Throwable $th ) { | |
| 272 | + return false; | |
| 273 | + } | |
| 267 | 274 | |
| 268 | 275 | if ( false === $document ) { |
| 269 | 276 | return false; |
| 270 | 277 | } |
| @@ -271,8 +278,9 @@ | ||
| 271 | 278 | |
| 272 | 279 | /** |
| 273 | 280 | * Conditionally kill indexing on a specific object |
| 274 | 281 | * |
| 282 | + * @deprecated 5.3.3 Use ep_{indexable_slug}_sync_kill instead | |
| 275 | 283 | * @hook ep_{indexable_slug}_index_kill |
| 276 | 284 | * @param {bool} $kill True to not index |
| 277 | 285 | * @param {int} $object_id Id of object to index |
| 278 | 286 | * @since 3.0 |
| @@ -277,13 +285,26 @@ | ||
| 277 | 285 | * @param {int} $object_id Id of object to index |
| 278 | 286 | * @since 3.0 |
| 279 | 287 | * @return {bool} New kill value |
| 280 | 288 | */ |
| 281 | - if ( apply_filters( 'ep_' . $this->slug . '_index_kill', false, $object_id ) ) { | |
| 289 | + if ( apply_filters_deprecated( 'ep_' . $this->slug . '_index_kill', [ false, $object_id ], 'ElasticPress 5.3.3', 'ep_' . $this->slug . '_sync_kill' ) ) { | |
| 282 | 290 | return false; |
| 283 | 291 | } |
| 284 | 292 | |
| 285 | 293 | /** |
| 294 | + * Conditionally kill indexing for an object. | |
| 295 | + * | |
| 296 | + * @hook ep_{$this->slug}_sync_kill | |
| 297 | + * @param {bool} $kill True means dont sync | |
| 298 | + * @param {int} $object_id Object ID | |
| 299 | + * @return {bool} New value | |
| 300 | + */ | |
| 301 | + $ep_indexable_sync_kill = apply_filters( 'ep_' . $this->slug . '_sync_kill', false, $object_id ); | |
| 302 | + if ( $ep_indexable_sync_kill ) { | |
| 303 | + return false; | |
| 304 | + } | |
| 305 | + | |
| 306 | + /** | |
| 286 | 307 | * Filter document before index |
| 287 | 308 | * |
| 288 | 309 | * @hook ep_pre_index_{indexable_slug} |
| 289 | 310 | * @param {array} $document Document to index |
| @@ -298,9 +319,9 @@ | ||
| 298 | 319 | * Fires after document is indexed |
| 299 | 320 | * |
| 300 | 321 | * @hook ep_after_index_{indexable_slug} |
| 301 | 322 | * @param {array} $document Document to index |
| 302 | - * @param {array|boolean} $return ES response on success, false on failure | |
| 323 | + * @param {object|boolean} $return ES response on success, false on failure | |
| 303 | 324 | * @since 3.0 |
| 304 | 325 | */ |
| 305 | 326 | do_action( 'ep_after_index_' . $this->slug, $document, $return ); |
| 306 | 327 | |
| @@ -327,8 +348,10 @@ | ||
| 327 | 348 | */ |
| 328 | 349 | public function bulk_index( $object_ids ) { |
| 329 | 350 | $body = ''; |
| 330 | 351 | |
| 352 | + $non_es_errors = []; | |
| 353 | + | |
| 331 | 354 | foreach ( $object_ids as $object_id ) { |
| 332 | 355 | $action_args = array( |
| 333 | 356 | 'index' => array( |
| 334 | 357 | '_id' => absint( $object_id ), |
| @@ -334,9 +357,22 @@ | ||
| 334 | 357 | '_id' => absint( $object_id ), |
| 335 | 358 | ), |
| 336 | 359 | ); |
| 337 | 360 | |
| 338 | - $document = $this->prepare_document( $object_id ); | |
| 361 | + try { | |
| 362 | + $document = $this->prepare_document( $object_id ); | |
| 363 | + } catch ( \Throwable $th ) { | |
| 364 | + $non_es_errors[] = [ | |
| 365 | + 'index' => [ | |
| 366 | + '_id' => absint( $object_id ), | |
| 367 | + 'error' => [ | |
| 368 | + 'type' => 'prepare_document_error', | |
| 369 | + 'reason' => $th->getMessage(), | |
| 370 | + ], | |
| 371 | + ], | |
| 372 | + ]; | |
| 373 | + continue; | |
| 374 | + } | |
| 339 | 375 | |
| 340 | 376 | /** |
| 341 | 377 | * Conditionally kill indexing on a specific object |
| 342 | 378 | * |
| @@ -353,8 +389,13 @@ | ||
| 353 | 389 | } |
| 354 | 390 | |
| 355 | 391 | $result = Elasticsearch::factory()->bulk_index( $this->get_index_name(), $this->slug, $body ); |
| 356 | 392 | |
| 393 | + if ( ! empty( $non_es_errors ) ) { | |
| 394 | + $result['errors'] = true; | |
| 395 | + $result['items'] = isset( $result['items'] ) ? array_merge( $result['items'], $non_es_errors ) : $non_es_errors; | |
| 396 | + } | |
| 397 | + | |
| 357 | 398 | /** |
| 358 | 399 | * Perform actions after a bulk indexing is completed |
| 359 | 400 | * |
| 360 | 401 | * @hook ep_after_bulk_index |
| @@ -376,8 +417,10 @@ | ||
| 376 | 417 | */ |
| 377 | 418 | public function bulk_index_dynamically( $object_ids ) { |
| 378 | 419 | $documents = []; |
| 379 | 420 | |
| 421 | + $non_es_errors = []; | |
| 422 | + | |
| 380 | 423 | foreach ( $object_ids as $object_id ) { |
| 381 | 424 | $action_args = array( |
| 382 | 425 | 'index' => array( |
| 383 | 426 | '_id' => absint( $object_id ), |
| @@ -383,9 +426,22 @@ | ||
| 383 | 426 | '_id' => absint( $object_id ), |
| 384 | 427 | ), |
| 385 | 428 | ); |
| 386 | 429 | |
| 387 | - $document = $this->prepare_document( $object_id ); | |
| 430 | + try { | |
| 431 | + $document = $this->prepare_document( $object_id ); | |
| 432 | + } catch ( \Throwable $th ) { | |
| 433 | + $non_es_errors[] = [ | |
| 434 | + 'index' => [ | |
| 435 | + '_id' => absint( $object_id ), | |
| 436 | + 'error' => [ | |
| 437 | + 'type' => 'prepare_document_error', | |
| 438 | + 'reason' => $th->getMessage(), | |
| 439 | + ], | |
| 440 | + ], | |
| 441 | + ]; | |
| 442 | + continue; | |
| 443 | + } | |
| 388 | 444 | |
| 389 | 445 | if ( empty( $document ) ) { |
| 390 | 446 | continue; |
| 391 | 447 | } |
| @@ -406,15 +462,34 @@ | ||
| 406 | 462 | $documents[] = $document_str; |
| 407 | 463 | } |
| 408 | 464 | |
| 409 | 465 | if ( empty( $documents ) ) { |
| 410 | - return [ | |
| 411 | - new \WP_Error( 'ep_bulk_index_no_documents', esc_html__( 'It was not possible to create a body request with the document IDs provided.', 'elasticpress' ), $object_ids ), | |
| 412 | - ]; | |
| 466 | + return ( ! empty( $non_es_errors ) ? [ | |
| 467 | + [ | |
| 468 | + 'errors' => true, | |
| 469 | + 'items' => $non_es_errors, | |
| 470 | + ], | |
| 471 | + ] : [ | |
| 472 | + new \WP_Error( | |
| 473 | + 'ep_bulk_index_no_documents', | |
| 474 | + esc_html__( 'It was not possible to create a body request with the document IDs provided.', 'elasticpress' ), | |
| 475 | + $object_ids | |
| 476 | + ), | |
| 477 | + ] ); | |
| 413 | 478 | } |
| 414 | 479 | |
| 415 | 480 | $results = $this->send_bulk_index_request( $documents ); |
| 416 | 481 | |
| 482 | + if ( ! empty( $non_es_errors ) ) { | |
| 483 | + $results = [ | |
| 484 | + [ | |
| 485 | + 'errors' => true, | |
| 486 | + 'items' => $non_es_errors, | |
| 487 | + ], | |
| 488 | + ...$results, | |
| 489 | + ]; | |
| 490 | + } | |
| 491 | + | |
| 417 | 492 | /** |
| 418 | 493 | * Perform actions after a dynamic bulk indexing is completed |
| 419 | 494 | * |
| 420 | 495 | * @hook ep_after_bulk_index_dynamically |
| @@ -487,12 +562,12 @@ | ||
| 487 | 562 | } |
| 488 | 563 | |
| 489 | 564 | $results = []; |
| 490 | 565 | |
| 491 | - $body = []; | |
| 566 | + $body = []; | |
| 567 | + $current_body_size = 0; | |
| 568 | + $requests = 0; | |
| 492 | 569 | |
| 493 | - $requests = 0; | |
| 494 | - | |
| 495 | 570 | /* |
| 496 | 571 | * This script will use two main arrays: $body and $documents, being $body the |
| 497 | 572 | * documents to be sent in the next request and $documents the list of docs to be indexed. |
| 498 | 573 | * The do-while loop will stop if all documents are sent or if a request fails even sending |
| @@ -498,16 +573,18 @@ | ||
| 498 | 573 | * The do-while loop will stop if all documents are sent or if a request fails even sending |
| 499 | 574 | * a buffer as small as possible. |
| 500 | 575 | */ |
| 501 | 576 | do { |
| 502 | - $next_document = array_shift( $documents ); | |
| 577 | + $next_document = array_shift( $documents ); | |
| 578 | + $next_document_size = mb_strlen( $next_document ); | |
| 579 | + $has_buffered_documents = count( $body ) > 0; | |
| 503 | 580 | |
| 504 | 581 | // If the next document alone takes the entire current buffer size, |
| 505 | 582 | // let's add it back to the pipe and send what we have first |
| 506 | - if ( mb_strlen( $next_document ) > $current_buffer_size && count( $body ) > 0 ) { | |
| 583 | + if ( $next_document_size > $current_buffer_size && $has_buffered_documents ) { | |
| 507 | 584 | array_unshift( $documents, $next_document ); |
| 508 | 585 | } else { |
| 509 | - if ( mb_strlen( $next_document ) > $max_buffer_size ) { | |
| 586 | + if ( $next_document_size > $max_buffer_size ) { | |
| 510 | 587 | /** |
| 511 | 588 | * Perform actions when a post is bigger than the max buffer size. |
| 512 | 589 | * |
| 513 | 590 | * @hook ep_dynamic_bulk_post_too_big |
| @@ -517,15 +594,22 @@ | ||
| 517 | 594 | do_action( 'ep_dynamic_bulk_post_too_big', $next_document ); |
| 518 | 595 | $results[] = new \WP_Error( 'ep_too_big_request_skipped', 'Indexable too big. Request not sent.' ); |
| 519 | 596 | continue; |
| 520 | 597 | } |
| 521 | - $body[] = $next_document; | |
| 522 | - if ( mb_strlen( implode( '', $body ) ) < $current_buffer_size && ! empty( $documents ) ) { | |
| 598 | + | |
| 599 | + $body[] = $next_document; | |
| 600 | + $current_body_size += $next_document_size; | |
| 601 | + | |
| 602 | + $can_add_more_documents = ( $current_body_size < $current_buffer_size && ! empty( $documents ) ); | |
| 603 | + if ( $can_add_more_documents ) { | |
| 523 | 604 | continue; |
| 524 | 605 | } |
| 525 | - if ( mb_strlen( implode( '', $body ) ) > $max_buffer_size ) { | |
| 606 | + | |
| 607 | + if ( $current_body_size > $max_buffer_size ) { | |
| 526 | 608 | // The last document added to body made it too big, so let's give it back. |
| 527 | - array_unshift( $documents, array_pop( $body ) ); | |
| 609 | + $removed_document = array_pop( $body ); | |
| 610 | + $current_body_size -= mb_strlen( $removed_document ); | |
| 611 | + array_unshift( $documents, $removed_document ); | |
| 528 | 612 | } |
| 529 | 613 | } |
| 530 | 614 | |
| 531 | 615 | // Try the request. |
| @@ -531,9 +615,9 @@ | ||
| 531 | 615 | // Try the request. |
| 532 | 616 | timer_start(); |
| 533 | 617 | $result = Elasticsearch::factory()->bulk_index( $this->get_index_name(), $this->slug, implode( '', $body ) ); |
| 534 | 618 | $request_time = timer_stop(); |
| 535 | - $requests++; | |
| 619 | + ++$requests; | |
| 536 | 620 | |
| 537 | 621 | /** |
| 538 | 622 | * Perform actions before a new batch of documents is processed. |
| 539 | 623 | * |
| @@ -550,37 +634,42 @@ | ||
| 550 | 634 | do_action( 'ep_after_send_dynamic_bulk_request', $result, $body, $documents, $min_buffer_size, $max_buffer_size, $current_buffer_size, $request_time ); |
| 551 | 635 | |
| 552 | 636 | // It failed, possibly adjust the buffer size and try again. |
| 553 | 637 | if ( is_wp_error( $result ) ) { |
| 638 | + $error_code = $result->get_error_code(); | |
| 639 | + | |
| 554 | 640 | // Too many requests, wait and try again. |
| 555 | - if ( 429 === $result->get_error_code() ) { | |
| 641 | + if ( 429 === $error_code ) { | |
| 556 | 642 | sleep( 2 ); |
| 557 | 643 | } |
| 558 | 644 | |
| 559 | 645 | // If the error is not a "Request too big" then we really fail this batch of documents. |
| 560 | - if ( 413 !== $result->get_error_code() ) { | |
| 646 | + if ( 413 !== $error_code ) { | |
| 561 | 647 | $results[] = $result; |
| 562 | 648 | continue; |
| 563 | 649 | } |
| 564 | 650 | |
| 565 | 651 | if ( count( $body ) === 1 ) { |
| 566 | - $max_buffer_size = min( $max_buffer_size, mb_strlen( implode( '', $body ) ) ); | |
| 567 | - $results[] = $result; | |
| 568 | - $body = []; | |
| 652 | + $max_buffer_size = min( $max_buffer_size, $current_body_size ); | |
| 653 | + $results[] = $result; | |
| 654 | + $body = []; | |
| 655 | + $current_body_size = 0; | |
| 569 | 656 | continue; |
| 570 | 657 | } |
| 571 | 658 | |
| 572 | 659 | // As the buffer is as small as possible, return the error. |
| 573 | - if ( mb_strlen( implode( '', $body ) ) === $min_buffer_size ) { | |
| 660 | + if ( $current_body_size === $min_buffer_size ) { | |
| 574 | 661 | $results[] = $result; |
| 575 | 662 | continue; |
| 576 | 663 | } |
| 577 | 664 | |
| 578 | 665 | // We have a too big buffer. Remove one doc from the body, and set both max and current as its size. |
| 579 | - array_unshift( $documents, array_pop( $body ) ); | |
| 666 | + $removed_document = array_pop( $body ); | |
| 667 | + $current_body_size -= mb_strlen( $removed_document ); | |
| 668 | + array_unshift( $documents, $removed_document ); | |
| 580 | 669 | |
| 581 | 670 | $max_buffer_size = count( $body ) ? |
| 582 | - max( $min_buffer_size, mb_strlen( implode( '', $body ) ) ) : | |
| 671 | + max( $min_buffer_size, $current_body_size ) : | |
| 583 | 672 | $min_buffer_size; |
| 584 | 673 | |
| 585 | 674 | $current_buffer_size = $max_buffer_size; |
| 586 | 675 | continue; |
| @@ -586,15 +675,16 @@ | ||
| 586 | 675 | continue; |
| 587 | 676 | } |
| 588 | 677 | |
| 589 | 678 | // Things worked so we can try to bump the buffer size. |
| 590 | - if ( $current_buffer_size < $max_buffer_size && mb_strlen( implode( '', $body ) ) > $current_buffer_size ) { | |
| 679 | + if ( $current_buffer_size < $max_buffer_size && $current_body_size > $current_buffer_size ) { | |
| 591 | 680 | $current_buffer_size = min( ( $current_buffer_size + $incremental_step ), $max_buffer_size ); |
| 592 | 681 | } |
| 593 | 682 | |
| 594 | 683 | $results[] = $result; |
| 595 | 684 | |
| 596 | - $body = []; | |
| 685 | + $body = []; | |
| 686 | + $current_body_size = 0; | |
| 597 | 687 | } while ( ! empty( $documents ) ); |
| 598 | 688 | |
| 599 | 689 | /** |
| 600 | 690 | * Perform actions after a batch of documents was processed. |
| @@ -677,9 +767,8 @@ | ||
| 677 | 767 | $prepared_meta[ $meta_key ] = array_map( array( $this, 'prepare_meta_value_types' ), $meta_values ); |
| 678 | 768 | } |
| 679 | 769 | |
| 680 | 770 | return $prepared_meta; |
| 681 | - | |
| 682 | 771 | } |
| 683 | 772 | |
| 684 | 773 | /** |
| 685 | 774 | * Prepare meta types for meta value |
| @@ -1114,9 +1203,9 @@ | ||
| 1114 | 1203 | return ( (string) $new_mapping['settings']['index.number_of_shards'] === $stored_mapping[ $this->get_index_name() ]['settings']['index']['number_of_shards'] ); |
| 1115 | 1204 | } |
| 1116 | 1205 | |
| 1117 | 1206 | /** |
| 1118 | - * Utilitary function to check if the indexable is being fully reindexed, i.e., | |
| 1207 | + * Utility function to check if the indexable is being fully reindexed, i.e., | |
| 1119 | 1208 | * the index was deleted, a new mapping was sent and content is being reindexed. |
| 1120 | 1209 | * |
| 1121 | 1210 | * @param int|null $blog_id Blog ID |
| 1122 | 1211 | * @return boolean |
| @@ -1182,9 +1271,9 @@ | ||
| 1182 | 1271 | * @param array $search_fields Search fields |
| 1183 | 1272 | * @param array $query_vars Query vars |
| 1184 | 1273 | * @return SearchAlgorithm Instance of search algorithm to be used |
| 1185 | 1274 | */ |
| 1186 | - public function get_search_algorithm( string $search_text, array $search_fields, array $query_vars ) : \ElasticPress\SearchAlgorithm { | |
| 1275 | + public function get_search_algorithm( string $search_text, array $search_fields, array $query_vars ): \ElasticPress\SearchAlgorithm { | |
| 1187 | 1276 | /** |
| 1188 | 1277 | * Filter the search algorithm to be used |
| 1189 | 1278 | * |
| 1190 | 1279 | * @hook ep_{$indexable_slug}_search_algorithm |
| @@ -1212,11 +1301,11 @@ | ||
| 1212 | 1301 | $mapping = $this->get_mapping(); |
| 1213 | 1302 | |
| 1214 | 1303 | try { |
| 1215 | 1304 | if ( version_compare( (string) Elasticsearch::factory()->get_elasticsearch_version(), '7.0', '<' ) ) { |
| 1216 | - $meta_fields = $mapping[ $this->get_index_name( $blog_id ) ]['mappings']['post']['properties']['meta']['properties']; | |
| 1305 | + $meta_fields = (array) $mapping[ $this->get_index_name( $blog_id ) ]['mappings']['post']['properties']['meta']['properties']; | |
| 1217 | 1306 | } else { |
| 1218 | - $meta_fields = $mapping[ $this->get_index_name( $blog_id ) ]['mappings']['properties']['meta']['properties']; | |
| 1307 | + $meta_fields = (array) $mapping[ $this->get_index_name( $blog_id ) ]['mappings']['properties']['meta']['properties']; | |
| 1219 | 1308 | } |
| 1220 | 1309 | $meta_keys = array_values( array_keys( $meta_fields ) ); |
| 1221 | 1310 | sort( $meta_keys ); |
| 1222 | 1311 | } catch ( \Throwable $th ) { |
| @@ -1287,9 +1376,9 @@ | ||
| 1287 | 1376 | * @since 4.5.0 |
| 1288 | 1377 | * @param array $mapping The mapping |
| 1289 | 1378 | * @return array |
| 1290 | 1379 | */ |
| 1291 | - public function add_ngram_analyzer( array $mapping ) : array { | |
| 1380 | + public function add_ngram_analyzer( array $mapping ): array { | |
| 1292 | 1381 | $mapping['settings']['analysis']['analyzer']['edge_ngram_analyzer'] = array( |
| 1293 | 1382 | 'type' => 'custom', |
| 1294 | 1383 | 'tokenizer' => 'standard', |
| 1295 | 1384 | 'filter' => array( |
