| @@ -9,11 +9,10 @@ | ||
| 9 | 9 | */ |
| 10 | 10 | |
| 11 | 11 | namespace ElasticPress; |
| 12 | 12 | |
| 13 | -use ElasticPress\Elasticsearch as Elasticsearch; | |
| 14 | -use ElasticPress\SyncManager as SyncManager; | |
| 15 | -use ElasticPress\QueryIntegration as QueryIntegration; | |
| 13 | +use ElasticPress\Elasticsearch; | |
| 14 | +use ElasticPress\SyncManager; | |
| 16 | 15 | |
| 17 | 16 | if ( ! defined( 'ABSPATH' ) ) { |
| 18 | 17 | exit; // Exit if accessed directly. |
| 19 | 18 | } |
| @@ -48,9 +47,9 @@ | ||
| 48 | 47 | /** |
| 49 | 48 | * Instance of QueryIntegration. This should handle integrating with a default |
| 50 | 49 | * WP query. |
| 51 | 50 | * |
| 52 | - * @var QueryIntegration | |
| 51 | + * @var object | |
| 53 | 52 | * @since 3.0 |
| 54 | 53 | */ |
| 55 | 54 | public $query_integration; |
| 56 | 55 | |
| @@ -76,9 +75,12 @@ | ||
| 76 | 75 | * |
| 77 | 76 | * @since 4.5.0 |
| 78 | 77 | * @var array |
| 79 | 78 | */ |
| 80 | - public $labels = []; | |
| 79 | + public $labels = [ | |
| 80 | + 'plural' => '', | |
| 81 | + 'singular' => '', | |
| 82 | + ]; | |
| 81 | 83 | |
| 82 | 84 | /** |
| 83 | 85 | * Get number of bulk items to index per page |
| 84 | 86 | * |
| @@ -260,12 +262,16 @@ | ||
| 260 | 262 | * |
| 261 | 263 | * @param int $object_id Object to index. |
| 262 | 264 | * @param boolean $blocking Blocking HTTP request or not. |
| 263 | 265 | * @since 3.0 |
| 264 | - * @return boolean | |
| 266 | + * @return object|boolean | |
| 265 | 267 | */ |
| 266 | 268 | public function index( $object_id, $blocking = false ) { |
| 267 | - $document = $this->prepare_document( $object_id ); | |
| 269 | + try { | |
| 270 | + $document = $this->prepare_document( $object_id ); | |
| 271 | + } catch ( \Throwable $th ) { | |
| 272 | + return false; | |
| 273 | + } | |
| 268 | 274 | |
| 269 | 275 | if ( false === $document ) { |
| 270 | 276 | return false; |
| 271 | 277 | } |
| @@ -272,8 +278,9 @@ | ||
| 272 | 278 | |
| 273 | 279 | /** |
| 274 | 280 | * Conditionally kill indexing on a specific object |
| 275 | 281 | * |
| 282 | + * @deprecated 5.3.3 Use ep_{indexable_slug}_sync_kill instead | |
| 276 | 283 | * @hook ep_{indexable_slug}_index_kill |
| 277 | 284 | * @param {bool} $kill True to not index |
| 278 | 285 | * @param {int} $object_id Id of object to index |
| 279 | 286 | * @since 3.0 |
| @@ -278,13 +285,26 @@ | ||
| 278 | 285 | * @param {int} $object_id Id of object to index |
| 279 | 286 | * @since 3.0 |
| 280 | 287 | * @return {bool} New kill value |
| 281 | 288 | */ |
| 282 | - if ( apply_filters( 'ep_' . $this->slug . '_index_kill', false, $object_id ) ) { | |
| 289 | + if ( apply_filters_deprecated( 'ep_' . $this->slug . '_index_kill', [ false, $object_id ], 'ElasticPress 5.3.3', 'ep_' . $this->slug . '_sync_kill' ) ) { | |
| 283 | 290 | return false; |
| 284 | 291 | } |
| 285 | 292 | |
| 286 | 293 | /** |
| 294 | + * Conditionally kill indexing for an object. | |
| 295 | + * | |
| 296 | + * @hook ep_{$this->slug}_sync_kill | |
| 297 | + * @param {bool} $kill True means dont sync | |
| 298 | + * @param {int} $object_id Object ID | |
| 299 | + * @return {bool} New value | |
| 300 | + */ | |
| 301 | + $ep_indexable_sync_kill = apply_filters( 'ep_' . $this->slug . '_sync_kill', false, $object_id ); | |
| 302 | + if ( $ep_indexable_sync_kill ) { | |
| 303 | + return false; | |
| 304 | + } | |
| 305 | + | |
| 306 | + /** | |
| 287 | 307 | * Filter document before index |
| 288 | 308 | * |
| 289 | 309 | * @hook ep_pre_index_{indexable_slug} |
| 290 | 310 | * @param {array} $document Document to index |
| @@ -299,9 +319,9 @@ | ||
| 299 | 319 | * Fires after document is indexed |
| 300 | 320 | * |
| 301 | 321 | * @hook ep_after_index_{indexable_slug} |
| 302 | 322 | * @param {array} $document Document to index |
| 303 | - * @param {array|boolean} $return ES response on success, false on failure | |
| 323 | + * @param {object|boolean} $return ES response on success, false on failure | |
| 304 | 324 | * @since 3.0 |
| 305 | 325 | */ |
| 306 | 326 | do_action( 'ep_after_index_' . $this->slug, $document, $return ); |
| 307 | 327 | |
| @@ -328,8 +348,10 @@ | ||
| 328 | 348 | */ |
| 329 | 349 | public function bulk_index( $object_ids ) { |
| 330 | 350 | $body = ''; |
| 331 | 351 | |
| 352 | + $non_es_errors = []; | |
| 353 | + | |
| 332 | 354 | foreach ( $object_ids as $object_id ) { |
| 333 | 355 | $action_args = array( |
| 334 | 356 | 'index' => array( |
| 335 | 357 | '_id' => absint( $object_id ), |
| @@ -335,9 +357,22 @@ | ||
| 335 | 357 | '_id' => absint( $object_id ), |
| 336 | 358 | ), |
| 337 | 359 | ); |
| 338 | 360 | |
| 339 | - $document = $this->prepare_document( $object_id ); | |
| 361 | + try { | |
| 362 | + $document = $this->prepare_document( $object_id ); | |
| 363 | + } catch ( \Throwable $th ) { | |
| 364 | + $non_es_errors[] = [ | |
| 365 | + 'index' => [ | |
| 366 | + '_id' => absint( $object_id ), | |
| 367 | + 'error' => [ | |
| 368 | + 'type' => 'prepare_document_error', | |
| 369 | + 'reason' => $th->getMessage(), | |
| 370 | + ], | |
| 371 | + ], | |
| 372 | + ]; | |
| 373 | + continue; | |
| 374 | + } | |
| 340 | 375 | |
| 341 | 376 | /** |
| 342 | 377 | * Conditionally kill indexing on a specific object |
| 343 | 378 | * |
| @@ -354,8 +389,13 @@ | ||
| 354 | 389 | } |
| 355 | 390 | |
| 356 | 391 | $result = Elasticsearch::factory()->bulk_index( $this->get_index_name(), $this->slug, $body ); |
| 357 | 392 | |
| 393 | + if ( ! empty( $non_es_errors ) ) { | |
| 394 | + $result['errors'] = true; | |
| 395 | + $result['items'] = isset( $result['items'] ) ? array_merge( $result['items'], $non_es_errors ) : $non_es_errors; | |
| 396 | + } | |
| 397 | + | |
| 358 | 398 | /** |
| 359 | 399 | * Perform actions after a bulk indexing is completed |
| 360 | 400 | * |
| 361 | 401 | * @hook ep_after_bulk_index |
| @@ -377,8 +417,10 @@ | ||
| 377 | 417 | */ |
| 378 | 418 | public function bulk_index_dynamically( $object_ids ) { |
| 379 | 419 | $documents = []; |
| 380 | 420 | |
| 421 | + $non_es_errors = []; | |
| 422 | + | |
| 381 | 423 | foreach ( $object_ids as $object_id ) { |
| 382 | 424 | $action_args = array( |
| 383 | 425 | 'index' => array( |
| 384 | 426 | '_id' => absint( $object_id ), |
| @@ -384,10 +426,27 @@ | ||
| 384 | 426 | '_id' => absint( $object_id ), |
| 385 | 427 | ), |
| 386 | 428 | ); |
| 387 | 429 | |
| 388 | - $document = $this->prepare_document( $object_id ); | |
| 430 | + try { | |
| 431 | + $document = $this->prepare_document( $object_id ); | |
| 432 | + } catch ( \Throwable $th ) { | |
| 433 | + $non_es_errors[] = [ | |
| 434 | + 'index' => [ | |
| 435 | + '_id' => absint( $object_id ), | |
| 436 | + 'error' => [ | |
| 437 | + 'type' => 'prepare_document_error', | |
| 438 | + 'reason' => $th->getMessage(), | |
| 439 | + ], | |
| 440 | + ], | |
| 441 | + ]; | |
| 442 | + continue; | |
| 443 | + } | |
| 389 | 444 | |
| 445 | + if ( empty( $document ) ) { | |
| 446 | + continue; | |
| 447 | + } | |
| 448 | + | |
| 390 | 449 | /** |
| 391 | 450 | * Conditionally kill indexing on a specific object |
| 392 | 451 | * |
| 393 | 452 | * @hook ep_bulk_index_action_args |
| @@ -402,10 +461,35 @@ | ||
| 402 | 461 | |
| 403 | 462 | $documents[] = $document_str; |
| 404 | 463 | } |
| 405 | 464 | |
| 465 | + if ( empty( $documents ) ) { | |
| 466 | + return ( ! empty( $non_es_errors ) ? [ | |
| 467 | + [ | |
| 468 | + 'errors' => true, | |
| 469 | + 'items' => $non_es_errors, | |
| 470 | + ], | |
| 471 | + ] : [ | |
| 472 | + new \WP_Error( | |
| 473 | + 'ep_bulk_index_no_documents', | |
| 474 | + esc_html__( 'It was not possible to create a body request with the document IDs provided.', 'elasticpress' ), | |
| 475 | + $object_ids | |
| 476 | + ), | |
| 477 | + ] ); | |
| 478 | + } | |
| 479 | + | |
| 406 | 480 | $results = $this->send_bulk_index_request( $documents ); |
| 407 | 481 | |
| 482 | + if ( ! empty( $non_es_errors ) ) { | |
| 483 | + $results = [ | |
| 484 | + [ | |
| 485 | + 'errors' => true, | |
| 486 | + 'items' => $non_es_errors, | |
| 487 | + ], | |
| 488 | + ...$results, | |
| 489 | + ]; | |
| 490 | + } | |
| 491 | + | |
| 408 | 492 | /** |
| 409 | 493 | * Perform actions after a dynamic bulk indexing is completed |
| 410 | 494 | * |
| 411 | 495 | * @hook ep_after_bulk_index_dynamically |
| @@ -478,12 +562,12 @@ | ||
| 478 | 562 | } |
| 479 | 563 | |
| 480 | 564 | $results = []; |
| 481 | 565 | |
| 482 | - $body = []; | |
| 566 | + $body = []; | |
| 567 | + $current_body_size = 0; | |
| 568 | + $requests = 0; | |
| 483 | 569 | |
| 484 | - $requests = 0; | |
| 485 | - | |
| 486 | 570 | /* |
| 487 | 571 | * This script will use two main arrays: $body and $documents, being $body the |
| 488 | 572 | * documents to be sent in the next request and $documents the list of docs to be indexed. |
| 489 | 573 | * The do-while loop will stop if all documents are sent or if a request fails even sending |
| @@ -489,16 +573,18 @@ | ||
| 489 | 573 | * The do-while loop will stop if all documents are sent or if a request fails even sending |
| 490 | 574 | * a buffer as small as possible. |
| 491 | 575 | */ |
| 492 | 576 | do { |
| 493 | - $next_document = array_shift( $documents ); | |
| 577 | + $next_document = array_shift( $documents ); | |
| 578 | + $next_document_size = mb_strlen( $next_document ); | |
| 579 | + $has_buffered_documents = count( $body ) > 0; | |
| 494 | 580 | |
| 495 | 581 | // If the next document alone takes the entire current buffer size, |
| 496 | 582 | // let's add it back to the pipe and send what we have first |
| 497 | - if ( mb_strlen( $next_document ) > $current_buffer_size && count( $body ) > 0 ) { | |
| 583 | + if ( $next_document_size > $current_buffer_size && $has_buffered_documents ) { | |
| 498 | 584 | array_unshift( $documents, $next_document ); |
| 499 | 585 | } else { |
| 500 | - if ( mb_strlen( $next_document ) > $max_buffer_size ) { | |
| 586 | + if ( $next_document_size > $max_buffer_size ) { | |
| 501 | 587 | /** |
| 502 | 588 | * Perform actions when a post is bigger than the max buffer size. |
| 503 | 589 | * |
| 504 | 590 | * @hook ep_dynamic_bulk_post_too_big |
| @@ -508,15 +594,22 @@ | ||
| 508 | 594 | do_action( 'ep_dynamic_bulk_post_too_big', $next_document ); |
| 509 | 595 | $results[] = new \WP_Error( 'ep_too_big_request_skipped', 'Indexable too big. Request not sent.' ); |
| 510 | 596 | continue; |
| 511 | 597 | } |
| 512 | - $body[] = $next_document; | |
| 513 | - if ( mb_strlen( implode( '', $body ) ) < $current_buffer_size && ! empty( $documents ) ) { | |
| 598 | + | |
| 599 | + $body[] = $next_document; | |
| 600 | + $current_body_size += $next_document_size; | |
| 601 | + | |
| 602 | + $can_add_more_documents = ( $current_body_size < $current_buffer_size && ! empty( $documents ) ); | |
| 603 | + if ( $can_add_more_documents ) { | |
| 514 | 604 | continue; |
| 515 | 605 | } |
| 516 | - if ( mb_strlen( implode( '', $body ) ) > $max_buffer_size ) { | |
| 606 | + | |
| 607 | + if ( $current_body_size > $max_buffer_size ) { | |
| 517 | 608 | // The last document added to body made it too big, so let's give it back. |
| 518 | - array_unshift( $documents, array_pop( $body ) ); | |
| 609 | + $removed_document = array_pop( $body ); | |
| 610 | + $current_body_size -= mb_strlen( $removed_document ); | |
| 611 | + array_unshift( $documents, $removed_document ); | |
| 519 | 612 | } |
| 520 | 613 | } |
| 521 | 614 | |
| 522 | 615 | // Try the request. |
| @@ -522,9 +615,9 @@ | ||
| 522 | 615 | // Try the request. |
| 523 | 616 | timer_start(); |
| 524 | 617 | $result = Elasticsearch::factory()->bulk_index( $this->get_index_name(), $this->slug, implode( '', $body ) ); |
| 525 | 618 | $request_time = timer_stop(); |
| 526 | - $requests++; | |
| 619 | + ++$requests; | |
| 527 | 620 | |
| 528 | 621 | /** |
| 529 | 622 | * Perform actions before a new batch of documents is processed. |
| 530 | 623 | * |
| @@ -541,37 +634,42 @@ | ||
| 541 | 634 | do_action( 'ep_after_send_dynamic_bulk_request', $result, $body, $documents, $min_buffer_size, $max_buffer_size, $current_buffer_size, $request_time ); |
| 542 | 635 | |
| 543 | 636 | // It failed, possibly adjust the buffer size and try again. |
| 544 | 637 | if ( is_wp_error( $result ) ) { |
| 638 | + $error_code = $result->get_error_code(); | |
| 639 | + | |
| 545 | 640 | // Too many requests, wait and try again. |
| 546 | - if ( 429 === $result->get_error_code() ) { | |
| 641 | + if ( 429 === $error_code ) { | |
| 547 | 642 | sleep( 2 ); |
| 548 | 643 | } |
| 549 | 644 | |
| 550 | 645 | // If the error is not a "Request too big" then we really fail this batch of documents. |
| 551 | - if ( 413 !== $result->get_error_code() ) { | |
| 646 | + if ( 413 !== $error_code ) { | |
| 552 | 647 | $results[] = $result; |
| 553 | 648 | continue; |
| 554 | 649 | } |
| 555 | 650 | |
| 556 | 651 | if ( count( $body ) === 1 ) { |
| 557 | - $max_buffer_size = min( $max_buffer_size, mb_strlen( implode( '', $body ) ) ); | |
| 558 | - $results[] = $result; | |
| 559 | - $body = []; | |
| 652 | + $max_buffer_size = min( $max_buffer_size, $current_body_size ); | |
| 653 | + $results[] = $result; | |
| 654 | + $body = []; | |
| 655 | + $current_body_size = 0; | |
| 560 | 656 | continue; |
| 561 | 657 | } |
| 562 | 658 | |
| 563 | 659 | // As the buffer is as small as possible, return the error. |
| 564 | - if ( mb_strlen( implode( '', $body ) ) === $min_buffer_size ) { | |
| 660 | + if ( $current_body_size === $min_buffer_size ) { | |
| 565 | 661 | $results[] = $result; |
| 566 | 662 | continue; |
| 567 | 663 | } |
| 568 | 664 | |
| 569 | 665 | // We have a too big buffer. Remove one doc from the body, and set both max and current as its size. |
| 570 | - array_unshift( $documents, array_pop( $body ) ); | |
| 666 | + $removed_document = array_pop( $body ); | |
| 667 | + $current_body_size -= mb_strlen( $removed_document ); | |
| 668 | + array_unshift( $documents, $removed_document ); | |
| 571 | 669 | |
| 572 | 670 | $max_buffer_size = count( $body ) ? |
| 573 | - max( $min_buffer_size, mb_strlen( implode( '', $body ) ) ) : | |
| 671 | + max( $min_buffer_size, $current_body_size ) : | |
| 574 | 672 | $min_buffer_size; |
| 575 | 673 | |
| 576 | 674 | $current_buffer_size = $max_buffer_size; |
| 577 | 675 | continue; |
| @@ -577,15 +675,16 @@ | ||
| 577 | 675 | continue; |
| 578 | 676 | } |
| 579 | 677 | |
| 580 | 678 | // Things worked so we can try to bump the buffer size. |
| 581 | - if ( $current_buffer_size < $max_buffer_size && mb_strlen( implode( '', $body ) ) > $current_buffer_size ) { | |
| 679 | + if ( $current_buffer_size < $max_buffer_size && $current_body_size > $current_buffer_size ) { | |
| 582 | 680 | $current_buffer_size = min( ( $current_buffer_size + $incremental_step ), $max_buffer_size ); |
| 583 | 681 | } |
| 584 | 682 | |
| 585 | 683 | $results[] = $result; |
| 586 | 684 | |
| 587 | - $body = []; | |
| 685 | + $body = []; | |
| 686 | + $current_body_size = 0; | |
| 588 | 687 | } while ( ! empty( $documents ) ); |
| 589 | 688 | |
| 590 | 689 | /** |
| 591 | 690 | * Perform actions after a batch of documents was processed. |
| @@ -668,9 +767,8 @@ | ||
| 668 | 767 | $prepared_meta[ $meta_key ] = array_map( array( $this, 'prepare_meta_value_types' ), $meta_values ); |
| 669 | 768 | } |
| 670 | 769 | |
| 671 | 770 | return $prepared_meta; |
| 672 | - | |
| 673 | 771 | } |
| 674 | 772 | |
| 675 | 773 | /** |
| 676 | 774 | * Prepare meta types for meta value |
| @@ -1105,9 +1203,9 @@ | ||
| 1105 | 1203 | return ( (string) $new_mapping['settings']['index.number_of_shards'] === $stored_mapping[ $this->get_index_name() ]['settings']['index']['number_of_shards'] ); |
| 1106 | 1204 | } |
| 1107 | 1205 | |
| 1108 | 1206 | /** |
| 1109 | - * Utilitary function to check if the indexable is being fully reindexed, i.e., | |
| 1207 | + * Utility function to check if the indexable is being fully reindexed, i.e., | |
| 1110 | 1208 | * the index was deleted, a new mapping was sent and content is being reindexed. |
| 1111 | 1209 | * |
| 1112 | 1210 | * @param int|null $blog_id Blog ID |
| 1113 | 1211 | * @return boolean |
| @@ -1148,9 +1246,9 @@ | ||
| 1148 | 1246 | * in a standardized format. This is necessary so we can genericize the index |
| 1149 | 1247 | * process across indexables. |
| 1150 | 1248 | * |
| 1151 | 1249 | * @param array $args Array to query DB against. |
| 1152 | - * @return boolean | |
| 1250 | + * @return array | |
| 1153 | 1251 | */ |
| 1154 | 1252 | abstract public function query_db( $args ); |
| 1155 | 1253 | |
| 1156 | 1254 | /** |
| @@ -1173,9 +1271,9 @@ | ||
| 1173 | 1271 | * @param array $search_fields Search fields |
| 1174 | 1272 | * @param array $query_vars Query vars |
| 1175 | 1273 | * @return SearchAlgorithm Instance of search algorithm to be used |
| 1176 | 1274 | */ |
| 1177 | - public function get_search_algorithm( string $search_text, array $search_fields, array $query_vars ) : \ElasticPress\SearchAlgorithm { | |
| 1275 | + public function get_search_algorithm( string $search_text, array $search_fields, array $query_vars ): \ElasticPress\SearchAlgorithm { | |
| 1178 | 1276 | /** |
| 1179 | 1277 | * Filter the search algorithm to be used |
| 1180 | 1278 | * |
| 1181 | 1279 | * @hook ep_{$indexable_slug}_search_algorithm |
| @@ -1196,22 +1294,23 @@ | ||
| 1196 | 1294 | * |
| 1197 | 1295 | * @since 4.3.0 |
| 1198 | 1296 | * @param null|int $blog_id (Optional) The blog ID. Sending `null` will use the current blog ID. |
| 1199 | 1297 | * @return array |
| 1298 | + * @throws \Exception An exception if meta fields are not available. | |
| 1200 | 1299 | */ |
| 1201 | 1300 | public function get_distinct_meta_field_keys( $blog_id = null ) { |
| 1202 | 1301 | $mapping = $this->get_mapping(); |
| 1203 | 1302 | |
| 1204 | 1303 | try { |
| 1205 | - if ( version_compare( Elasticsearch::factory()->get_elasticsearch_version(), '7.0', '<' ) ) { | |
| 1206 | - $meta_fields = $mapping[ $this->get_index_name( $blog_id ) ]['mappings']['post']['properties']['meta']['properties']; | |
| 1304 | + if ( version_compare( (string) Elasticsearch::factory()->get_elasticsearch_version(), '7.0', '<' ) ) { | |
| 1305 | + $meta_fields = (array) $mapping[ $this->get_index_name( $blog_id ) ]['mappings']['post']['properties']['meta']['properties']; | |
| 1207 | 1306 | } else { |
| 1208 | - $meta_fields = $mapping[ $this->get_index_name( $blog_id ) ]['mappings']['properties']['meta']['properties']; | |
| 1307 | + $meta_fields = (array) $mapping[ $this->get_index_name( $blog_id ) ]['mappings']['properties']['meta']['properties']; | |
| 1209 | 1308 | } |
| 1210 | 1309 | $meta_keys = array_values( array_keys( $meta_fields ) ); |
| 1211 | 1310 | sort( $meta_keys ); |
| 1212 | 1311 | } catch ( \Throwable $th ) { |
| 1213 | - return new \Exception( 'Meta fields not available.', 0 ); | |
| 1312 | + throw new \Exception( 'Meta fields not available.', 0 ); | |
| 1214 | 1313 | } |
| 1215 | 1314 | |
| 1216 | 1315 | return $meta_keys; |
| 1217 | 1316 | } |
| @@ -1277,9 +1376,9 @@ | ||
| 1277 | 1376 | * @since 4.5.0 |
| 1278 | 1377 | * @param array $mapping The mapping |
| 1279 | 1378 | * @return array |
| 1280 | 1379 | */ |
| 1281 | - public function add_ngram_analyzer( array $mapping ) : array { | |
| 1380 | + public function add_ngram_analyzer( array $mapping ): array { | |
| 1282 | 1381 | $mapping['settings']['analysis']['analyzer']['edge_ngram_analyzer'] = array( |
| 1283 | 1382 | 'type' => 'custom', |
| 1284 | 1383 | 'tokenizer' => 'standard', |
| 1285 | 1384 | 'filter' => array( |
