From f5f5e03db6e238b02f213d25311fd43384670650 Mon Sep 17 00:00:00 2001 From: Mathieu Lamiot Date: Mon, 15 Jan 2024 10:11:04 +0100 Subject: [PATCH 01/50] Rework Preload process_pending_jobs to adapt batch size based on previous batch and errors --- inc/Engine/Preload/Controller/PreloadUrl.php | 68 ++++++++++++++++++-- 1 file changed, 63 insertions(+), 5 deletions(-) diff --git a/inc/Engine/Preload/Controller/PreloadUrl.php b/inc/Engine/Preload/Controller/PreloadUrl.php index b2272cc53a..13f4af63be 100755 --- a/inc/Engine/Preload/Controller/PreloadUrl.php +++ b/inc/Engine/Preload/Controller/PreloadUrl.php @@ -5,6 +5,7 @@ use WP_Rocket\Admin\Options_Data; use WP_Rocket\Engine\Preload\Database\Queries\Cache; use WP_Filesystem_Direct; +use WP_Rocket\Logger\Logger; class PreloadUrl { use CheckExcludedTrait; @@ -181,9 +182,21 @@ protected function get_mobile_user_agent_prefix() { */ public function process_pending_jobs() { - $pending_actions = $this->queue->get_pending_preload_actions(); + // Retrieve previous batch size information + $max_batch_size = ( (int) apply_filters( 'rocket_preload_cache_pending_jobs_cron_rows_count', 45 ) ); + $min_batch_size = ( (int) apply_filters( 'rocket_preload_cache_min_in_progress_jobs_count', 5 ) ); + + $previous_batch_size = get_transient( 'rocket_preload_batch_size' ); + if ( ! $previous_batch_size ){ + $previous_batch_size = $max_batch_size; + } + + $last_batch_size_with_failures = get_transient( 'rocket_preload_batch_size_with_failures' ); + if ( ! $last_batch_size_with_failures ){ + $last_batch_size_with_failures = $max_batch_size; + } - $count = ( (int) apply_filters( 'rocket_preload_cache_pending_jobs_cron_rows_count', 45 ) ) - count( $pending_actions ); + // Get all in-progress jobs with request sent and no results. /** * Set the delay before an in-progress row is considered as outdated. * @@ -198,11 +211,12 @@ public function process_pending_jobs() { * @param int $count number of rows in batches. * @return int */ - (int) ( $count / 15 ) + 0 ); - $stuck_rows = $this->query->get_outdated_in_progress_jobs( $delay ); + // Make sure the request has been sent for those jobs + $pending_actions = $this->queue->get_pending_preload_actions(); $stuck_rows = array_filter( $stuck_rows, function ( $row ) use ( $pending_actions ) { @@ -215,10 +229,36 @@ function ( $row ) use ( $pending_actions ) { } ); + $n_failed_jobs = count( $stuck_rows ); + + // Make those hanging jobs failed. foreach ( $stuck_rows as $row ) { $this->query->make_status_failed( $row->id ); } - $rows = $this->query->get_pending_jobs( $count ); + + // Define the new batch size + if ( $n_failed_jobs > 0 ){ // Errors, reduce batch size + $next_batch_size = $previous_batch_size - $n_failed_jobs ; + } + else { // No errors, try to increase batch size + $next_batch_size = ($previous_batch_size + $last_batch_size_with_failures ) / 2; + } + + // Limit next_batch_size + $next_batch_size = min( $next_batch_size, $max_batch_size); //Not higher than 45 + $next_batch_size = max( $next_batch_size, $min_batch_size); //Not lower than 5 + + // Save the new transients + if ( $n_failed_jobs > 0 ){ // Errors, keep track + set_transient( 'rocket_preload_batch_size_with_failures', $previous_batch_size, HOUR_IN_SECONDS ); + } + set_transient( 'rocket_preload_batch_size', $next_batch_size, HOUR_IN_SECONDS ); + + // Apply a rate-limit at ActionScheduler level: no more than 45 pending actions. + $jobs_to_add = max( $next_batch_size - count( $pending_actions ), 0); + + //Add new jobs in progress + $rows = $this->query->get_pending_jobs( $jobs_to_add ); foreach ( $rows as $row ) { if ( $this->is_excluded_by_filter( $row->url ) ) { @@ -230,6 +270,24 @@ function ( $row ) use ( $pending_actions ) { $this->queue->add_job_preload_job_preload_url_async( $row->url ); } + + //Logs + $log_file = ABSPATH . "/MLT_preload_batch.log"; + error_log( "Preload - process_pendings_jobs", 3, $log_file ); + error_log( '\n', 3, $log_file ); + error_log( "previous_batch_size = " . $previous_batch_size, 3, $log_file ); + error_log( '\n', 3, $log_file ); + error_log( "last_batch_size_with_failures = " . $last_batch_size_with_failures, 3, $log_file ); + error_log( '\n', 3, $log_file ); + error_log( "n_failed_jobs = " . $n_failed_jobs, 3, $log_file ); + error_log( '\n', 3, $log_file ); + error_log( "next_batch_size = " . $next_batch_size, 3, $log_file ); + error_log( '\n', 3, $log_file ); + error_log( "pending_actions = " . count( $pending_actions ), 3, $log_file ); + error_log( '\n', 3, $log_file ); + error_log( "jobs_to_add = " . $jobs_to_add, 3, $log_file ); + error_log( '\n', 3, $log_file ); + } /** From fbdeb7e990fc195b96b7955886cdb733c803359b Mon Sep 17 00:00:00 2001 From: Mathieu Lamiot Date: Thu, 18 Jan 2024 18:10:13 +0100 Subject: [PATCH 02/50] Rework the batch size to depend on a request duration measurement --- inc/Engine/Preload/Controller/PreloadUrl.php | 84 ++++++++------------ 1 file changed, 33 insertions(+), 51 deletions(-) diff --git a/inc/Engine/Preload/Controller/PreloadUrl.php b/inc/Engine/Preload/Controller/PreloadUrl.php index 13f4af63be..b917d04f06 100755 --- a/inc/Engine/Preload/Controller/PreloadUrl.php +++ b/inc/Engine/Preload/Controller/PreloadUrl.php @@ -67,6 +67,12 @@ public function preload_url( string $url ) { return; } + // Do we need to compute the duration transient? + $check_duration = false; + if( ! get_transient( 'rocket_preload_request_duration' ) ) { + $check_duration = true; + } + $requests = [ [ 'url' => $url, @@ -121,7 +127,12 @@ public function preload_url( string $url ) { */ 'sslverify' => apply_filters( 'https_local_ssl_verify', false ), // phpcs:ignore WordPress.NamingConventions.PrefixAllGlobals.NonPrefixedHooknameFound ] - ); + ); + + if ( $check_duration ) { + $requests['headers']['blocking'] = true; + $requests['headers']['timeout'] = 20; + } /** * Filters the arguments for the preload request. @@ -137,10 +148,20 @@ public function preload_url( string $url ) { return; } + if ( $check_duration ) { + $start = microtime(true); + } + wp_safe_remote_get( user_trailingslashit( $request['url'] ), $headers ); + + if ( $check_duration ) { + $duration = (microtime(true) - $start); + set_transient('rocket_preload_request_duration', $duration); + $check_duration = false; + } /** * Filter the delay between each preload request. * @@ -186,16 +207,18 @@ public function process_pending_jobs() { $max_batch_size = ( (int) apply_filters( 'rocket_preload_cache_pending_jobs_cron_rows_count', 45 ) ); $min_batch_size = ( (int) apply_filters( 'rocket_preload_cache_min_in_progress_jobs_count', 5 ) ); - $previous_batch_size = get_transient( 'rocket_preload_batch_size' ); - if ( ! $previous_batch_size ){ - $previous_batch_size = $max_batch_size; + $preload_request_duration = get_transient( 'rocket_preload_request_duration' ); + if ( ! $preload_request_duration || $preload_request_duration <= 0 ){ + $next_batch_size = $max_batch_size; } - - $last_batch_size_with_failures = get_transient( 'rocket_preload_batch_size_with_failures' ); - if ( ! $last_batch_size_with_failures ){ - $last_batch_size_with_failures = $max_batch_size; + else{ // jobs per second / seconds per minute -> jobs per minute + $next_batch_size = (1/$preload_request_duration) / 2 * 60; // /2 for mobile/desktop } + // Limit next_batch_size + $next_batch_size = min( $next_batch_size, $max_batch_size); //Not higher than 45 + $next_batch_size = max( $next_batch_size, $min_batch_size); //Not lower than 5 + // Get all in-progress jobs with request sent and no results. /** * Set the delay before an in-progress row is considered as outdated. @@ -211,7 +234,7 @@ public function process_pending_jobs() { * @param int $count number of rows in batches. * @return int */ - 0 + (int) ( $max_batch_size / 15 ) ); $stuck_rows = $this->query->get_outdated_in_progress_jobs( $delay ); @@ -229,36 +252,13 @@ function ( $row ) use ( $pending_actions ) { } ); - $n_failed_jobs = count( $stuck_rows ); - // Make those hanging jobs failed. foreach ( $stuck_rows as $row ) { $this->query->make_status_failed( $row->id ); } - // Define the new batch size - if ( $n_failed_jobs > 0 ){ // Errors, reduce batch size - $next_batch_size = $previous_batch_size - $n_failed_jobs ; - } - else { // No errors, try to increase batch size - $next_batch_size = ($previous_batch_size + $last_batch_size_with_failures ) / 2; - } - - // Limit next_batch_size - $next_batch_size = min( $next_batch_size, $max_batch_size); //Not higher than 45 - $next_batch_size = max( $next_batch_size, $min_batch_size); //Not lower than 5 - - // Save the new transients - if ( $n_failed_jobs > 0 ){ // Errors, keep track - set_transient( 'rocket_preload_batch_size_with_failures', $previous_batch_size, HOUR_IN_SECONDS ); - } - set_transient( 'rocket_preload_batch_size', $next_batch_size, HOUR_IN_SECONDS ); - - // Apply a rate-limit at ActionScheduler level: no more than 45 pending actions. - $jobs_to_add = max( $next_batch_size - count( $pending_actions ), 0); - //Add new jobs in progress - $rows = $this->query->get_pending_jobs( $jobs_to_add ); + $rows = $this->query->get_pending_jobs( $next_batch_size ); foreach ( $rows as $row ) { if ( $this->is_excluded_by_filter( $row->url ) ) { @@ -270,24 +270,6 @@ function ( $row ) use ( $pending_actions ) { $this->queue->add_job_preload_job_preload_url_async( $row->url ); } - - //Logs - $log_file = ABSPATH . "/MLT_preload_batch.log"; - error_log( "Preload - process_pendings_jobs", 3, $log_file ); - error_log( '\n', 3, $log_file ); - error_log( "previous_batch_size = " . $previous_batch_size, 3, $log_file ); - error_log( '\n', 3, $log_file ); - error_log( "last_batch_size_with_failures = " . $last_batch_size_with_failures, 3, $log_file ); - error_log( '\n', 3, $log_file ); - error_log( "n_failed_jobs = " . $n_failed_jobs, 3, $log_file ); - error_log( '\n', 3, $log_file ); - error_log( "next_batch_size = " . $next_batch_size, 3, $log_file ); - error_log( '\n', 3, $log_file ); - error_log( "pending_actions = " . count( $pending_actions ), 3, $log_file ); - error_log( '\n', 3, $log_file ); - error_log( "jobs_to_add = " . $jobs_to_add, 3, $log_file ); - error_log( '\n', 3, $log_file ); - } /** From 375d2ec6d4b1c888ed52e1cf70bb8062de8232a4 Mon Sep 17 00:00:00 2001 From: Mathieu Lamiot Date: Thu, 18 Jan 2024 18:11:36 +0100 Subject: [PATCH 03/50] Set transient expiration --- inc/Engine/Preload/Controller/PreloadUrl.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inc/Engine/Preload/Controller/PreloadUrl.php b/inc/Engine/Preload/Controller/PreloadUrl.php index b917d04f06..cf0b3a1075 100755 --- a/inc/Engine/Preload/Controller/PreloadUrl.php +++ b/inc/Engine/Preload/Controller/PreloadUrl.php @@ -159,7 +159,7 @@ public function preload_url( string $url ) { if ( $check_duration ) { $duration = (microtime(true) - $start); - set_transient('rocket_preload_request_duration', $duration); + set_transient('rocket_preload_request_duration', $duration, 5 * 60); $check_duration = false; } /** From a0af9fc539d3cd8bfb9263ed6d58581b603c1414 Mon Sep 17 00:00:00 2001 From: Mathieu Lamiot Date: Thu, 18 Jan 2024 18:13:31 +0100 Subject: [PATCH 04/50] Move code around to limit changes --- inc/Engine/Preload/Controller/PreloadUrl.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/inc/Engine/Preload/Controller/PreloadUrl.php b/inc/Engine/Preload/Controller/PreloadUrl.php index cf0b3a1075..28ce307d3b 100755 --- a/inc/Engine/Preload/Controller/PreloadUrl.php +++ b/inc/Engine/Preload/Controller/PreloadUrl.php @@ -203,6 +203,8 @@ protected function get_mobile_user_agent_prefix() { */ public function process_pending_jobs() { + $pending_actions = $this->queue->get_pending_preload_actions(); + // Retrieve previous batch size information $max_batch_size = ( (int) apply_filters( 'rocket_preload_cache_pending_jobs_cron_rows_count', 45 ) ); $min_batch_size = ( (int) apply_filters( 'rocket_preload_cache_min_in_progress_jobs_count', 5 ) ); @@ -239,7 +241,6 @@ public function process_pending_jobs() { $stuck_rows = $this->query->get_outdated_in_progress_jobs( $delay ); // Make sure the request has been sent for those jobs - $pending_actions = $this->queue->get_pending_preload_actions(); $stuck_rows = array_filter( $stuck_rows, function ( $row ) use ( $pending_actions ) { From c5763cf235d7a6dafbaa84b0fa1b8ac3f6eb5ff1 Mon Sep 17 00:00:00 2001 From: Opeyemi Ibrahim Date: Fri, 26 Jan 2024 12:11:41 +0100 Subject: [PATCH 05/50] Compute all request duration, used to determine the next batch size --- inc/Engine/Preload/Controller/PreloadUrl.php | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/inc/Engine/Preload/Controller/PreloadUrl.php b/inc/Engine/Preload/Controller/PreloadUrl.php index 28ce307d3b..7a1ab980bf 100755 --- a/inc/Engine/Preload/Controller/PreloadUrl.php +++ b/inc/Engine/Preload/Controller/PreloadUrl.php @@ -69,7 +69,9 @@ public function preload_url( string $url ) { // Do we need to compute the duration transient? $check_duration = false; + $previous_request_durations = 0; if( ! get_transient( 'rocket_preload_request_duration' ) ) { + $previous_request_durations = get_transient( 'rocket_preload_previous_request_durations' ) ?? 0; $check_duration = true; } @@ -128,7 +130,7 @@ public function preload_url( string $url ) { 'sslverify' => apply_filters( 'https_local_ssl_verify', false ), // phpcs:ignore WordPress.NamingConventions.PrefixAllGlobals.NonPrefixedHooknameFound ] ); - + if ( $check_duration ) { $requests['headers']['blocking'] = true; $requests['headers']['timeout'] = 20; @@ -149,7 +151,7 @@ public function preload_url( string $url ) { } if ( $check_duration ) { - $start = microtime(true); + $start = microtime(true); } wp_safe_remote_get( @@ -160,6 +162,8 @@ public function preload_url( string $url ) { if ( $check_duration ) { $duration = (microtime(true) - $start); set_transient('rocket_preload_request_duration', $duration, 5 * 60); + $previous_request_durations = $previous_request_durations + $duration; + set_transient('rocket_preload_previous_request_durations', $previous_request_durations, 60 * 60); $check_duration = false; } /** @@ -168,9 +172,9 @@ public function preload_url( string $url ) { * @param float $delay_between the defined delay. * @returns float */ - $delay_between = apply_filters( 'rocket_preload_delay_between_requests', 500000 ); + $delay_between = apply_filters( 'rocket_preload_delay_between_requests', 2 ); - usleep( $delay_between ); + sleep( $delay_between ); } } @@ -209,7 +213,7 @@ public function process_pending_jobs() { $max_batch_size = ( (int) apply_filters( 'rocket_preload_cache_pending_jobs_cron_rows_count', 45 ) ); $min_batch_size = ( (int) apply_filters( 'rocket_preload_cache_min_in_progress_jobs_count', 5 ) ); - $preload_request_duration = get_transient( 'rocket_preload_request_duration' ); + $preload_request_duration = get_transient( 'rocket_preload_previous_request_durations' ); if ( ! $preload_request_duration || $preload_request_duration <= 0 ){ $next_batch_size = $max_batch_size; } From 42f85be969d9fd39c52334ec88d5576aedf759e0 Mon Sep 17 00:00:00 2001 From: Opeyemi Ibrahim Date: Mon, 29 Jan 2024 13:42:41 +0100 Subject: [PATCH 06/50] Modified computation as suggested by @mathieu --- inc/Engine/Preload/Controller/PreloadUrl.php | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/inc/Engine/Preload/Controller/PreloadUrl.php b/inc/Engine/Preload/Controller/PreloadUrl.php index 7a1ab980bf..08ff9162ec 100755 --- a/inc/Engine/Preload/Controller/PreloadUrl.php +++ b/inc/Engine/Preload/Controller/PreloadUrl.php @@ -162,7 +162,13 @@ public function preload_url( string $url ) { if ( $check_duration ) { $duration = (microtime(true) - $start); set_transient('rocket_preload_request_duration', $duration, 5 * 60); - $previous_request_durations = $previous_request_durations + $duration; + + if ($previous_request_durations <= 0) { + $previous_request_durations = $duration; + } else { + $previous_request_durations = $previous_request_durations * 0.7 + $duration * 0.3; + } + set_transient('rocket_preload_previous_request_durations', $previous_request_durations, 60 * 60); $check_duration = false; } From 52eab5fc7872bbe7f6ddb8f9e05f3309ce343532 Mon Sep 17 00:00:00 2001 From: Mathieu Lamiot Date: Mon, 29 Jan 2024 18:07:47 +0100 Subject: [PATCH 07/50] Clean float to int conversion Conservative approach by initializing batch size to min value Check duration once per batch for the average Remove unused transient Remove transient on uninstall --- inc/Engine/Preload/Controller/PreloadUrl.php | 13 ++++--------- .../inc/Engine/WPRocketUninstall/uninstall.php | 1 + 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/inc/Engine/Preload/Controller/PreloadUrl.php b/inc/Engine/Preload/Controller/PreloadUrl.php index 08ff9162ec..e099622603 100755 --- a/inc/Engine/Preload/Controller/PreloadUrl.php +++ b/inc/Engine/Preload/Controller/PreloadUrl.php @@ -68,12 +68,8 @@ public function preload_url( string $url ) { } // Do we need to compute the duration transient? - $check_duration = false; - $previous_request_durations = 0; - if( ! get_transient( 'rocket_preload_request_duration' ) ) { - $previous_request_durations = get_transient( 'rocket_preload_previous_request_durations' ) ?? 0; - $check_duration = true; - } + $check_duration = true; + $previous_request_durations = get_transient( 'rocket_preload_previous_request_durations' ) ?? 0; $requests = [ [ @@ -161,7 +157,6 @@ public function preload_url( string $url ) { if ( $check_duration ) { $duration = (microtime(true) - $start); - set_transient('rocket_preload_request_duration', $duration, 5 * 60); if ($previous_request_durations <= 0) { $previous_request_durations = $duration; @@ -221,10 +216,10 @@ public function process_pending_jobs() { $preload_request_duration = get_transient( 'rocket_preload_previous_request_durations' ); if ( ! $preload_request_duration || $preload_request_duration <= 0 ){ - $next_batch_size = $max_batch_size; + $next_batch_size = $min_batch_size; } else{ // jobs per second / seconds per minute -> jobs per minute - $next_batch_size = (1/$preload_request_duration) / 2 * 60; // /2 for mobile/desktop + $next_batch_size = round((1/$preload_request_duration) / 2 * 60); // /2 for mobile/desktop } // Limit next_batch_size diff --git a/tests/Integration/inc/Engine/WPRocketUninstall/uninstall.php b/tests/Integration/inc/Engine/WPRocketUninstall/uninstall.php index 087b72b9d1..4752469d05 100644 --- a/tests/Integration/inc/Engine/WPRocketUninstall/uninstall.php +++ b/tests/Integration/inc/Engine/WPRocketUninstall/uninstall.php @@ -41,6 +41,7 @@ class Test_Uninstall extends FilesystemTestCase { 'rocket_preload_complete' => null, 'rocket_preload_complete_time' => null, 'rocket_preload_errors' => null, + 'rocket_preload_previous_request_durations' => null, 'rocket_database_optimization_process' => null, 'rocket_database_optimization_process_complete' => null, ]; From 38fe3e4313a72882c44b5d8e84d3ac89ab23d703 Mon Sep 17 00:00:00 2001 From: Mathieu Lamiot Date: Mon, 29 Jan 2024 18:35:28 +0100 Subject: [PATCH 08/50] Prevent the check_duration to happen for every AS task --- inc/Engine/Preload/Controller/PreloadUrl.php | 14 ++++++++------ .../inc/Engine/WPRocketUninstall/uninstall.php | 1 + 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/inc/Engine/Preload/Controller/PreloadUrl.php b/inc/Engine/Preload/Controller/PreloadUrl.php index e099622603..0bbe135f16 100755 --- a/inc/Engine/Preload/Controller/PreloadUrl.php +++ b/inc/Engine/Preload/Controller/PreloadUrl.php @@ -67,10 +67,9 @@ public function preload_url( string $url ) { return; } - // Do we need to compute the duration transient? - $check_duration = true; - $previous_request_durations = get_transient( 'rocket_preload_previous_request_durations' ) ?? 0; - + // Should we perform a duration check? + $check_duration = get_transient( 'rocket_preload_check_duration' ) ? false : true; + $requests = [ [ 'url' => $url, @@ -156,15 +155,18 @@ public function preload_url( string $url ) { ); if ( $check_duration ) { - $duration = (microtime(true) - $start); + $duration = (microtime(true) - $start); //Duration of the request + //Update average duration + $previous_request_durations = get_transient( 'rocket_preload_previous_request_durations' ) ?? 0; if ($previous_request_durations <= 0) { $previous_request_durations = $duration; } else { $previous_request_durations = $previous_request_durations * 0.7 + $duration * 0.3; } - set_transient('rocket_preload_previous_request_durations', $previous_request_durations, 60 * 60); + + set_transient('rocket_preload_check_duration', $duration, 60); //Don't check request duration for 1 minute. $check_duration = false; } /** diff --git a/tests/Integration/inc/Engine/WPRocketUninstall/uninstall.php b/tests/Integration/inc/Engine/WPRocketUninstall/uninstall.php index 4752469d05..be70d3052d 100644 --- a/tests/Integration/inc/Engine/WPRocketUninstall/uninstall.php +++ b/tests/Integration/inc/Engine/WPRocketUninstall/uninstall.php @@ -42,6 +42,7 @@ class Test_Uninstall extends FilesystemTestCase { 'rocket_preload_complete_time' => null, 'rocket_preload_errors' => null, 'rocket_preload_previous_request_durations' => null, + 'rocket_preload_check_duration' => null, 'rocket_database_optimization_process' => null, 'rocket_database_optimization_process_complete' => null, ]; From 73c7dbc96210b8b610c69367d2e998072ae3fc3f Mon Sep 17 00:00:00 2001 From: Mathieu Lamiot Date: Tue, 30 Jan 2024 08:50:05 +0100 Subject: [PATCH 09/50] Bring back behaviors like delay_between and limit on number of pending actions as develop --- inc/Engine/Preload/Controller/PreloadUrl.php | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/inc/Engine/Preload/Controller/PreloadUrl.php b/inc/Engine/Preload/Controller/PreloadUrl.php index 0bbe135f16..77835a48a6 100755 --- a/inc/Engine/Preload/Controller/PreloadUrl.php +++ b/inc/Engine/Preload/Controller/PreloadUrl.php @@ -5,7 +5,6 @@ use WP_Rocket\Admin\Options_Data; use WP_Rocket\Engine\Preload\Database\Queries\Cache; use WP_Filesystem_Direct; -use WP_Rocket\Logger\Logger; class PreloadUrl { use CheckExcludedTrait; @@ -175,9 +174,9 @@ public function preload_url( string $url ) { * @param float $delay_between the defined delay. * @returns float */ - $delay_between = apply_filters( 'rocket_preload_delay_between_requests', 2 ); + $delay_between = apply_filters( 'rocket_preload_delay_between_requests', 500000 ); - sleep( $delay_between ); + usleep( $delay_between ); } } @@ -213,7 +212,7 @@ public function process_pending_jobs() { $pending_actions = $this->queue->get_pending_preload_actions(); // Retrieve previous batch size information - $max_batch_size = ( (int) apply_filters( 'rocket_preload_cache_pending_jobs_cron_rows_count', 45 ) ); + $max_batch_size = ( (int) apply_filters( 'rocket_preload_cache_pending_jobs_cron_rows_count', 45 ) ) - count( $pending_actions ); $min_batch_size = ( (int) apply_filters( 'rocket_preload_cache_min_in_progress_jobs_count', 5 ) ); $preload_request_duration = get_transient( 'rocket_preload_previous_request_durations' ); From 979e196b0da52a7e9df0efb00d6768194530a65c Mon Sep 17 00:00:00 2001 From: Mathieu Lamiot Date: Tue, 30 Jan 2024 22:32:43 +0100 Subject: [PATCH 10/50] Fix blocking headers that was not applied --- inc/Engine/Preload/Controller/PreloadUrl.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inc/Engine/Preload/Controller/PreloadUrl.php b/inc/Engine/Preload/Controller/PreloadUrl.php index 77835a48a6..bea6b077de 100755 --- a/inc/Engine/Preload/Controller/PreloadUrl.php +++ b/inc/Engine/Preload/Controller/PreloadUrl.php @@ -126,8 +126,8 @@ public function preload_url( string $url ) { ); if ( $check_duration ) { - $requests['headers']['blocking'] = true; - $requests['headers']['timeout'] = 20; + $headers['blocking'] = true; + $headers['timeout'] = 20; } /** From 640b38343c8270256f46b69884ef9de52bec1628 Mon Sep 17 00:00:00 2001 From: Mathieu Lamiot Date: Thu, 1 Feb 2024 14:33:03 +0100 Subject: [PATCH 11/50] Adjust batch size formula based on local and gamma experiments --- inc/Engine/Preload/Controller/PreloadUrl.php | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/inc/Engine/Preload/Controller/PreloadUrl.php b/inc/Engine/Preload/Controller/PreloadUrl.php index bea6b077de..76bb98d45b 100755 --- a/inc/Engine/Preload/Controller/PreloadUrl.php +++ b/inc/Engine/Preload/Controller/PreloadUrl.php @@ -163,7 +163,7 @@ public function preload_url( string $url ) { } else { $previous_request_durations = $previous_request_durations * 0.7 + $duration * 0.3; } - set_transient('rocket_preload_previous_request_durations', $previous_request_durations, 60 * 60); + set_transient('rocket_preload_previous_request_durations', $previous_request_durations, 5 * 60); set_transient('rocket_preload_check_duration', $duration, 60); //Don't check request duration for 1 minute. $check_duration = false; @@ -211,16 +211,17 @@ public function process_pending_jobs() { $pending_actions = $this->queue->get_pending_preload_actions(); - // Retrieve previous batch size information + // Retrieve batch size limits and request timing estimaiton $max_batch_size = ( (int) apply_filters( 'rocket_preload_cache_pending_jobs_cron_rows_count', 45 ) ) - count( $pending_actions ); $min_batch_size = ( (int) apply_filters( 'rocket_preload_cache_min_in_progress_jobs_count', 5 ) ); - $preload_request_duration = get_transient( 'rocket_preload_previous_request_durations' ); - if ( ! $preload_request_duration || $preload_request_duration <= 0 ){ - $next_batch_size = $min_batch_size; - } - else{ // jobs per second / seconds per minute -> jobs per minute - $next_batch_size = round((1/$preload_request_duration) / 2 * 60); // /2 for mobile/desktop + + //Estimate batch size based on request duration + if ( ! $preload_request_duration ){ + $next_batch_size = $min_batch_size; // In case no estimation or there is an issue with the value. + } else { + // Linear function: 2s -> 45 jobs // 10s -> 5 jobs + $next_batch_size = round( -5 * $preload_request_duration + 55 ); } // Limit next_batch_size From 7eb83c9e01073d80419d7e7c8930cea8fd919e76 Mon Sep 17 00:00:00 2001 From: Mathieu Lamiot Date: Thu, 1 Feb 2024 14:50:01 +0100 Subject: [PATCH 12/50] fix linter --- inc/Engine/Preload/Controller/PreloadUrl.php | 40 ++++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/inc/Engine/Preload/Controller/PreloadUrl.php b/inc/Engine/Preload/Controller/PreloadUrl.php index 76bb98d45b..9b68958c5d 100755 --- a/inc/Engine/Preload/Controller/PreloadUrl.php +++ b/inc/Engine/Preload/Controller/PreloadUrl.php @@ -68,7 +68,7 @@ public function preload_url( string $url ) { // Should we perform a duration check? $check_duration = get_transient( 'rocket_preload_check_duration' ) ? false : true; - + $requests = [ [ 'url' => $url, @@ -145,7 +145,7 @@ public function preload_url( string $url ) { } if ( $check_duration ) { - $start = microtime(true); + $start = microtime( true ); } wp_safe_remote_get( @@ -154,18 +154,18 @@ public function preload_url( string $url ) { ); if ( $check_duration ) { - $duration = (microtime(true) - $start); //Duration of the request + $duration = ( microtime( true ) - $start ); // Duration of the request. - //Update average duration - $previous_request_durations = get_transient( 'rocket_preload_previous_request_durations' ) ?? 0; - if ($previous_request_durations <= 0) { + // Update average duration. + $previous_request_durations = get_transient( 'rocket_preload_previous_request_durations' ) ?? 0; + if ( $previous_request_durations <= 0 ) { $previous_request_durations = $duration; } else { $previous_request_durations = $previous_request_durations * 0.7 + $duration * 0.3; } - set_transient('rocket_preload_previous_request_durations', $previous_request_durations, 5 * 60); + set_transient( 'rocket_preload_previous_request_durations', $previous_request_durations, 5 * 60 ); - set_transient('rocket_preload_check_duration', $duration, 60); //Don't check request duration for 1 minute. + set_transient( 'rocket_preload_check_duration', $duration, 60 ); // Don't check request duration for 1 minute. $check_duration = false; } /** @@ -211,22 +211,22 @@ public function process_pending_jobs() { $pending_actions = $this->queue->get_pending_preload_actions(); - // Retrieve batch size limits and request timing estimaiton - $max_batch_size = ( (int) apply_filters( 'rocket_preload_cache_pending_jobs_cron_rows_count', 45 ) ) - count( $pending_actions ); - $min_batch_size = ( (int) apply_filters( 'rocket_preload_cache_min_in_progress_jobs_count', 5 ) ); + // Retrieve batch size limits and request timing estimaiton. + $max_batch_size = ( (int) apply_filters( 'rocket_preload_cache_pending_jobs_cron_rows_count', 45 ) ) - count( $pending_actions ); + $min_batch_size = ( (int) apply_filters( 'rocket_preload_cache_min_in_progress_jobs_count', 5 ) ); $preload_request_duration = get_transient( 'rocket_preload_previous_request_durations' ); - //Estimate batch size based on request duration - if ( ! $preload_request_duration ){ + // Estimate batch size based on request duration/ + if ( ! $preload_request_duration ) { $next_batch_size = $min_batch_size; // In case no estimation or there is an issue with the value. } else { - // Linear function: 2s -> 45 jobs // 10s -> 5 jobs - $next_batch_size = round( -5 * $preload_request_duration + 55 ); + // Linear function: 2s -> 45 jobs // 10s -> 5 jobs. + $next_batch_size = round( -5 * $preload_request_duration + 55 ); } - // Limit next_batch_size - $next_batch_size = min( $next_batch_size, $max_batch_size); //Not higher than 45 - $next_batch_size = max( $next_batch_size, $min_batch_size); //Not lower than 5 + // Limit next_batch_size. + $next_batch_size = min( $next_batch_size, $max_batch_size ); // Not higher than 45. + $next_batch_size = max( $next_batch_size, $min_batch_size ); // Not lower than 5. // Get all in-progress jobs with request sent and no results. /** @@ -247,7 +247,7 @@ public function process_pending_jobs() { ); $stuck_rows = $this->query->get_outdated_in_progress_jobs( $delay ); - // Make sure the request has been sent for those jobs + // Make sure the request has been sent for those jobs. $stuck_rows = array_filter( $stuck_rows, function ( $row ) use ( $pending_actions ) { @@ -265,7 +265,7 @@ function ( $row ) use ( $pending_actions ) { $this->query->make_status_failed( $row->id ); } - //Add new jobs in progress + // Add new jobs in progress. $rows = $this->query->get_pending_jobs( $next_batch_size ); foreach ( $rows as $row ) { From 320bc7e8f8c3686c09dea6d1e23623cd3602eb9c Mon Sep 17 00:00:00 2001 From: Mathieu Lamiot Date: Thu, 1 Feb 2024 14:52:28 +0100 Subject: [PATCH 13/50] Fix typo while fixing linter... duh --- inc/Engine/Preload/Controller/PreloadUrl.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inc/Engine/Preload/Controller/PreloadUrl.php b/inc/Engine/Preload/Controller/PreloadUrl.php index 9b68958c5d..2b04f628be 100755 --- a/inc/Engine/Preload/Controller/PreloadUrl.php +++ b/inc/Engine/Preload/Controller/PreloadUrl.php @@ -216,7 +216,7 @@ public function process_pending_jobs() { $min_batch_size = ( (int) apply_filters( 'rocket_preload_cache_min_in_progress_jobs_count', 5 ) ); $preload_request_duration = get_transient( 'rocket_preload_previous_request_durations' ); - // Estimate batch size based on request duration/ + // Estimate batch size based on request duration. if ( ! $preload_request_duration ) { $next_batch_size = $min_batch_size; // In case no estimation or there is an issue with the value. } else { From 2986803730e4b29b92b4d4d982e04095f31dfc26 Mon Sep 17 00:00:00 2001 From: Opeyemi Ibrahim Date: Wed, 14 Feb 2024 11:43:56 +0100 Subject: [PATCH 14/50] Improve testing --- .../inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php b/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php index 691d60c026..961ab7a4f0 100644 --- a/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php +++ b/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php @@ -30,7 +30,8 @@ 'sslverify' => false, ] ], - ] + 'rocket_preload_check_duration' => 1, + ], ], 'mobileActivatedShouldPreloadTwice' => [ 'config' => [ @@ -54,6 +55,7 @@ 'sslverify' => false, ] ], + 'rocket_preload_check_duration' => 1, ] ] ]; From 93c3e4802337d600a0ee3965482848c5dafcce7d Mon Sep 17 00:00:00 2001 From: Opeyemi Ibrahim Date: Wed, 14 Feb 2024 12:12:00 +0100 Subject: [PATCH 15/50] Add get-transient method to test method --- .../inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php | 4 ++-- .../inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php b/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php index 961ab7a4f0..35178645ad 100644 --- a/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php +++ b/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php @@ -30,7 +30,7 @@ 'sslverify' => false, ] ], - 'rocket_preload_check_duration' => 1, + 'transient' => 0, ], ], 'mobileActivatedShouldPreloadTwice' => [ @@ -55,7 +55,7 @@ 'sslverify' => false, ] ], - 'rocket_preload_check_duration' => 1, + 'transient' => 0, ] ] ]; diff --git a/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php b/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php index ffdef2f31a..fbd90efc64 100644 --- a/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php +++ b/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php @@ -41,6 +41,8 @@ protected function setUp(): void * @dataProvider configTestData */ public function testShouldDoAsExpected($config) { + Functions\when( 'get_transient' ) + ->justReturn( $config['transient'] ); $this->options->expects()->get('do_caching_mobile_files', false)->andReturn($config['cache_mobile']); $this->controller->expects()->is_already_cached($config['url'])->andReturn($config['cache_exists']); $this->configureRequest($config); From 0cfb05e065f1c6428b993fd4eae8992972577343 Mon Sep 17 00:00:00 2001 From: Opeyemi Ibrahim Date: Wed, 14 Feb 2024 12:54:25 +0100 Subject: [PATCH 16/50] Fixed undefined test error --- .../inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php b/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php index 35178645ad..7827152d59 100644 --- a/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php +++ b/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php @@ -14,6 +14,7 @@ 'sslverify' => false, ] ], + 'transient' => 0, ] ], 'mobileNotActivatedShouldPreloadOnlyOnce' => [ From 159840dd5d55692d15b83ac95bd8077f4a9561a3 Mon Sep 17 00:00:00 2001 From: Opeyemi Ibrahim Date: Wed, 14 Feb 2024 14:48:39 +0100 Subject: [PATCH 17/50] Modified test method :phew: --- .../inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php b/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php index fbd90efc64..3869ee5f6a 100644 --- a/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php +++ b/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php @@ -43,6 +43,9 @@ protected function setUp(): void public function testShouldDoAsExpected($config) { Functions\when( 'get_transient' ) ->justReturn( $config['transient'] ); + Functions\when( 'wp_safe_remote_get' ) + ->justReturn(); + $this->options->expects()->get('do_caching_mobile_files', false)->andReturn($config['cache_mobile']); $this->controller->expects()->is_already_cached($config['url'])->andReturn($config['cache_exists']); $this->configureRequest($config); From ab3c7bfc428ca32ea6081815266a9bb2d41c10ab Mon Sep 17 00:00:00 2001 From: Opeyemi Ibrahim Date: Wed, 14 Feb 2024 19:00:53 +0100 Subject: [PATCH 18/50] Modified testing --- .../inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php | 6 +++--- .../Preload/Controller/PreloadUrl/processPendingJobs.php | 5 +++-- .../inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php | 4 +++- .../Preload/Controller/PreloadUrl/processPendingJobs.php | 4 +++- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php b/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php index 7827152d59..21edbdd4c2 100644 --- a/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php +++ b/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php @@ -25,13 +25,13 @@ 'user_agent' => 'user_agent', 'request' => [ 'config' => [ - 'blocking' => false, - 'timeout' => 0.01, + 'blocking' => true, + 'timeout' => 20, 'user-agent' => 'WP Rocket/Preload', 'sslverify' => false, ] ], - 'transient' => 0, + 'transient' => 1, ], ], 'mobileActivatedShouldPreloadTwice' => [ diff --git a/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/processPendingJobs.php b/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/processPendingJobs.php index 6d97f12d37..4d08396aed 100644 --- a/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/processPendingJobs.php +++ b/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/processPendingJobs.php @@ -48,12 +48,13 @@ false, true, ], - 'rows' => 101, + 'rows' => 5, 'jobs' => [ $row1, $row2, $row3 - ] + ], + 'transient' => 0, ], 'expected' => [ 'outdated_jobs_id' => [ diff --git a/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php b/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php index 3869ee5f6a..36763d7a78 100644 --- a/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php +++ b/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php @@ -42,7 +42,9 @@ protected function setUp(): void */ public function testShouldDoAsExpected($config) { Functions\when( 'get_transient' ) - ->justReturn( $config['transient'] ); + ->justReturn(); + Functions\when( 'set_transient' ) + ->justReturn(); Functions\when( 'wp_safe_remote_get' ) ->justReturn(); diff --git a/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/processPendingJobs.php b/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/processPendingJobs.php index 68fbf0791e..467d880f5c 100644 --- a/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/processPendingJobs.php +++ b/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/processPendingJobs.php @@ -11,6 +11,7 @@ use WP_Rocket\Engine\Preload\Database\Queries\RocketCache; use WP_Rocket\Tests\Unit\TestCase; use Brain\Monkey\Filters; +use Brain\Monkey\Functions; /** * @covers \WP_Rocket\Engine\Preload\Controller\PreloadUrl::process_pending_jobs @@ -38,7 +39,8 @@ protected function setUp(): void * @dataProvider configTestData */ public function testShouldDoAsExpected($config, $expected) { - + Functions\when( 'get_transient' ) + ->justReturn( $config['transient'] ); $this->queue->expects()->get_pending_preload_actions()->andReturn([]); Filters\expectApplied('rocket_preload_cache_pending_jobs_cron_rows_count')->with(100)->andReturn($config['rows']); $this->query->expects(self::once())->method( 'get_outdated_in_progress_jobs' )->with()->willReturn($config['outdated_jobs']); From 08657883eea18f08fc9cbd99310688236a821a72 Mon Sep 17 00:00:00 2001 From: Opeyemi Ibrahim Date: Thu, 15 Feb 2024 00:22:35 +0100 Subject: [PATCH 19/50] Fixed test error :feat: --- .../inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php | 4 ++-- .../inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php b/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php index 21edbdd4c2..c4e7175dc9 100644 --- a/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php +++ b/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php @@ -42,8 +42,8 @@ 'user_agent' => 'user_agent', 'request' => [ 'config' => [ - 'blocking' => false, - 'timeout' => 0.01, + 'blocking' => true, + 'timeout' => 20, 'user-agent' => 'WP Rocket/Preload', 'sslverify' => false, ] diff --git a/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php b/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php index 36763d7a78..1a6e74ad02 100644 --- a/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php +++ b/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php @@ -45,8 +45,6 @@ public function testShouldDoAsExpected($config) { ->justReturn(); Functions\when( 'set_transient' ) ->justReturn(); - Functions\when( 'wp_safe_remote_get' ) - ->justReturn(); $this->options->expects()->get('do_caching_mobile_files', false)->andReturn($config['cache_mobile']); $this->controller->expects()->is_already_cached($config['url'])->andReturn($config['cache_exists']); From 4b9f9669596298cb1ea9bd63559bbf4750665163 Mon Sep 17 00:00:00 2001 From: Opeyemi Ibrahim Date: Thu, 15 Feb 2024 01:54:19 +0100 Subject: [PATCH 20/50] fixed integration test --- .../Preload/Cron/Subscriber/processPendingUrls.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/Fixtures/inc/Engine/Preload/Cron/Subscriber/processPendingUrls.php b/tests/Fixtures/inc/Engine/Preload/Cron/Subscriber/processPendingUrls.php index 69225f2e07..7c1a0c56a6 100644 --- a/tests/Fixtures/inc/Engine/Preload/Cron/Subscriber/processPendingUrls.php +++ b/tests/Fixtures/inc/Engine/Preload/Cron/Subscriber/processPendingUrls.php @@ -120,7 +120,7 @@ ], [ 'url' => 'http://example.org/test3', - 'status' => 'pending' + 'status' => 'failed' ], [ 'url' => 'http://example.org/test4', @@ -147,15 +147,15 @@ ], [ 'url' => 'http://example.org/test3', - 'status' => 'pending' + 'status' => 'failed' ], [ 'url' => 'http://example.org/test4', - 'status' => 'pending' + 'status' => 'in-progress' ], [ 'url' => 'http://example.org/test5', - 'status' => 'pending' + 'status' => 'in-progress' ], ], 'actions' => [ @@ -181,14 +181,14 @@ ], ], [ - 'exists' => false, + 'exists' => true, 'args' => [ 'hook' => 'rocket_preload_job_preload_url', 'args' => ['http://example.org/test4'] ], ], [ - 'exists' => false, + 'exists' => true, 'args' => [ 'hook' => 'rocket_preload_job_preload_url', 'args' => ['http://example.org/test5'] From 37611072528dfa3f01a50c093d7097c5e4936bc6 Mon Sep 17 00:00:00 2001 From: Opeyemi Ibrahim Date: Thu, 15 Feb 2024 11:35:15 +0100 Subject: [PATCH 21/50] Add transients --- inc/Engine/WPRocketUninstall.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/inc/Engine/WPRocketUninstall.php b/inc/Engine/WPRocketUninstall.php index 77b60a3ef9..47f2e8653c 100644 --- a/inc/Engine/WPRocketUninstall.php +++ b/inc/Engine/WPRocketUninstall.php @@ -79,6 +79,8 @@ class WPRocketUninstall { 'wp_rocket_pricing_timeout', 'wp_rocket_pricing_timeout_active', 'rocket_get_refreshed_fragments_cache', + 'rocket_preload_previous_request_durations', + 'rocket_preload_check_duration' ]; /** From 25a43d081100fa5b18f70b82b8d0f9fb5880f95a Mon Sep 17 00:00:00 2001 From: Opeyemi Ibrahim Date: Thu, 15 Feb 2024 11:39:27 +0100 Subject: [PATCH 22/50] fixed lint error --- inc/Engine/WPRocketUninstall.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inc/Engine/WPRocketUninstall.php b/inc/Engine/WPRocketUninstall.php index 47f2e8653c..c51b782c7a 100644 --- a/inc/Engine/WPRocketUninstall.php +++ b/inc/Engine/WPRocketUninstall.php @@ -80,7 +80,7 @@ class WPRocketUninstall { 'wp_rocket_pricing_timeout_active', 'rocket_get_refreshed_fragments_cache', 'rocket_preload_previous_request_durations', - 'rocket_preload_check_duration' + 'rocket_preload_check_duration', ]; /** From 3c706035932f389170fb0cb9575631d98a3d3c79 Mon Sep 17 00:00:00 2001 From: Opeyemi Ibrahim Date: Wed, 28 Feb 2024 11:09:04 +0100 Subject: [PATCH 23/50] Review modifications, add comment to filter --- inc/Engine/Preload/Controller/PreloadUrl.php | 30 +++++++++++++------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/inc/Engine/Preload/Controller/PreloadUrl.php b/inc/Engine/Preload/Controller/PreloadUrl.php index 2b04f628be..4dec759c8b 100755 --- a/inc/Engine/Preload/Controller/PreloadUrl.php +++ b/inc/Engine/Preload/Controller/PreloadUrl.php @@ -211,18 +211,28 @@ public function process_pending_jobs() { $pending_actions = $this->queue->get_pending_preload_actions(); - // Retrieve batch size limits and request timing estimaiton. - $max_batch_size = ( (int) apply_filters( 'rocket_preload_cache_pending_jobs_cron_rows_count', 45 ) ) - count( $pending_actions ); - $min_batch_size = ( (int) apply_filters( 'rocket_preload_cache_min_in_progress_jobs_count', 5 ) ); + // Retrieve batch size limits and request timing estimation. + /** + * Get the number of pending cron job + * + * @param int $args Maximum number of job size. + */ + $max_batch_size = ( (int) apply_filters( 'rocket_preload_cache_pending_jobs_cron_rows_count', 45 ) ) - count( $pending_actions ); + + /** + * Get the number of in progress cron job + * + * @param int $args Minimum number of job size. + */ + $min_batch_size = ( (int) apply_filters( 'rocket_preload_cache_min_in_progress_jobs_count', 5 ) ); + $preload_request_duration = get_transient( 'rocket_preload_previous_request_durations' ); - // Estimate batch size based on request duration. - if ( ! $preload_request_duration ) { - $next_batch_size = $min_batch_size; // In case no estimation or there is an issue with the value. - } else { - // Linear function: 2s -> 45 jobs // 10s -> 5 jobs. - $next_batch_size = round( -5 * $preload_request_duration + 55 ); - } + /** + * Estimate batch size based on request duration. + * In case no estimation or there is an issue with the value use $min_batch_size. + */ + $next_batch_size = ! $preload_request_duration ? $min_batch_size : round( -5 * $preload_request_duration + 55 ); // Limit next_batch_size. $next_batch_size = min( $next_batch_size, $max_batch_size ); // Not higher than 45. From 18f4178f30b9adbf1cbc4e7a9796cb482baed001 Mon Sep 17 00:00:00 2001 From: Opeyemi Ibrahim Date: Thu, 29 Feb 2024 08:28:43 +0100 Subject: [PATCH 24/50] code modification :sparkles: --- inc/Engine/Preload/Controller/PreloadUrl.php | 35 +++++++++++--------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/inc/Engine/Preload/Controller/PreloadUrl.php b/inc/Engine/Preload/Controller/PreloadUrl.php index 4dec759c8b..ca2a0b8184 100755 --- a/inc/Engine/Preload/Controller/PreloadUrl.php +++ b/inc/Engine/Preload/Controller/PreloadUrl.php @@ -153,21 +153,6 @@ public function preload_url( string $url ) { $headers ); - if ( $check_duration ) { - $duration = ( microtime( true ) - $start ); // Duration of the request. - - // Update average duration. - $previous_request_durations = get_transient( 'rocket_preload_previous_request_durations' ) ?? 0; - if ( $previous_request_durations <= 0 ) { - $previous_request_durations = $duration; - } else { - $previous_request_durations = $previous_request_durations * 0.7 + $duration * 0.3; - } - set_transient( 'rocket_preload_previous_request_durations', $previous_request_durations, 5 * 60 ); - - set_transient( 'rocket_preload_check_duration', $duration, 60 ); // Don't check request duration for 1 minute. - $check_duration = false; - } /** * Filter the delay between each preload request. * @@ -176,6 +161,26 @@ public function preload_url( string $url ) { */ $delay_between = apply_filters( 'rocket_preload_delay_between_requests', 500000 ); + if ( ! $check_duration ) { + usleep( $delay_between ); + + return; + } + + $duration = ( microtime( true ) - $start ); // Duration of the request. + + // Update average duration. + $previous_request_durations = get_transient( 'rocket_preload_previous_request_durations' ) ?? 0; + if ( $previous_request_durations <= 0 ) { + $previous_request_durations = $duration; + } else { + $previous_request_durations = $previous_request_durations * 0.7 + $duration * 0.3; + } + set_transient( 'rocket_preload_previous_request_durations', $previous_request_durations, 5 * 60 ); + + set_transient( 'rocket_preload_check_duration', $duration, 60 ); // Don't check request duration for 1 minute. + $check_duration = false; + usleep( $delay_between ); } } From c4cca9c4ac1b156fd4f451bffc8afaa252fbb57f Mon Sep 17 00:00:00 2001 From: Opeyemi Ibrahim Date: Thu, 29 Feb 2024 08:46:25 +0100 Subject: [PATCH 25/50] Change package version and modify preload url loop --- composer.json | 2 +- inc/Engine/Preload/Controller/PreloadUrl.php | 20 ++++++++------------ 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/composer.json b/composer.json index 96cd62bc59..32d58edacb 100644 --- a/composer.json +++ b/composer.json @@ -68,7 +68,7 @@ "wpackagist-plugin/amp": "^1.1.4", "wpackagist-plugin/hummingbird-performance": "2.0.1", "wpackagist-plugin/jetpack": "9.3.2", - "wpackagist-plugin/pdf-embedder": "^4.6", + "wpackagist-plugin/pdf-embedder": "4.6.*", "wpackagist-plugin/simple-custom-css": "^4.0.3", "wpackagist-plugin/spinupwp": "^1.1", "wpackagist-plugin/woocommerce": "^8", diff --git a/inc/Engine/Preload/Controller/PreloadUrl.php b/inc/Engine/Preload/Controller/PreloadUrl.php index ca2a0b8184..b541d0c22e 100755 --- a/inc/Engine/Preload/Controller/PreloadUrl.php +++ b/inc/Engine/Preload/Controller/PreloadUrl.php @@ -153,6 +153,10 @@ public function preload_url( string $url ) { $headers ); + if ( $check_duration ) { + $duration = ( microtime( true ) - $start ); // Duration of the request. + } + /** * Filter the delay between each preload request. * @@ -161,27 +165,19 @@ public function preload_url( string $url ) { */ $delay_between = apply_filters( 'rocket_preload_delay_between_requests', 500000 ); + usleep( $delay_between ); if ( ! $check_duration ) { - usleep( $delay_between ); - - return; + continue; } - $duration = ( microtime( true ) - $start ); // Duration of the request. - // Update average duration. $previous_request_durations = get_transient( 'rocket_preload_previous_request_durations' ) ?? 0; - if ( $previous_request_durations <= 0 ) { - $previous_request_durations = $duration; - } else { - $previous_request_durations = $previous_request_durations * 0.7 + $duration * 0.3; - } + $previous_request_durations = ( $previous_request_durations <= 0 ) ? $duration : ( $previous_request_durations * 0.7 + $duration * 0.3 ); + set_transient( 'rocket_preload_previous_request_durations', $previous_request_durations, 5 * 60 ); set_transient( 'rocket_preload_check_duration', $duration, 60 ); // Don't check request duration for 1 minute. $check_duration = false; - - usleep( $delay_between ); } } From a1662ff29cc4f482694f90b7de758a0f1198af0c Mon Sep 17 00:00:00 2001 From: Mathieu Lamiot Date: Mon, 11 Mar 2024 11:28:54 +0100 Subject: [PATCH 26/50] Make the max size of the preload batch the top priority constraint to avoid filling the AS queue at all cost. --- inc/Engine/Preload/Controller/PreloadUrl.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inc/Engine/Preload/Controller/PreloadUrl.php b/inc/Engine/Preload/Controller/PreloadUrl.php index b541d0c22e..0bf0483f73 100755 --- a/inc/Engine/Preload/Controller/PreloadUrl.php +++ b/inc/Engine/Preload/Controller/PreloadUrl.php @@ -236,8 +236,8 @@ public function process_pending_jobs() { $next_batch_size = ! $preload_request_duration ? $min_batch_size : round( -5 * $preload_request_duration + 55 ); // Limit next_batch_size. - $next_batch_size = min( $next_batch_size, $max_batch_size ); // Not higher than 45. $next_batch_size = max( $next_batch_size, $min_batch_size ); // Not lower than 5. + $next_batch_size = min( $next_batch_size, $max_batch_size ); // Not higher than 45. // Get all in-progress jobs with request sent and no results. /** From 8951cc3113a164be0705a700dd5a22436d476a18 Mon Sep 17 00:00:00 2001 From: Mathieu Lamiot Date: Mon, 11 Mar 2024 12:16:09 +0100 Subject: [PATCH 27/50] Fix tests --- inc/Engine/Preload/Controller/PreloadUrl.php | 1 + .../Engine/Preload/Cron/Subscriber/processPendingUrls.php | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/inc/Engine/Preload/Controller/PreloadUrl.php b/inc/Engine/Preload/Controller/PreloadUrl.php index 0bf0483f73..f0d088b027 100755 --- a/inc/Engine/Preload/Controller/PreloadUrl.php +++ b/inc/Engine/Preload/Controller/PreloadUrl.php @@ -238,6 +238,7 @@ public function process_pending_jobs() { // Limit next_batch_size. $next_batch_size = max( $next_batch_size, $min_batch_size ); // Not lower than 5. $next_batch_size = min( $next_batch_size, $max_batch_size ); // Not higher than 45. + $next_batch_size = max( $next_batch_size, 0 ); // Not lower than 0. // Get all in-progress jobs with request sent and no results. /** diff --git a/tests/Fixtures/inc/Engine/Preload/Cron/Subscriber/processPendingUrls.php b/tests/Fixtures/inc/Engine/Preload/Cron/Subscriber/processPendingUrls.php index 7c1a0c56a6..f65344c68f 100644 --- a/tests/Fixtures/inc/Engine/Preload/Cron/Subscriber/processPendingUrls.php +++ b/tests/Fixtures/inc/Engine/Preload/Cron/Subscriber/processPendingUrls.php @@ -120,7 +120,7 @@ ], [ 'url' => 'http://example.org/test3', - 'status' => 'failed' + 'status' => 'in-progress' ], [ 'url' => 'http://example.org/test4', @@ -147,7 +147,7 @@ ], [ 'url' => 'http://example.org/test3', - 'status' => 'failed' + 'status' => 'in-progress' ], [ 'url' => 'http://example.org/test4', @@ -155,7 +155,7 @@ ], [ 'url' => 'http://example.org/test5', - 'status' => 'in-progress' + 'status' => 'pending' ], ], 'actions' => [ @@ -188,7 +188,7 @@ ], ], [ - 'exists' => true, + 'exists' => false, 'args' => [ 'hook' => 'rocket_preload_job_preload_url', 'args' => ['http://example.org/test5'] From 3a9a2add5da04dba80307cfe1a5b0cd247e56739 Mon Sep 17 00:00:00 2001 From: Mathieu Lamiot Date: Mon, 11 Mar 2024 12:27:11 +0100 Subject: [PATCH 28/50] InProgressShouldNotExceedMaxQueue data set needs update --- .../inc/Engine/Preload/Cron/Subscriber/processPendingUrls.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Fixtures/inc/Engine/Preload/Cron/Subscriber/processPendingUrls.php b/tests/Fixtures/inc/Engine/Preload/Cron/Subscriber/processPendingUrls.php index f65344c68f..d364f77779 100644 --- a/tests/Fixtures/inc/Engine/Preload/Cron/Subscriber/processPendingUrls.php +++ b/tests/Fixtures/inc/Engine/Preload/Cron/Subscriber/processPendingUrls.php @@ -106,7 +106,7 @@ ], 'InProgressShouldNotExceedMaxQueue' => [ 'config' => [ - 'rocket_preload_cache_pending_jobs_cron_rows_count' => 3, + 'rocket_preload_cache_pending_jobs_cron_rows_count' => 5, 'manual_preload' => true, 'rocket_preload_outdated' => 1000, 'rows' => [ From cd57efa1373fa4b3768a5a39548706da4abe4a80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Perona?= Date: Wed, 12 Jun 2024 14:33:24 -0400 Subject: [PATCH 29/50] update test config --- inc/Engine/Preload/Controller/PreloadUrl.php | 4 +- .../Controller/PreloadUrl/preloadUrl.php | 3 + .../Controller/PreloadUrl/preloadUrl.php | 80 +++++++++++-------- 3 files changed, 53 insertions(+), 34 deletions(-) diff --git a/inc/Engine/Preload/Controller/PreloadUrl.php b/inc/Engine/Preload/Controller/PreloadUrl.php index 845d522f11..32ab830a5e 100755 --- a/inc/Engine/Preload/Controller/PreloadUrl.php +++ b/inc/Engine/Preload/Controller/PreloadUrl.php @@ -1,4 +1,5 @@ options->get( 'do_caching_mobile_files', false ); + if ( $this->is_already_cached( $url ) && ( ! $is_mobile || $this->is_already_cached( $url, true ) ) ) { $this->query->make_status_complete( $url ); return; diff --git a/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php b/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php index c4e7175dc9..ff0d78c83e 100644 --- a/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php +++ b/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php @@ -2,6 +2,7 @@ return [ 'alreadyPresentShouldDoNothing' => [ 'config' => [ + 'transient_check_duration' => false, 'url' => 'url', 'cache_exists' => true, 'cache_mobile' => false, @@ -19,6 +20,7 @@ ], 'mobileNotActivatedShouldPreloadOnlyOnce' => [ 'config' => [ + 'transient_check_duration' => false, 'url' => 'url', 'cache_exists' => false, 'cache_mobile' => false, @@ -36,6 +38,7 @@ ], 'mobileActivatedShouldPreloadTwice' => [ 'config' => [ + 'transient_check_duration' => false, 'url' => 'url', 'cache_exists' => false, 'cache_mobile' => true, diff --git a/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php b/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php index ffe512492e..584baaf6e1 100644 --- a/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php +++ b/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php @@ -1,75 +1,89 @@ options = Mockery::mock(Options_Data::class); - $this->query = $this->createMock(Cache::class); - $this->queue = Mockery::mock(Queue::class); - $this->file_system = Mockery::mock(WP_Filesystem_Direct::class); - $this->controller = Mockery::mock(PreloadUrl::class . '[get_mobile_user_agent_prefix,is_already_cached]', - [$this->options, - $this->queue, $this->query, $this->file_system])->shouldAllowMockingProtectedMethods(); + + $this->options = Mockery::mock( Options_Data::class ); + $this->query = $this->createMock( Cache::class ); + $this->queue = Mockery::mock( Queue::class ); + $this->file_system = Mockery::mock( WP_Filesystem_Direct::class ); + $this->controller = Mockery::mock( PreloadUrl::class . '[get_mobile_user_agent_prefix,is_already_cached]', + [ + $this->options, + $this->queue, + $this->query, + $this->file_system, + ] + )->shouldAllowMockingProtectedMethods(); } /** * @dataProvider configTestData */ - public function testShouldDoAsExpected($config) { - Functions\when( 'get_transient' ) - ->justReturn(); + public function testShouldDoAsExpected( $config ) { + Functions\expects( 'get_transient' ) + ->once() + ->with( 'rocket_preload_check_duration' ) + ->andReturn( $config['transient_check_duration'] ); + Functions\when( 'set_transient' ) ->justReturn(); - $this->options->expects()->get('do_caching_mobile_files', false)->andReturn($config['cache_mobile']); - $this->controller->expects()->is_already_cached($config['url'])->andReturn($config['cache_exists']); - $this->configureRequest($config); - $this->configureMobileRequest($config); - $this->controller->preload_url($config['url']); + $this->options->expects()->get( 'do_caching_mobile_files', false ) + ->andReturn( $config['cache_mobile'] ); + $this->controller->expects()->is_already_cached( $config['url'] ) + ->andReturn( $config['cache_exists'] ); + $this->configureRequest( $config ); + $this->configureMobileRequest( $config ); + + $this->controller->preload_url( $config['url'] ); } - protected function configureRequest($config) { - if($config['cache_exists']) { + protected function configureRequest( $config ) { + if ( $config['cache_exists'] ) { return; } - Functions\expect('wp_safe_remote_get')->with($config['url'] . '/', $config['request']['config']); + Functions\expect( 'wp_safe_remote_get' ) + ->with( $config['url'] . '/', $config['request']['config'] ); } - protected function configureMobileRequest($config) { - if($config['cache_exists']) { + protected function configureMobileRequest( $config ) { + if ( $config['cache_exists'] ) { return; } - if(! $config['cache_mobile']) { + if ( ! $config['cache_mobile'] ) { return; } - $this->controller->expects()->get_mobile_user_agent_prefix()->andReturn($config['user_agent']); - Functions\expect('wp_safe_remote_get')->with($config['url'] . '/', $config['request_mobile']['config']); + + $this->controller->expects()->get_mobile_user_agent_prefix() + ->andReturn( $config['user_agent'] ); + + Functions\expect( 'wp_safe_remote_get' ) + ->with( $config['url'] . '/', $config['request_mobile']['config'] ); } } From ba634ac9f16761b237a123a8dbf819efc638fccb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Perona?= Date: Wed, 12 Jun 2024 14:41:22 -0400 Subject: [PATCH 30/50] fix function name --- .../inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php b/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php index 584baaf6e1..2264e991df 100644 --- a/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php +++ b/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php @@ -44,7 +44,7 @@ protected function setUp(): void { * @dataProvider configTestData */ public function testShouldDoAsExpected( $config ) { - Functions\expects( 'get_transient' ) + Functions\expect( 'get_transient' ) ->once() ->with( 'rocket_preload_check_duration' ) ->andReturn( $config['transient_check_duration'] ); From 8e8e205e4c37ae70db0509dc593995b0474ceb7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Perona?= Date: Wed, 12 Jun 2024 14:43:45 -0400 Subject: [PATCH 31/50] update count expectation --- .../Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php b/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php index 2264e991df..21680686c2 100644 --- a/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php +++ b/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php @@ -45,6 +45,7 @@ protected function setUp(): void { */ public function testShouldDoAsExpected( $config ) { Functions\expect( 'get_transient' ) + ->atMost() ->once() ->with( 'rocket_preload_check_duration' ) ->andReturn( $config['transient_check_duration'] ); From d6beb2ff6af9aba2d61ed7e1a99ee8286b335ffd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Perona?= Date: Wed, 12 Jun 2024 14:58:03 -0400 Subject: [PATCH 32/50] force int type --- inc/Engine/Preload/Controller/PreloadUrl.php | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/inc/Engine/Preload/Controller/PreloadUrl.php b/inc/Engine/Preload/Controller/PreloadUrl.php index 32ab830a5e..c2ece4929e 100755 --- a/inc/Engine/Preload/Controller/PreloadUrl.php +++ b/inc/Engine/Preload/Controller/PreloadUrl.php @@ -206,7 +206,6 @@ protected function get_mobile_user_agent_prefix() { * @return void */ public function process_pending_jobs() { - $pending_actions = $this->queue->get_pending_preload_actions(); // Retrieve batch size limits and request timing estimation. @@ -271,7 +270,7 @@ function ( $row ) use ( $pending_actions ) { // Make those hanging jobs failed. foreach ( $stuck_rows as $row ) { - $this->query->make_status_failed( $row->id ); + $this->query->make_status_failed( (int) $row->id ); } // Add new jobs in progress. @@ -283,7 +282,7 @@ function ( $row ) use ( $pending_actions ) { continue; } - $this->query->make_status_inprogress( $row->id ); + $this->query->make_status_inprogress( (int) $row->id ); $this->queue->add_job_preload_job_preload_url_async( $row->url ); } From daba3612e8a954ec21c3e5fdf6ad680e96fceabc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Perona?= Date: Wed, 12 Jun 2024 15:15:19 -0400 Subject: [PATCH 33/50] cleanup --- inc/Engine/Preload/Controller/PreloadUrl.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inc/Engine/Preload/Controller/PreloadUrl.php b/inc/Engine/Preload/Controller/PreloadUrl.php index c2ece4929e..cc3852ac43 100755 --- a/inc/Engine/Preload/Controller/PreloadUrl.php +++ b/inc/Engine/Preload/Controller/PreloadUrl.php @@ -158,11 +158,11 @@ public function preload_url( string $url ) { * Filter the delay between each preload request. * * @param float $delay_between the defined delay. - * @returns float */ $delay_between = apply_filters( 'rocket_preload_delay_between_requests', 500000 ); usleep( $delay_between ); + if ( ! $check_duration ) { continue; } From 16f4dbb75b68b604dd3f79ba8d4cf03543550dee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Perona?= Date: Wed, 12 Jun 2024 15:15:24 -0400 Subject: [PATCH 34/50] update expectations --- .../Controller/PreloadUrl/preloadUrl.php | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php b/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php index 21680686c2..6272291960 100644 --- a/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php +++ b/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php @@ -57,14 +57,19 @@ public function testShouldDoAsExpected( $config ) { ->andReturn( $config['cache_mobile'] ); $this->controller->expects()->is_already_cached( $config['url'] ) ->andReturn( $config['cache_exists'] ); - $this->configureRequest( $config ); - $this->configureMobileRequest( $config ); + + $this->expectDesktopRequest( $config ); + $this->expectMobileRequest( $config ); $this->controller->preload_url( $config['url'] ); } - protected function configureRequest( $config ) { + protected function expectDesktopRequest( $config ) { if ( $config['cache_exists'] ) { + Functions\expect( 'wp_safe_remote_get' ) + ->with( $config['url'] . '/', $config['request']['config'] ) + ->never(); + return; } @@ -72,12 +77,16 @@ protected function configureRequest( $config ) { ->with( $config['url'] . '/', $config['request']['config'] ); } - protected function configureMobileRequest( $config ) { - if ( $config['cache_exists'] ) { - return; - } + protected function expectMobileRequest( $config ) { + if ( + $config['cache_exists'] + || + ! $config['cache_mobile'] + ) { + Functions\expect( 'wp_safe_remote_get' ) + ->with( $config['url'] . '/', $config['request_mobile']['config'] ) + ->never(); - if ( ! $config['cache_mobile'] ) { return; } From 1c7b9c2250366f4c40d5706c989b487fcae522fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Perona?= Date: Wed, 12 Jun 2024 15:31:44 -0400 Subject: [PATCH 35/50] update test config --- .../Engine/Preload/Controller/PreloadUrl/preloadUrl.php | 9 +++------ .../Engine/Preload/Controller/PreloadUrl/preloadUrl.php | 4 ---- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php b/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php index ff0d78c83e..160b85b436 100644 --- a/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php +++ b/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php @@ -1,6 +1,6 @@ [ + 'shouldDoNothingWhenAlreadyCached' => [ 'config' => [ 'transient_check_duration' => false, 'url' => 'url', @@ -15,10 +15,9 @@ 'sslverify' => false, ] ], - 'transient' => 0, ] ], - 'mobileNotActivatedShouldPreloadOnlyOnce' => [ + 'shouldPreloadOnlyOnceWhenMobileCacheDisabled' => [ 'config' => [ 'transient_check_duration' => false, 'url' => 'url', @@ -33,10 +32,9 @@ 'sslverify' => false, ] ], - 'transient' => 1, ], ], - 'mobileActivatedShouldPreloadTwice' => [ + 'ShouldPreloadTwiceWhenMobileCacheEnabled' => [ 'config' => [ 'transient_check_duration' => false, 'url' => 'url', @@ -59,7 +57,6 @@ 'sslverify' => false, ] ], - 'transient' => 0, ] ] ]; diff --git a/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php b/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php index 6272291960..3a5a0c8140 100644 --- a/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php +++ b/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php @@ -83,10 +83,6 @@ protected function expectMobileRequest( $config ) { || ! $config['cache_mobile'] ) { - Functions\expect( 'wp_safe_remote_get' ) - ->with( $config['url'] . '/', $config['request_mobile']['config'] ) - ->never(); - return; } From 70f0f7119696ec5937b980829dfc1349a9c44029 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Perona?= Date: Wed, 12 Jun 2024 16:03:38 -0400 Subject: [PATCH 36/50] update condition --- inc/Engine/Preload/Controller/PreloadUrl.php | 6 ++-- .../Controller/PreloadUrl/preloadUrl.php | 28 +++++++++---------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/inc/Engine/Preload/Controller/PreloadUrl.php b/inc/Engine/Preload/Controller/PreloadUrl.php index cc3852ac43..a96f13c83e 100755 --- a/inc/Engine/Preload/Controller/PreloadUrl.php +++ b/inc/Engine/Preload/Controller/PreloadUrl.php @@ -69,7 +69,7 @@ public function preload_url( string $url ) { } // Should we perform a duration check? - $check_duration = get_transient( 'rocket_preload_check_duration' ) ? false : true; + $check_duration = ( false === get_transient( 'rocket_preload_check_duration' ) ) ? true : false; $requests = [ [ @@ -171,9 +171,9 @@ public function preload_url( string $url ) { $previous_request_durations = get_transient( 'rocket_preload_previous_request_durations' ) ?? 0; $previous_request_durations = ( $previous_request_durations <= 0 ) ? $duration : ( $previous_request_durations * 0.7 + $duration * 0.3 ); - set_transient( 'rocket_preload_previous_request_durations', $previous_request_durations, 5 * 60 ); + set_transient( 'rocket_preload_previous_request_durations', $previous_request_durations, 5 * MINUTE_IN_SECONDS ); - set_transient( 'rocket_preload_check_duration', $duration, 60 ); // Don't check request duration for 1 minute. + set_transient( 'rocket_preload_check_duration', $duration, MINUTE_IN_SECONDS ); // Don't check request duration for 1 minute. $check_duration = false; } } diff --git a/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php b/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php index 160b85b436..d9e33f9161 100644 --- a/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php +++ b/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php @@ -2,7 +2,7 @@ return [ 'shouldDoNothingWhenAlreadyCached' => [ 'config' => [ - 'transient_check_duration' => false, + 'transient_check_duration' => 500, 'url' => 'url', 'cache_exists' => true, 'cache_mobile' => false, @@ -13,41 +13,41 @@ 'timeout' => 0.01, 'user-agent' => 'WP Rocket/Preload', 'sslverify' => false, - ] + ], ], - ] + ], ], 'shouldPreloadOnlyOnceWhenMobileCacheDisabled' => [ 'config' => [ - 'transient_check_duration' => false, + 'transient_check_duration' => 500, 'url' => 'url', 'cache_exists' => false, 'cache_mobile' => false, 'user_agent' => 'user_agent', 'request' => [ 'config' => [ - 'blocking' => true, - 'timeout' => 20, + 'blocking' => false, + 'timeout' => 0.01, 'user-agent' => 'WP Rocket/Preload', 'sslverify' => false, - ] + ], ], ], ], 'ShouldPreloadTwiceWhenMobileCacheEnabled' => [ 'config' => [ - 'transient_check_duration' => false, + 'transient_check_duration' => 500, 'url' => 'url', 'cache_exists' => false, 'cache_mobile' => true, 'user_agent' => 'user_agent', 'request' => [ 'config' => [ - 'blocking' => true, - 'timeout' => 20, + 'blocking' => false, + 'timeout' => 0.01, 'user-agent' => 'WP Rocket/Preload', 'sslverify' => false, - ] + ], ], 'request_mobile' => [ 'config' => [ @@ -55,8 +55,8 @@ 'timeout' => 0.01, 'user-agent' => 'user_agent', 'sslverify' => false, - ] + ], ], - ] - ] + ], + ], ]; From 1b99daac80c391c4e26f2545133b3db4a64f1153 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Perona?= Date: Wed, 12 Jun 2024 16:16:09 -0400 Subject: [PATCH 37/50] add test for check duration --- .../Controller/PreloadUrl/preloadUrl.php | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php b/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php index d9e33f9161..ec5904278a 100644 --- a/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php +++ b/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php @@ -59,4 +59,21 @@ ], ], ], + 'shouldPreloadOnlyOnceWhenMobileCacheDisabledAndCheckDuration' => [ + 'config' => [ + 'transient_check_duration' => false, + 'url' => 'url', + 'cache_exists' => false, + 'cache_mobile' => false, + 'user_agent' => 'user_agent', + 'request' => [ + 'config' => [ + 'blocking' => true, + 'timeout' => 20, + 'user-agent' => 'WP Rocket/Preload', + 'sslverify' => false, + ], + ], + ], + ], ]; From 2483a4638dc7c10f02624ec2b83a3b9aee9b9c5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Perona?= Date: Thu, 13 Jun 2024 12:10:56 -0400 Subject: [PATCH 38/50] update transient name and code --- inc/Engine/Preload/Controller/PreloadUrl.php | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/inc/Engine/Preload/Controller/PreloadUrl.php b/inc/Engine/Preload/Controller/PreloadUrl.php index a96f13c83e..a0013bd906 100755 --- a/inc/Engine/Preload/Controller/PreloadUrl.php +++ b/inc/Engine/Preload/Controller/PreloadUrl.php @@ -167,11 +167,13 @@ public function preload_url( string $url ) { continue; } + $duration_transient = get_transient( 'rocket_preload_previous_requests_durations' ); + $average_duration = ( false !== $duration_transient ) ? $duration_transient : 0; + // Update average duration. - $previous_request_durations = get_transient( 'rocket_preload_previous_request_durations' ) ?? 0; - $previous_request_durations = ( $previous_request_durations <= 0 ) ? $duration : ( $previous_request_durations * 0.7 + $duration * 0.3 ); + $average_duration = ( $average_duration <= 0 ) ? $duration : ( $average_duration * 0.7 + $duration * 0.3 ); - set_transient( 'rocket_preload_previous_request_durations', $previous_request_durations, 5 * MINUTE_IN_SECONDS ); + set_transient( 'rocket_preload_previous_requests_durations', $average_duration, 5 * MINUTE_IN_SECONDS ); set_transient( 'rocket_preload_check_duration', $duration, MINUTE_IN_SECONDS ); // Don't check request duration for 1 minute. $check_duration = false; @@ -223,7 +225,7 @@ public function process_pending_jobs() { */ $min_batch_size = ( (int) apply_filters( 'rocket_preload_cache_min_in_progress_jobs_count', 5 ) ); - $preload_request_duration = get_transient( 'rocket_preload_previous_request_durations' ); + $preload_request_duration = get_transient( 'rocket_preload_previous_requests_durations' ); /** * Estimate batch size based on request duration. From 9d68828df853788d658dd8a4b7873bd131c65380 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Perona?= Date: Thu, 13 Jun 2024 12:11:05 -0400 Subject: [PATCH 39/50] update transient name --- inc/Engine/WPRocketUninstall.php | 2 +- tests/Integration/inc/Engine/WPRocketUninstall/uninstall.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/inc/Engine/WPRocketUninstall.php b/inc/Engine/WPRocketUninstall.php index b2f4f9c606..d2999c6204 100644 --- a/inc/Engine/WPRocketUninstall.php +++ b/inc/Engine/WPRocketUninstall.php @@ -80,7 +80,7 @@ class WPRocketUninstall { 'wp_rocket_pricing_timeout', 'wp_rocket_pricing_timeout_active', 'rocket_get_refreshed_fragments_cache', - 'rocket_preload_previous_request_durations', + 'rocket_preload_previous_requests_durations', 'rocket_preload_check_duration', ]; diff --git a/tests/Integration/inc/Engine/WPRocketUninstall/uninstall.php b/tests/Integration/inc/Engine/WPRocketUninstall/uninstall.php index 2759dc0a2e..f237f5c252 100644 --- a/tests/Integration/inc/Engine/WPRocketUninstall/uninstall.php +++ b/tests/Integration/inc/Engine/WPRocketUninstall/uninstall.php @@ -45,7 +45,7 @@ class Test_Uninstall extends FilesystemTestCase { 'rocket_preload_complete' => null, 'rocket_preload_complete_time' => null, 'rocket_preload_errors' => null, - 'rocket_preload_previous_request_durations' => null, + 'rocket_preload_previous_requests_durations' => null, 'rocket_preload_check_duration' => null, 'rocket_database_optimization_process' => null, 'rocket_database_optimization_process_complete' => null, From f8064820d727f7de1e500add1c06e760d532294a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Perona?= Date: Thu, 13 Jun 2024 12:11:15 -0400 Subject: [PATCH 40/50] improve expectations for transients --- .../Controller/PreloadUrl/preloadUrl.php | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php b/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php index 3a5a0c8140..ae26f31e89 100644 --- a/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php +++ b/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php @@ -50,9 +50,6 @@ public function testShouldDoAsExpected( $config ) { ->with( 'rocket_preload_check_duration' ) ->andReturn( $config['transient_check_duration'] ); - Functions\when( 'set_transient' ) - ->justReturn(); - $this->options->expects()->get( 'do_caching_mobile_files', false ) ->andReturn( $config['cache_mobile'] ); $this->controller->expects()->is_already_cached( $config['url'] ) @@ -61,6 +58,24 @@ public function testShouldDoAsExpected( $config ) { $this->expectDesktopRequest( $config ); $this->expectMobileRequest( $config ); + if ( false === $config['transient_check_duration'] ) { + Functions\expect( 'get_transient' ) + ->atMost() + ->once() + ->with( 'rocket_preload_previous_requests_durations' ) + ->andReturn( false ); + + Functions\expect( 'set_transient' ) + ->atMost() + ->once() + ->with( 'rocket_preload_previous_requests_durations', Mockery::type( 'int' ), 300 ); + + Functions\expect( 'set_transient' ) + ->atMost() + ->once() + ->with( 'rocket_preload_check_duration', Mockery::type( 'int' ), 60 ); + } + $this->controller->preload_url( $config['url'] ); } From b1ab24b91651c3f93913eac94f50731bf85e55de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Perona?= Date: Thu, 13 Jun 2024 15:45:03 -0400 Subject: [PATCH 41/50] rename variable --- inc/Engine/Preload/Controller/PreloadUrl.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inc/Engine/Preload/Controller/PreloadUrl.php b/inc/Engine/Preload/Controller/PreloadUrl.php index a0013bd906..6b3adbd0ae 100755 --- a/inc/Engine/Preload/Controller/PreloadUrl.php +++ b/inc/Engine/Preload/Controller/PreloadUrl.php @@ -225,13 +225,13 @@ public function process_pending_jobs() { */ $min_batch_size = ( (int) apply_filters( 'rocket_preload_cache_min_in_progress_jobs_count', 5 ) ); - $preload_request_duration = get_transient( 'rocket_preload_previous_requests_durations' ); + $average_duration = get_transient( 'rocket_preload_previous_requests_durations' ); /** * Estimate batch size based on request duration. * In case no estimation or there is an issue with the value use $min_batch_size. */ - $next_batch_size = ! $preload_request_duration ? $min_batch_size : round( -5 * $preload_request_duration + 55 ); + $next_batch_size = ! $average_duration ? $min_batch_size : round( -5 * $average_duration + 55 ); // Limit next_batch_size. $next_batch_size = max( $next_batch_size, $min_batch_size ); // Not lower than 5. From d00b8ee1e935cf3ab61c733e9784aa8b7861149f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Perona?= Date: Thu, 13 Jun 2024 16:41:26 -0400 Subject: [PATCH 42/50] code cleanup --- .../PreloadUrl/processPendingJobs.php | 20 ++--- .../PreloadUrl/processPendingJobs.php | 73 ++++++++++++------- 2 files changed, 57 insertions(+), 36 deletions(-) diff --git a/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/processPendingJobs.php b/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/processPendingJobs.php index 4d08396aed..011a079288 100644 --- a/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/processPendingJobs.php +++ b/tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/processPendingJobs.php @@ -41,35 +41,35 @@ 'shouldPassJobsInPending' => [ 'config' => [ 'outdated_jobs' => [ - $outdated_row + $outdated_row, ], 'excluded' => [ - false, - false, - true, + false, + false, + true, ], 'rows' => 5, 'jobs' => [ $row1, $row2, - $row3 + $row3, ], 'transient' => 0, ], 'expected' => [ 'outdated_jobs_id' => [ - [14] + [ 14 ], ], 'job_ids' => [ - [10], - [11], + [ 10 ], + [ 11 ], ], 'job_urls' => [ 'http://example1', 'http://example2', ], 'job_deleted' => - ['http://example3'], - ] + [ 'http://example3' ], + ], ], ]; diff --git a/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/processPendingJobs.php b/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/processPendingJobs.php index e168e3f48a..a2df1ebe50 100644 --- a/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/processPendingJobs.php +++ b/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/processPendingJobs.php @@ -1,56 +1,77 @@ options = Mockery::mock(Options_Data::class); - $this->query = $this->createMock(Cache::class); - $this->queue = Mockery::mock(Queue::class); - $this->file_system = Mockery::mock(WP_Filesystem_Direct::class); - $this->controller = Mockery::mock( PreloadUrl::class . '[is_excluded_by_filter]' , [$this->options, $this->queue, $this->query, $this->file_system])->shouldAllowMockingProtectedMethods(); + + $this->options = Mockery::mock( Options_Data::class ); + $this->query = $this->createMock( Cache::class ); + $this->queue = Mockery::mock( Queue::class ); + $this->file_system = Mockery::mock( WP_Filesystem_Direct::class ); + $this->controller = Mockery::mock( PreloadUrl::class . '[is_excluded_by_filter]', [ $this->options, $this->queue, $this->query, $this->file_system ] )->shouldAllowMockingProtectedMethods(); } /** * @dataProvider configTestData */ - public function testShouldDoAsExpected($config, $expected) { + public function testShouldDoAsExpected( $config, $expected ) { Functions\when( 'get_transient' ) ->justReturn( $config['transient'] ); - $this->queue->expects()->get_pending_preload_actions()->andReturn([]); - Filters\expectApplied('rocket_preload_cache_pending_jobs_cron_rows_count')->with(100)->andReturn($config['rows']); - $this->query->expects(self::once())->method( 'get_outdated_in_progress_jobs' )->with()->willReturn($config['outdated_jobs']); - $this->query->expects(self::atLeastOnce())->method('make_status_failed')->withConsecutive(...$expected['outdated_jobs_id']); - $this->query->expects(self::once())->method('get_pending_jobs')->with($config['rows'])->willReturn($config['jobs']); - $this->query->expects(self::atLeastOnce())->method('make_status_inprogress')->withConsecutive(...$expected['job_ids']); - $this->query->expects(self::atLeast(0))->method('delete_by_url')->withConsecutive($expected['job_deleted']); - $this->controller->shouldReceive('is_excluded_by_filter')->zeroOrMoreTimes()->andReturnValues($config['excluded']); - foreach ($expected['job_urls'] as $url) { - $this->queue->expects()->add_job_preload_job_preload_url_async( $url ); + + $this->queue->expects()->get_pending_preload_actions() + ->andReturn( [] ); + + Filters\expectApplied( 'rocket_preload_cache_pending_jobs_cron_rows_count' ) + ->with( 100 ) + ->andReturn( $config['rows'] ); + + $this->query->expects( self::once() ) + ->method( 'get_outdated_in_progress_jobs' ) + ->with() + ->willReturn( $config['outdated_jobs'] ); + $this->query->expects( self::atLeastOnce() ) + ->method( 'make_status_failed' ) + ->withConsecutive( ...$expected['outdated_jobs_id'] ); + $this->query->expects( self::once() ) + ->method( 'get_pending_jobs' ) + ->with( $config['rows'] ) + ->willReturn( $config['jobs'] ); + $this->query->expects( self::atLeastOnce() ) + ->method( 'make_status_inprogress' ) + ->withConsecutive( ...$expected['job_ids'] ); + $this->query->expects( self::atLeast( 0 ) ) + ->method( 'delete_by_url' ) + ->withConsecutive( $expected['job_deleted'] ); + + $this->controller->shouldReceive( 'is_excluded_by_filter' ) + ->zeroOrMoreTimes() + ->andReturnValues( $config['excluded'] ); + + foreach ( $expected['job_urls'] as $url ) { + $this->queue->expects() + ->add_job_preload_job_preload_url_async( $url ); } $this->controller->process_pending_jobs(); From 2f34f9d4fa84d2c5f30d43ee9d9d04f00f3cbd1f Mon Sep 17 00:00:00 2001 From: Mathieu Lamiot Date: Tue, 25 Jun 2024 10:05:56 +0200 Subject: [PATCH 43/50] add logs --- inc/Engine/Preload/Controller/PreloadUrl.php | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/inc/Engine/Preload/Controller/PreloadUrl.php b/inc/Engine/Preload/Controller/PreloadUrl.php index 6b3adbd0ae..079c9ec9be 100755 --- a/inc/Engine/Preload/Controller/PreloadUrl.php +++ b/inc/Engine/Preload/Controller/PreloadUrl.php @@ -5,10 +5,13 @@ use WP_Rocket\Admin\Options_Data; use WP_Rocket\Engine\Preload\Database\Queries\Cache; +use WP_Rocket\Logger\LoggerAware; +use WP_Rocket\Logger\LoggerAwareInterface; use WP_Filesystem_Direct; -class PreloadUrl { +class PreloadUrl implements LoggerAwareInterface { use CheckExcludedTrait; + use LoggerAware; /** * Preload queue. @@ -176,6 +179,12 @@ public function preload_url( string $url ) { set_transient( 'rocket_preload_previous_requests_durations', $average_duration, 5 * MINUTE_IN_SECONDS ); set_transient( 'rocket_preload_check_duration', $duration, MINUTE_IN_SECONDS ); // Don't check request duration for 1 minute. + $this->logger::debug( + "Preload request duration: {$duration}s. New average duration is {$average_duration}s.", + [ + 'method' => __METHOD__, + ] + ); $check_duration = false; } } @@ -238,6 +247,13 @@ public function process_pending_jobs() { $next_batch_size = min( $next_batch_size, $max_batch_size ); // Not higher than 45. $next_batch_size = max( $next_batch_size, 0 ); // Not lower than 0. + $this->logger::debug( + "Average request duration is {$average_duration}s. Next batch size will be {$next_batch_size}s", + [ + 'method' => __METHOD__, + ] + ); + // Get all in-progress jobs with request sent and no results. /** * Set the delay before an in-progress row is considered as outdated. From f016139bc395bfd5c133b29731be3540f20296f5 Mon Sep 17 00:00:00 2001 From: Mathieu Lamiot Date: Tue, 25 Jun 2024 10:25:21 +0200 Subject: [PATCH 44/50] Fix log typo --- inc/Engine/Preload/Controller/PreloadUrl.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inc/Engine/Preload/Controller/PreloadUrl.php b/inc/Engine/Preload/Controller/PreloadUrl.php index 079c9ec9be..facaccfea7 100755 --- a/inc/Engine/Preload/Controller/PreloadUrl.php +++ b/inc/Engine/Preload/Controller/PreloadUrl.php @@ -248,7 +248,7 @@ public function process_pending_jobs() { $next_batch_size = max( $next_batch_size, 0 ); // Not lower than 0. $this->logger::debug( - "Average request duration is {$average_duration}s. Next batch size will be {$next_batch_size}s", + "Average request duration is {$average_duration}s. Next batch size will be {$next_batch_size}.", [ 'method' => __METHOD__, ] From 48f03ee4de1eadbab4bf704a6874b5f8418adc10 Mon Sep 17 00:00:00 2001 From: Mathieu Lamiot Date: Tue, 25 Jun 2024 10:40:45 +0200 Subject: [PATCH 45/50] Revert "Fix log typo" This reverts commit f016139bc395bfd5c133b29731be3540f20296f5. --- inc/Engine/Preload/Controller/PreloadUrl.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inc/Engine/Preload/Controller/PreloadUrl.php b/inc/Engine/Preload/Controller/PreloadUrl.php index facaccfea7..079c9ec9be 100755 --- a/inc/Engine/Preload/Controller/PreloadUrl.php +++ b/inc/Engine/Preload/Controller/PreloadUrl.php @@ -248,7 +248,7 @@ public function process_pending_jobs() { $next_batch_size = max( $next_batch_size, 0 ); // Not lower than 0. $this->logger::debug( - "Average request duration is {$average_duration}s. Next batch size will be {$next_batch_size}.", + "Average request duration is {$average_duration}s. Next batch size will be {$next_batch_size}s", [ 'method' => __METHOD__, ] From eeb8c34df7f467957eb298fa7e7bbca705a623ef Mon Sep 17 00:00:00 2001 From: Mathieu Lamiot Date: Tue, 25 Jun 2024 10:40:49 +0200 Subject: [PATCH 46/50] Revert "add logs" This reverts commit 2f34f9d4fa84d2c5f30d43ee9d9d04f00f3cbd1f. --- inc/Engine/Preload/Controller/PreloadUrl.php | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/inc/Engine/Preload/Controller/PreloadUrl.php b/inc/Engine/Preload/Controller/PreloadUrl.php index 079c9ec9be..6b3adbd0ae 100755 --- a/inc/Engine/Preload/Controller/PreloadUrl.php +++ b/inc/Engine/Preload/Controller/PreloadUrl.php @@ -5,13 +5,10 @@ use WP_Rocket\Admin\Options_Data; use WP_Rocket\Engine\Preload\Database\Queries\Cache; -use WP_Rocket\Logger\LoggerAware; -use WP_Rocket\Logger\LoggerAwareInterface; use WP_Filesystem_Direct; -class PreloadUrl implements LoggerAwareInterface { +class PreloadUrl { use CheckExcludedTrait; - use LoggerAware; /** * Preload queue. @@ -179,12 +176,6 @@ public function preload_url( string $url ) { set_transient( 'rocket_preload_previous_requests_durations', $average_duration, 5 * MINUTE_IN_SECONDS ); set_transient( 'rocket_preload_check_duration', $duration, MINUTE_IN_SECONDS ); // Don't check request duration for 1 minute. - $this->logger::debug( - "Preload request duration: {$duration}s. New average duration is {$average_duration}s.", - [ - 'method' => __METHOD__, - ] - ); $check_duration = false; } } @@ -247,13 +238,6 @@ public function process_pending_jobs() { $next_batch_size = min( $next_batch_size, $max_batch_size ); // Not higher than 45. $next_batch_size = max( $next_batch_size, 0 ); // Not lower than 0. - $this->logger::debug( - "Average request duration is {$average_duration}s. Next batch size will be {$next_batch_size}s", - [ - 'method' => __METHOD__, - ] - ); - // Get all in-progress jobs with request sent and no results. /** * Set the delay before an in-progress row is considered as outdated. From 43fbc5de1401cee7453ec35b5dd870952d2724c4 Mon Sep 17 00:00:00 2001 From: Mathieu Lamiot Date: Tue, 25 Jun 2024 10:40:53 +0200 Subject: [PATCH 47/50] revert logs as we have the transients already --- .../inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php b/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php index ae26f31e89..4a6d4c8447 100644 --- a/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php +++ b/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php @@ -10,13 +10,15 @@ use WP_Rocket\Engine\Preload\Controller\{PreloadUrl, Queue}; use WP_Rocket\Engine\Preload\Database\Queries\Cache; use WP_Rocket\Tests\Unit\TestCase; - +use WP_Rocket\Tests\Unit\HasLoggerTrait; /** * Test class covering \WP_Rocket\Engine\Preload\Controller\PreloadUrl::preload_url * * @group Preload */ class Test_PreloadUrl extends TestCase { + use HasLoggerTrait; + protected $queue; protected $query; protected $options; @@ -38,6 +40,7 @@ protected function setUp(): void { $this->file_system, ] )->shouldAllowMockingProtectedMethods(); + $this->set_logger($this->controller); } /** From 495d74915a1d2cdfe65a4eed9ffbb44523696785 Mon Sep 17 00:00:00 2001 From: Mathieu Lamiot Date: Tue, 25 Jun 2024 10:43:04 +0200 Subject: [PATCH 48/50] Revert "revert logs as we have the transients already" This reverts commit 43fbc5de1401cee7453ec35b5dd870952d2724c4. --- .../inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php b/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php index 4a6d4c8447..ae26f31e89 100644 --- a/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php +++ b/tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php @@ -10,15 +10,13 @@ use WP_Rocket\Engine\Preload\Controller\{PreloadUrl, Queue}; use WP_Rocket\Engine\Preload\Database\Queries\Cache; use WP_Rocket\Tests\Unit\TestCase; -use WP_Rocket\Tests\Unit\HasLoggerTrait; + /** * Test class covering \WP_Rocket\Engine\Preload\Controller\PreloadUrl::preload_url * * @group Preload */ class Test_PreloadUrl extends TestCase { - use HasLoggerTrait; - protected $queue; protected $query; protected $options; @@ -40,7 +38,6 @@ protected function setUp(): void { $this->file_system, ] )->shouldAllowMockingProtectedMethods(); - $this->set_logger($this->controller); } /** From f4c9af68961bf84a4ebbf316690482ba72e60dbc Mon Sep 17 00:00:00 2001 From: Mathieu Lamiot Date: Wed, 26 Jun 2024 16:55:18 +0200 Subject: [PATCH 49/50] cast next_batch_size to int --- inc/Engine/Preload/Controller/PreloadUrl.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inc/Engine/Preload/Controller/PreloadUrl.php b/inc/Engine/Preload/Controller/PreloadUrl.php index 6b3adbd0ae..af6fd00ac2 100755 --- a/inc/Engine/Preload/Controller/PreloadUrl.php +++ b/inc/Engine/Preload/Controller/PreloadUrl.php @@ -231,7 +231,7 @@ public function process_pending_jobs() { * Estimate batch size based on request duration. * In case no estimation or there is an issue with the value use $min_batch_size. */ - $next_batch_size = ! $average_duration ? $min_batch_size : round( -5 * $average_duration + 55 ); + $next_batch_size = (int) (! $average_duration ? $min_batch_size : round( -5 * $average_duration + 55 )); // Limit next_batch_size. $next_batch_size = max( $next_batch_size, $min_batch_size ); // Not lower than 5. From cab5b2894eb320a5799502a713acfad2c2ba22c8 Mon Sep 17 00:00:00 2001 From: Mathieu Lamiot Date: Wed, 26 Jun 2024 18:10:57 +0200 Subject: [PATCH 50/50] fix linter --- inc/Engine/Preload/Controller/PreloadUrl.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inc/Engine/Preload/Controller/PreloadUrl.php b/inc/Engine/Preload/Controller/PreloadUrl.php index af6fd00ac2..a085e0ad8c 100755 --- a/inc/Engine/Preload/Controller/PreloadUrl.php +++ b/inc/Engine/Preload/Controller/PreloadUrl.php @@ -231,7 +231,7 @@ public function process_pending_jobs() { * Estimate batch size based on request duration. * In case no estimation or there is an issue with the value use $min_batch_size. */ - $next_batch_size = (int) (! $average_duration ? $min_batch_size : round( -5 * $average_duration + 55 )); + $next_batch_size = (int) ( ! $average_duration ? $min_batch_size : round( -5 * $average_duration + 55 ) ); // Limit next_batch_size. $next_batch_size = max( $next_batch_size, $min_batch_size ); // Not lower than 5.