Preload adaptative batch

inc/Engine/Preload/Controller/PreloadUrl.php

@@ -66,6 +66,9 @@
 			return;
 		}
 
+		// Should we perform a duration check?
+		$check_duration = get_transient( 'rocket_preload_check_duration' ) ? false : true;
+
 		$requests = [
 			[
 				'url'       => $url,
@@ -120,7 +123,12 @@
 					 */
 					'sslverify' => apply_filters( 'https_local_ssl_verify', false ), // phpcs:ignore WordPress.NamingConventions.PrefixAllGlobals.NonPrefixedHooknameFound
 				]
-				);
+			);
+
+			if ( $check_duration ) {
+				$headers['blocking'] = true;
+				$headers['timeout']  = 20;
+			}
 
 			/**
 			 * Filters the arguments for the preload request.
@@ -136,10 +144,30 @@
 				return;
 			}
 
+			if ( $check_duration ) {
+				$start = microtime( true );
+			}
+
 			wp_safe_remote_get(
 				user_trailingslashit( $request['url'] ),
 				$headers
 			);
+
+			if ( $check_duration ) {
+				$duration = ( microtime( true ) - $start ); // Duration of the request.
+
+				// Update average duration.
+				$previous_request_durations = get_transient( 'rocket_preload_previous_request_durations' ) ?? 0;
+				if ( $previous_request_durations <= 0 ) {
+					$previous_request_durations = $duration;
+				} else {
+					$previous_request_durations = $previous_request_durations * 0.7 + $duration * 0.3;
+				}
+				set_transient( 'rocket_preload_previous_request_durations', $previous_request_durations, 5 * 60 );
+
+				set_transient( 'rocket_preload_check_duration', $duration, 60 ); // Don't check request duration for 1 minute.
+				$check_duration = false;
+			}
 			/**
 			 * Filter the delay between each preload request.
 			 *
@@ -183,7 +211,24 @@
 
 		$pending_actions = $this->queue->get_pending_preload_actions();
 
-		$count = ( (int) apply_filters( 'rocket_preload_cache_pending_jobs_cron_rows_count', 45 ) ) - count( $pending_actions );
+		// Retrieve batch size limits and request timing estimaiton.
+		$max_batch_size           = ( (int) apply_filters( 'rocket_preload_cache_pending_jobs_cron_rows_count', 45 ) ) - count( $pending_actions );
+		$min_batch_size           = ( (int) apply_filters( 'rocket_preload_cache_min_in_progress_jobs_count', 5 ) );
+		$preload_request_duration = get_transient( 'rocket_preload_previous_request_durations' );
+
+		// Estimate batch size based on request duration.
+		if ( ! $preload_request_duration ) {
+			$next_batch_size = $min_batch_size; // In case no estimation or there is an issue with the value.
+		} else {
+			// Linear function: 2s -> 45 jobs // 10s -> 5 jobs.
+			$next_batch_size = round( -5 * $preload_request_duration + 55 );
+		}
+
+		// Limit next_batch_size.
+		$next_batch_size = min( $next_batch_size, $max_batch_size ); // Not higher than 45.
+		$next_batch_size = max( $next_batch_size, $min_batch_size ); // Not lower than 5.
+
+		// Get all in-progress jobs with request sent and no results.
 		/**
 		 * Set the delay before an in-progress row is considered as outdated.
 		 *
@@ -198,11 +243,11 @@
 			 * @param int $count number of rows in batches.
 			 * @return int
 			 */
-			(int) ( $count / 15 )
+			(int) ( $max_batch_size / 15 )
 		);
-
 		$stuck_rows = $this->query->get_outdated_in_progress_jobs( $delay );
 
+		// Make sure the request has been sent for those jobs.
 		$stuck_rows = array_filter(
 			$stuck_rows,
 			function ( $row ) use ( $pending_actions ) {
@@ -215,10 +260,13 @@
 			}
 			);
 
+		// Make those hanging jobs failed.
 		foreach ( $stuck_rows as $row ) {
 			$this->query->make_status_failed( $row->id );
 		}
-		$rows = $this->query->get_pending_jobs( $count );
+
+		// Add new jobs in progress.
+		$rows = $this->query->get_pending_jobs( $next_batch_size );
 		foreach ( $rows as $row ) {
 
 			if ( $this->is_excluded_by_filter( $row->url ) ) {

inc/Engine/WPRocketUninstall.php

@@ -79,6 +79,8 @@ class WPRocketUninstall {
 		'wp_rocket_pricing_timeout',
 		'wp_rocket_pricing_timeout_active',
 		'rocket_get_refreshed_fragments_cache',
+		'rocket_preload_previous_request_durations',
+		'rocket_preload_check_duration',
 	];
 
 	/**

tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php

@@ -14,6 +14,7 @@
 					'sslverify' => false,
 				]
 			],
+			'transient' => 0,
 		]
 	],
 	'mobileNotActivatedShouldPreloadOnlyOnce' => [
@@ -24,13 +25,14 @@
 			'user_agent' => 'user_agent',
 			'request' => [
 				'config' => [
-					'blocking' => false,
-					'timeout'  => 0.01,
+					'blocking' => true,
+					'timeout'  => 20,
 					'user-agent' => 'WP Rocket/Preload',
 					'sslverify' => false,
 				]
 			],
-		]
+			'transient' => 1,
+		],
 	],
 	'mobileActivatedShouldPreloadTwice' => [
 		'config' => [
@@ -40,8 +42,8 @@
 			'user_agent' => 'user_agent',
 			'request' => [
 				'config' => [
-					'blocking' => false,
-					'timeout'  => 0.01,
+					'blocking' => true,
+					'timeout'  => 20,
 					'user-agent' => 'WP Rocket/Preload',
 					'sslverify' => false,
 				]
@@ -54,6 +56,7 @@
 					'sslverify' => false,
 				]
 			],
+			'transient' => 0,
 		]
 	]
 ];

tests/Fixtures/inc/Engine/Preload/Controller/PreloadUrl/processPendingJobs.php

@@ -48,12 +48,13 @@
 			false,
 			true,
 			],
-			'rows' => 101,
+			'rows' => 5,
 			'jobs' => [
 				$row1,
 				$row2,
 				$row3
-			]
+			],
+			'transient' => 0,
 		],
 		'expected' => [
 			'outdated_jobs_id' => [

tests/Fixtures/inc/Engine/Preload/Cron/Subscriber/processPendingUrls.php

@@ -120,7 +120,7 @@
 				],
 				[
 					'url' => 'http://example.org/test3',
-					'status' => 'pending'
+					'status' => 'failed'
 				],
 				[
 					'url' => 'http://example.org/test4',
@@ -147,15 +147,15 @@
 				],
 				[
 					'url' => 'http://example.org/test3',
-					'status' => 'pending'
+					'status' => 'failed'
 				],
 				[
 					'url' => 'http://example.org/test4',
-					'status' => 'pending'
+					'status' => 'in-progress'
 				],
 				[
 					'url' => 'http://example.org/test5',
-					'status' => 'pending'
+					'status' => 'in-progress'
 				],
 			],
 			'actions' => [
@@ -181,14 +181,14 @@
 					],
 				],
 				[
-					'exists' => false,
+					'exists' => true,
 					'args' => [
 						'hook'   => 'rocket_preload_job_preload_url',
 						'args' => ['http://example.org/test4']
 					],
 				],
 				[
-					'exists' => false,
+					'exists' => true,
 					'args' => [
 						'hook'   => 'rocket_preload_job_preload_url',
 						'args' => ['http://example.org/test5']

tests/Integration/inc/Engine/WPRocketUninstall/uninstall.php

@@ -41,6 +41,8 @@ class Test_Uninstall extends FilesystemTestCase {
 		'rocket_preload_complete'                         => null,
 		'rocket_preload_complete_time'                    => null,
 		'rocket_preload_errors'                           => null,
+		'rocket_preload_previous_request_durations'       => null,
+		'rocket_preload_check_duration'                   => null,
 		'rocket_database_optimization_process'            => null,
 		'rocket_database_optimization_process_complete'   => null,
 	];

tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/preloadUrl.php

@@ -41,6 +41,11 @@ protected function setUp(): void
 	 * @dataProvider configTestData
 	 */
 	public function testShouldDoAsExpected($config) {
+		Functions\when( 'get_transient' )
+			->justReturn();
+		Functions\when( 'set_transient' )
+			->justReturn();
+
 		$this->options->expects()->get('do_caching_mobile_files', false)->andReturn($config['cache_mobile']);
 		$this->controller->expects()->is_already_cached($config['url'])->andReturn($config['cache_exists']);
 		$this->configureRequest($config);

tests/Unit/inc/Engine/Preload/Controller/PreloadUrl/processPendingJobs.php

@@ -11,6 +11,7 @@
 use WP_Rocket\Engine\Preload\Database\Queries\RocketCache;
 use WP_Rocket\Tests\Unit\TestCase;
 use Brain\Monkey\Filters;
+use Brain\Monkey\Functions;
 
 /**
  * @covers \WP_Rocket\Engine\Preload\Controller\PreloadUrl::process_pending_jobs
@@ -38,7 +39,8 @@ protected function setUp(): void
 	 * @dataProvider configTestData
 	 */
 	public function testShouldDoAsExpected($config, $expected) {
-
+		Functions\when( 'get_transient' )
+			->justReturn( $config['transient'] );
 		$this->queue->expects()->get_pending_preload_actions()->andReturn([]);
 		Filters\expectApplied('rocket_preload_cache_pending_jobs_cron_rows_count')->with(100)->andReturn($config['rows']);
 		$this->query->expects(self::once())->method( 'get_outdated_in_progress_jobs' )->with()->willReturn($config['outdated_jobs']);