Skip to content

Commit 92163bc

Browse files
chubes4chubes4
committed
Bound URL fetch retry pacing
OpenAI gpt-5.6-sol via OpenCode was used to trace the collector scheduling path and draft the implementation and deterministic tests. Chris Huber reviewed and remains responsible for the change.
1 parent 9e8a5e9 commit 92163bc

3 files changed

Lines changed: 92 additions & 54 deletions

File tree

‎includes/class-static-site-importer-url-site-collector.php‎

Lines changed: 49 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,9 @@ class Static_Site_Importer_URL_Site_Collector {
2727
private const MAX_RESPONSE_BYTES = 10485760;
2828
private const MAX_SITEMAP_DOCUMENTS = 100;
2929
private const MAX_DISCOVERED_ROUTES = 5000;
30+
// The hardened socket transport is blocking, so one PHP invocation can safely admit one request at a time.
31+
private const SAME_ORIGIN_CONCURRENCY = 1;
32+
private const CROSS_ORIGIN_CONCURRENCY = 1;
3033

3134
/**
3235
* Collect a public static site.
@@ -45,26 +48,8 @@ public static function collect( string $url, array $args = array(), ?callable $f
4548
$max_pages = min( self::MAX_PAGES, max( 1, (int) ( $args['max_pages'] ?? self::DEFAULT_MAX_PAGES ) ) );
4649
$max_assets = min( self::MAX_ASSETS, max( 0, (int) ( $args['max_assets'] ?? self::DEFAULT_MAX_ASSETS ) ) );
4750
$max_total_bytes = min( self::MAX_TOTAL_BYTES, max( 1, (int) ( $args['max_total_bytes'] ?? self::DEFAULT_MAX_TOTAL_BYTES ) ) );
48-
$request_delay = min( 2000, max( 0, (int) ( $args['request_delay_ms'] ?? 100 ) ) );
4951
$fetcher = $fetcher ?? static fn ( string $resource_url, array $fetch_args ) => Static_Site_Importer_URL_Fetcher::fetch( $resource_url, $fetch_args );
50-
$fetch_attempts = min( 3, max( 1, (int) ( $args['fetch_attempts'] ?? 2 ) ) );
51-
$fetch_resource = $fetcher;
52-
$fetcher = static function ( string $resource_url, array $fetch_args ) use ( $fetch_resource, $fetch_attempts ) {
53-
$response = null;
54-
for ( $attempt = 0; $attempt < $fetch_attempts; $attempt++ ) {
55-
$response = $fetch_resource( $resource_url, $fetch_args );
56-
if ( ! is_wp_error( $response ) ) {
57-
return $response;
58-
}
59-
}
60-
$data = is_wp_error( $response ) && is_array( $response->get_error_data() ) ? $response->get_error_data() : array();
61-
if ( ! empty( $data['_static_site_importer_cache_aware'] ) ) {
62-
unset( $data['_static_site_importer_cache_aware'] );
63-
$response = new WP_Error( $response->get_error_code(), $response->get_error_message(), $data ?: null );
64-
$fetch_resource( $resource_url, $fetch_args + array( '_static_site_importer_cache_failure' => $response ) );
65-
}
66-
return $response;
67-
};
52+
$fetcher = self::scheduled_fetcher( $fetcher, $args );
6853
$fetch_args = array_intersect_key( $args, array_flip( array( 'timeout' ) ) );
6954
$fetch_args['max_bytes'] = min( self::MAX_RESPONSE_BYTES, $max_total_bytes, max( 1, (int) ( $args['max_bytes'] ?? 5242880 ) ) );
7055

@@ -105,7 +90,6 @@ public static function collect( string $url, array $args = array(), ?callable $f
10590
while ( $page_queue && count( array_filter( $resources, static fn ( array $resource ): bool => 'html' === $resource['kind'] ) ) < $max_pages ) {
10691
$page_url = array_shift( $page_queue );
10792
$response = $fetcher( $page_url, array_merge( $fetch_args, array( 'content_types' => array( 'text/html', 'application/xhtml+xml' ) ) ) );
108-
self::delay_after_fetch( $response, $request_delay, $args );
10993
$response = self::without_cache_marker( $response );
11094
if ( is_wp_error( $response ) ) {
11195
if ( $page_url === $entry_url ) {
@@ -191,7 +175,6 @@ public static function collect( string $url, array $args = array(), ?callable $f
191175
while ( $asset_queue && self::resource_count( $resources, 'asset' ) < $max_assets ) {
192176
$asset_url = array_shift( $asset_queue );
193177
$response = $fetcher( $asset_url, array_merge( $fetch_args, array( 'content_types' => array() ) ) );
194-
self::delay_after_fetch( $response, $request_delay, $args );
195178
$response = self::without_cache_marker( $response );
196179
if ( is_wp_error( $response ) ) {
197180
if ( $preserve_failed_assets ) {
@@ -350,6 +333,7 @@ public static function collect( string $url, array $args = array(), ?callable $f
350333
'source_exclusions' => $source_exclusions,
351334
'truncated' => array_keys( $truncated ),
352335
'sitemap_urls' => count( $sitemap_urls ),
336+
'fetch_scheduling' => self::scheduling_limits( $args ),
353337
'external_asset_retained' => array(
354338
'count' => count( $external_assets ),
355339
'samples' => array_slice( array_map( static fn( string $url, string $reason ): array => array(
@@ -373,22 +357,7 @@ public static function discover_routes( string $url, array $args = array(), ?cal
373357
return new WP_Error( 'static_site_importer_site_collection_invalid_url', 'Enter a valid public site URL.' );
374358
}
375359
$fetcher = $fetcher ?? static fn ( string $resource_url, array $fetch_args ) => Static_Site_Importer_URL_Fetcher::fetch( $resource_url, $fetch_args );
376-
$fetch_attempts = min( 3, max( 1, (int) ( $args['fetch_attempts'] ?? 2 ) ) );
377-
$fetch_resource = $fetcher;
378-
$fetcher = static function ( string $resource_url, array $fetch_args ) use ( $fetch_resource, $fetch_attempts ) {
379-
for ( $attempt = 0; $attempt < $fetch_attempts; $attempt++ ) {
380-
$response = $fetch_resource( $resource_url, $fetch_args );
381-
if ( ! is_wp_error( $response ) ) {
382-
return $response; }
383-
}
384-
$data = is_wp_error( $response ) && is_array( $response->get_error_data() ) ? $response->get_error_data() : array();
385-
if ( ! empty( $data['_static_site_importer_cache_aware'] ) ) {
386-
unset( $data['_static_site_importer_cache_aware'] );
387-
$response = new WP_Error( $response->get_error_code(), $response->get_error_message(), $data ?: null );
388-
$fetch_resource( $resource_url, $fetch_args + array( '_static_site_importer_cache_failure' => $response ) );
389-
}
390-
return $response;
391-
};
360+
$fetcher = self::scheduled_fetcher( $fetcher, $args );
392361
$fetch_args = array_intersect_key( $args, array_flip( array( 'timeout' ) ) );
393362
$fetch_args['max_bytes'] = min( 10485760, max( 1, (int) ( $args['max_bytes'] ?? 5242880 ) ) );
394363
$routes = self::sitemap_urls( $entry_url, $fetcher, $fetch_args );
@@ -884,20 +853,44 @@ private static function failure( string $url, WP_Error $error, string $kind = 'a
884853
);
885854
}
886855

887-
private static function delay_after_fetch( $response, int $milliseconds, array $args ): void {
888-
$error_data = is_wp_error( $response ) ? $response->get_error_data() : null;
889-
if ( is_array( $error_data ) && ! empty( $error_data['_static_site_importer_negative_cache_hit'] ) ) {
890-
return;
891-
}
892-
if ( is_array( $response ) && ! empty( $response['metadata']['_static_site_importer_cache_hit'] ) ) {
893-
unset( $response['metadata']['_static_site_importer_cache_hit'] );
894-
return;
895-
}
896-
if ( isset( $args['_static_site_importer_delay_callback'] ) && is_callable( $args['_static_site_importer_delay_callback'] ) ) {
897-
call_user_func( $args['_static_site_importer_delay_callback'] );
898-
return;
899-
}
900-
self::delay( $milliseconds );
856+
/** @return callable */
857+
private static function scheduled_fetcher( callable $fetch_resource, array $args ): callable {
858+
$fetch_attempts = min( 3, max( 1, (int) ( $args['fetch_attempts'] ?? 2 ) ) );
859+
$retry_delay = min( 2000, max( 0, (int) ( $args['request_delay_ms'] ?? 0 ) ) );
860+
$clock = isset( $args['_static_site_importer_scheduler_clock'] ) && is_callable( $args['_static_site_importer_scheduler_clock'] ) ? $args['_static_site_importer_scheduler_clock'] : static fn (): float => microtime( true );
861+
return static function ( string $resource_url, array $fetch_args ) use ( $fetch_resource, $fetch_attempts, $retry_delay, $clock, $args ) {
862+
$response = null;
863+
$next_allowed = array();
864+
$origin = self::origin( $resource_url );
865+
for ( $attempt = 0; $attempt < $fetch_attempts; $attempt++ ) {
866+
$wait = max( 0, ( $next_allowed[ $origin ] ?? 0 ) - (float) call_user_func( $clock ) );
867+
if ( $wait > 0 ) {
868+
self::delay( (int) ceil( $wait * 1000 ), $args );
869+
}
870+
$response = $fetch_resource( $resource_url, $fetch_args );
871+
if ( ! is_wp_error( $response ) ) {
872+
return $response;
873+
}
874+
// Successful and cached responses are immediately eligible; only a retry is paced.
875+
$next_allowed[ $origin ] = (float) call_user_func( $clock ) + ( $retry_delay / 1000 );
876+
}
877+
$data = is_array( $response->get_error_data() ) ? $response->get_error_data() : array();
878+
if ( ! empty( $data['_static_site_importer_cache_aware'] ) ) {
879+
unset( $data['_static_site_importer_cache_aware'] );
880+
$response = new WP_Error( $response->get_error_code(), $response->get_error_message(), $data ?: null );
881+
$fetch_resource( $resource_url, $fetch_args + array( '_static_site_importer_cache_failure' => $response ) );
882+
}
883+
return $response;
884+
};
885+
}
886+
887+
/** @return array<string,int> */
888+
private static function scheduling_limits( array $args ): array {
889+
return array(
890+
'same_origin_concurrency' => self::SAME_ORIGIN_CONCURRENCY,
891+
'cross_origin_concurrency' => self::CROSS_ORIGIN_CONCURRENCY,
892+
'retry_delay_ms' => min( 2000, max( 0, (int) ( $args['request_delay_ms'] ?? 0 ) ) ),
893+
);
901894
}
902895

903896
private static function without_cache_marker( $response ) {
@@ -912,8 +905,12 @@ private static function without_cache_marker( $response ) {
912905
return new WP_Error( $response->get_error_code(), $response->get_error_message(), $data ?: null );
913906
}
914907

915-
private static function delay( int $milliseconds ): void {
908+
private static function delay( int $milliseconds, array $args = array() ): void {
916909
if ( $milliseconds > 0 ) {
910+
if ( isset( $args['_static_site_importer_delay_callback'] ) && is_callable( $args['_static_site_importer_delay_callback'] ) ) {
911+
call_user_func( $args['_static_site_importer_delay_callback'], $milliseconds );
912+
return;
913+
}
917914
usleep( $milliseconds * 1000 );
918915
}
919916
}

‎tests/smoke-url-batch-import.php‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -175,11 +175,11 @@ function wp_json_encode( $value, int $options = 0 ) { return json_encode( $value
175175
$negative_workspace->purge();
176176
$delay_calls = 0; $shared_asset_calls = 0;
177177
$delay_result = Static_Site_Importer_URL_Batch_Import::import( array( 'url' => 'https://delay.test/', 'work_dir' => sys_get_temp_dir() . '/ssi-delay-' . bin2hex( random_bytes( 4 ) ), 'provider_args' => array( 'collect_site' => true, 'batch_pages' => 1, 'request_delay_ms' => 1, '_static_site_importer_delay_callback' => static function () use ( &$delay_calls ) { $delay_calls++; } ) ), array(), static function ( string $url, array $args ) use ( &$shared_asset_calls ) { if ( 'https://delay.test/sitemap.xml' === $url ) { return array( 'body' => '<urlset><url><loc>https://delay.test/</loc></url><url><loc>https://delay.test/p/</loc></url></urlset>', 'metadata' => array( 'content_type' => 'application/xml', 'final_url' => $url ) ); } if ( 'https://delay.test/shared.png' === $url ) { $shared_asset_calls++; return array( 'body' => 'asset', 'metadata' => array( 'content_type' => 'image/png', 'final_url' => $url ) ); } return array( 'body' => '<img src="/shared.png">', 'metadata' => array( 'content_type' => 'text/html', 'final_url' => $url ) ); }, static fn() => array( 'theme_slug' => 'delay', 'import_report_summary' => array( 'status' => 'completed' ) ) );
178-
if ( is_wp_error( $delay_result ) || 1 !== $shared_asset_calls || 3 !== $delay_calls || 1 > ( $delay_result['url_batch_run']['fetch_cache']['network_requests_avoided'] ?? 0 ) ) { throw new RuntimeException( 'cached shared assets must avoid request delay while cache misses retain it' ); }
178+
if ( is_wp_error( $delay_result ) || 1 !== $shared_asset_calls || 0 !== $delay_calls || 1 > ( $delay_result['url_batch_run']['fetch_cache']['network_requests_avoided'] ?? 0 ) ) { throw new RuntimeException( 'successful cache misses and hits must not incur retry pacing' ); }
179179
$negative_asset_calls = 0; $negative_delays = 0;
180180
$negative_request = array( 'url' => 'https://negative.test/', 'work_dir' => sys_get_temp_dir() . '/ssi-negative-' . bin2hex( random_bytes( 4 ) ), 'provider_args' => array( 'collect_site' => true, 'batch_pages' => 1, 'fetch_attempts' => 2, 'request_delay_ms' => 1, '_static_site_importer_delay_callback' => static function () use ( &$negative_delays ) { $negative_delays++; } ) );
181181
$negative_fetcher = static function ( string $url, array $args ) use ( &$negative_asset_calls ) { if ( 'https://negative.test/sitemap.xml' === $url ) { return array( 'body' => '<urlset><url><loc>https://negative.test/</loc></url><url><loc>https://negative.test/p/</loc></url></urlset>', 'metadata' => array( 'content_type' => 'application/xml', 'final_url' => $url ) ); } if ( 'https://negative.test/shared.png' === $url ) { $negative_asset_calls++; return new WP_Error( 'asset_timeout', 'temporary failure' ); } return array( 'body' => '<img src="/shared.png">', 'metadata' => array( 'content_type' => 'text/html', 'final_url' => $url ) ); };
182182
$negative_result = Static_Site_Importer_URL_Batch_Import::import( $negative_request, array(), $negative_fetcher, static fn() => array( 'theme_slug' => 'negative', 'import_report_summary' => array( 'status' => 'completed' ) ) );
183183
$negative_resume = Static_Site_Importer_URL_Batch_Import::import( $negative_request, array(), $negative_fetcher, static fn() => array() );
184-
if ( is_wp_error( $negative_result ) || is_wp_error( $negative_resume ) || 2 !== $negative_asset_calls || 3 !== $negative_delays || 1 > ( $negative_result['url_batch_run']['fetch_cache']['negative_writes'] ?? 0 ) || 2 !== ( $negative_result['url_batch_run']['external_asset_retained']['count'] ?? 0 ) ) { throw new RuntimeException( 'negative cache failures must retain optional singleton assets without exposing internal cache hooks' ); }
184+
if ( is_wp_error( $negative_result ) || is_wp_error( $negative_resume ) || 2 !== $negative_asset_calls || 1 > ( $negative_result['url_batch_run']['fetch_cache']['negative_writes'] ?? 0 ) || 2 !== ( $negative_result['url_batch_run']['external_asset_retained']['count'] ?? 0 ) ) { throw new RuntimeException( 'negative cache failures must retain optional singleton assets without exposing internal cache hooks' ); }
185185
echo "URL batch import smoke passed.\n";

‎tests/smoke-url-site-collector.php‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -169,6 +169,47 @@ static function ( string $url, array $args ) use ( $shuffled_responses ) {
169169
);
170170
$assert( ! is_wp_error( $shuffled ) && ( $snapshot['sha256'] ?? '' ) === ( $shuffled['source_metadata']['snapshot']['sha256'] ?? null ), 'snapshot-hash-independent-of-discovery-order' );
171171

172+
$schedule_calls = array();
173+
$schedule_delays = array();
174+
$scheduled = Static_Site_Importer_URL_Site_Collector::collect(
175+
'https://schedule.test/',
176+
array( 'max_pages' => 3, 'max_assets' => 0, '_route_set' => array( 'https://schedule.test/', 'https://schedule.test/a/', 'https://schedule.test/b/' ), '_static_site_importer_delay_callback' => static function ( int $milliseconds ) use ( &$schedule_delays ): void { $schedule_delays[] = $milliseconds; } ),
177+
static function ( string $url, array $args ) use ( &$schedule_calls ): array {
178+
unset( $args );
179+
$schedule_calls[] = $url;
180+
return array( 'body' => '<main>' . $url . '</main>', 'metadata' => array( 'content_type' => 'text/html', 'final_url' => $url ) );
181+
}
182+
);
183+
$assert( ! is_wp_error( $scheduled ) && 3 === count( $schedule_calls ) && array() === $schedule_delays, 'successful-uncached-fetches-have-no-default-pacing' );
184+
$assert( array( 'same_origin_concurrency' => 1, 'cross_origin_concurrency' => 1, 'retry_delay_ms' => 0 ) === ( $scheduled['source_metadata']['collection']['fetch_scheduling'] ?? null ), 'blocking-transport-scheduling-limits-are-explicit' );
185+
186+
$retry_now = 0.0;
187+
$retry_calls = 0;
188+
$retry_delays = array();
189+
$retried = Static_Site_Importer_URL_Site_Collector::collect(
190+
'https://retry.test/',
191+
array( 'max_pages' => 1, 'max_assets' => 0, '_route_set' => array( 'https://retry.test/' ), 'fetch_attempts' => 2, 'request_delay_ms' => 125, '_static_site_importer_scheduler_clock' => static function () use ( &$retry_now ): float { return $retry_now; }, '_static_site_importer_delay_callback' => static function ( int $milliseconds ) use ( &$retry_now, &$retry_delays ): void { $retry_delays[] = $milliseconds; $retry_now += $milliseconds / 1000; } ),
192+
static function ( string $url, array $args ) use ( &$retry_calls ) {
193+
unset( $url, $args );
194+
$retry_calls++;
195+
return 1 === $retry_calls ? new WP_Error( 'temporary_failure', 'Retry me.' ) : array( 'body' => '<main>Recovered</main>', 'metadata' => array( 'content_type' => 'text/html', 'final_url' => 'https://retry.test/' ) );
196+
}
197+
);
198+
$assert( ! is_wp_error( $retried ) && 2 === $retry_calls && array( 125 ) === $retry_delays, 'retry-pacing-is-per-origin-and-deterministic' );
199+
200+
$cache_calls = 0;
201+
$cache_delays = array();
202+
$cached = Static_Site_Importer_URL_Site_Collector::collect(
203+
'https://cache-schedule.test/',
204+
array( 'max_pages' => 1, 'max_assets' => 0, '_route_set' => array( 'https://cache-schedule.test/' ), 'request_delay_ms' => 125, '_static_site_importer_delay_callback' => static function ( int $milliseconds ) use ( &$cache_delays ): void { $cache_delays[] = $milliseconds; } ),
205+
static function ( string $url, array $args ) use ( &$cache_calls ): array {
206+
unset( $args );
207+
$cache_calls++;
208+
return array( 'body' => '<main>Cached</main>', 'metadata' => array( 'content_type' => 'text/html', 'final_url' => $url, '_static_site_importer_cache_hit' => true ) );
209+
}
210+
);
211+
$assert( ! is_wp_error( $cached ) && 1 === $cache_calls && array() === $cache_delays, 'cache-hits-never-consume-pacing-budget' );
212+
172213
$compiled = blocks_engine_php_transformer_compile_artifact( $result['artifact'] );
173214
$site_plan = $compiled['source_reports']['wordpress_site_plan'] ?? array();
174215
$site_diagnostics = $compiled['source_reports']['wordpress_site_plan_diagnostics'] ?? array();

0 commit comments

Comments
 (0)