Skip to content

Materialize pages before optional URL hydration #819

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
chubes4 merged 5 commits into from
Aug 3, 2026
Merged

Materialize pages before optional URL hydration #819

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
a8f1df8
Materialize pages before optional URL hydration
chubes4 Aug 3, 2026
20ad534
Merge main and repair URL pipeline lint
chubes4 Aug 3, 2026
9a5fc0f
Merge latest main into page-first materialization
chubes4 Aug 3, 2026
8c8b944
Align page-first URL pipeline formatting
chubes4 Aug 3, 2026
7ee7570
Complete page-ready checkpoint alignment
chubes4 Aug 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 104 additions & 17 deletions includes/class-static-site-importer-url-batch-import.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -167,14 +167,71 @@ static function ( string $bytes, array $metadata ): bool {
return $run_manifest->save( $manifest );
}return self::continuation_result( $manifest, $manifest_path, $index, $effective_batches, $max_effective_batches );
}
$batch = $cursor[ $index ];
$routes = array_values( array_intersect_key( $manifest['routes'], array_flip( $batch['units'] ) ) );
$batch_entry = in_array( $url, $routes, true ) ? $url : ( $routes[0] ?? $url );
$cache_name = 'batches/' . $batch['batch_id'] . '.json';
$old_cache = trailingslashit( $work_dir ) . 'url-site-batch-cache-' . $identity . '-' . $index . '.json';
$raw = self::retained_runtime( $workspace, $cache_name, 'batches/' . $index . '.json', $old_cache, $routes );
$decoded = is_string( $raw ) ? json_decode( $raw, true ) : null;
$runtime = is_array( $decoded ) ? $decoded : array();
$batch = $cursor[ $index ];
$routes = array_values( array_intersect_key( $manifest['routes'], array_flip( $batch['units'] ) ) );
$batch_entry = in_array( $url, $routes, true ) ? $url : ( $routes[0] ?? $url );
$cache_name = 'batches/' . $batch['batch_id'] . '.json';
$ready_cache_name = 'batches/' . $batch['batch_id'] . '.page-ready.json';
$old_cache = trailingslashit( $work_dir ) . 'url-site-batch-cache-' . $identity . '-' . $index . '.json';
if ( null !== $deadline ) {
$ready_raw = self::retained_runtime( $workspace, $ready_cache_name, $ready_cache_name, $ready_cache_name, $routes );
$ready_runtime = array();
if ( is_string( $ready_raw ) && is_array( json_decode( $ready_raw, true ) ) ) {
$ready_runtime = json_decode( $ready_raw, true );
}
if ( empty( $ready_runtime ) ) {
$ready_args = $args;
$ready_args['_route_set'] = array_values( array_unique( $routes ) );
$ready_args['max_pages'] = min( self::MAX_BATCH_PAGES + 1, count( $ready_args['_route_set'] ) + 1 );
$ready_args['require_complete_collection'] = true;
$ready_args['asset_failure_policy'] = count( $routes ) > 1 ? 'preserve_failed_external_assets' : 'preserve_external';
$ready_args['hydration_mode'] = 'page_ready';
$ready_runtime = Static_Site_Importer_URL_Site_Collector::collect( $batch_entry, $ready_args, $fetcher );
if ( is_wp_error( $ready_runtime ) ) {
if ( self::deadline_error( $ready_runtime ) ) {
$manifest['batches'] = self::legacy_batches( $cursor );
self::checkpoint_cache( $manifest, $cache );
$write = $run_manifest->save( $manifest );
if ( is_wp_error( $write ) ) {
return $write;
}
return self::continuation_result( $manifest, $manifest_path, $index, $effective_batches, $max_effective_batches, $max_invocation_seconds, 'deadline_exhausted' );
}
return self::failed( $run_manifest, $workspace, $manifest, $cursor, $index, $ready_runtime, $cache );
}
$write = $workspace->publish_json( $ready_cache_name, $ready_runtime );
if ( is_wp_error( $write ) ) {
return self::failed( $run_manifest, $workspace, $manifest, $cursor, $index, $write, $cache );
}
}
if ( 'page_ready' === $batch['state'] && ( $batch['result']['snapshot_sha256'] ?? '' ) !== ( $ready_runtime['source_metadata']['snapshot']['sha256'] ?? '' ) ) {
return self::failed( $run_manifest, $workspace, $manifest, $cursor, $index, new WP_Error( 'static_site_importer_page_ready_checkpoint_mismatch', 'The immutable page-ready checkpoint no longer matches its persisted receipt.' ), $cache );
}
if ( 'page_ready' !== $batch['state'] && 'pending' === ( $ready_runtime['source_metadata']['collection']['readiness']['optional_assets'] ?? '' ) ) {
$ready_import_args = Static_Site_Importer_URL_Import_Runtime::batch_import_args( $input, $ready_runtime );
$ready_import_args['activate'] = false;
$ready_import_args['batch_import'] = true;
$ready_import_args['preserve_existing_theme_bootstrap'] = $index > 0;
$ready_import_args['import_run_id'] = $identity;
$ready_import_args['page_ready_checkpoint'] = true;
$ready_result = $importer( $ready_runtime['artifact'], $ready_import_args );
if ( is_wp_error( $ready_result ) ) {
return self::failed( $run_manifest, $workspace, $manifest, $cursor, $index, $ready_result, $cache );
}
$cursor[ $index ]['state'] = 'page_ready';
$cursor[ $index ]['result'] = self::result_evidence( $ready_result, $ready_runtime );
$batch = $cursor[ $index ];
$manifest['batches'] = self::legacy_batches( $cursor );
self::checkpoint_cache( $manifest, $cache );
$write = $run_manifest->save( $manifest );
if ( is_wp_error( $write ) ) {
return $write;
}
}
}
$raw = self::retained_runtime( $workspace, $cache_name, 'batches/' . $index . '.json', $old_cache, $routes );
$decoded = is_string( $raw ) ? json_decode( $raw, true ) : null;
$runtime = is_array( $decoded ) ? $decoded : array();
if ( empty( $runtime ) ) {
$collect_args = $args;
$collect_args['_route_set'] = array_values( array_unique( $routes ) );
Expand Down Expand Up @@ -304,16 +361,20 @@ static function ( string $bytes, array $metadata ): bool {
* once per retained resource digest; page envelopes remain batch-local.
*/
private static function prepare_staged_plans( Static_Site_Importer_Artifact_Run_Workspace $workspace, array $artifact, string $resource_digest ): array|WP_Error {
$compiler_class = 'Automattic\\BlocksEngine\\PhpTransformer\\ArtifactCompiler\\ArtifactCompiler';
if ( ! class_exists( $compiler_class ) ) {
return new WP_Error( 'static_site_importer_missing_transformer', 'Blocks Engine php-transformer is required to prepare staged URL batch plans.' );
$compiler = self::staged_compiler();
if ( is_wp_error( $compiler ) ) {
return $compiler;
}
$stored = $workspace->read_raw( 'staged-compiler-shared.json' );
$stored = is_string( $stored ) ? json_decode( $stored, true ) : null;
$shared_prepared = ! is_array( $stored ) || ( $stored['resource_digest'] ?? null ) !== $resource_digest || ! is_array( $stored['plan'] ?? null );
try {
$compiler = new $compiler_class();
$shared = $shared_prepared ? $compiler->prepareShared( $artifact ) : $stored['plan'];
$prepare_shared = array( $compiler, 'prepareShared' );
$prepare_page = array( $compiler, 'preparePage' );
if ( ! is_callable( $prepare_shared ) || ! is_callable( $prepare_page ) ) {
return new WP_Error( 'static_site_importer_missing_transformer_capability', 'The Blocks Engine php-transformer does not support staged URL batch plans.' );
}
$shared = $shared_prepared ? call_user_func( $prepare_shared, $artifact ) : $stored['plan'];
if ( $shared_prepared ) {
$write = $workspace->publish_json(
'staged-compiler-shared.json',
Expand All @@ -331,7 +392,7 @@ private static function prepare_staged_plans( Static_Site_Importer_Artifact_Run_
if ( ! is_array( $file ) || 'text/html' !== strtolower( (string) ( $file['mime_type'] ?? '' ) ) || '' === (string) ( $file['path'] ?? '' ) ) {
continue;
}
$page_plans[] = $compiler->preparePage( $artifact, $shared, (string) $file['path'] );
$page_plans[] = call_user_func( $prepare_page, $artifact, $shared, (string) $file['path'] );
}
return array(
'shared_plan' => $shared,
Expand All @@ -343,13 +404,31 @@ private static function prepare_staged_plans( Static_Site_Importer_Artifact_Run_
}
}
private static function compose_staged_plans( array $staged ): array|WP_Error {
$compiler_class = 'Automattic\\BlocksEngine\\PhpTransformer\\ArtifactCompiler\\ArtifactCompiler';
$compiler = self::staged_compiler();
if ( is_wp_error( $compiler ) ) {
return $compiler;
}
try {
return ( new $compiler_class() )->compose( $staged['shared_plan'], $staged['page_plans'] )->toArray();
$compose = array( $compiler, 'compose' );
if ( ! is_callable( $compose ) ) {
return new WP_Error( 'static_site_importer_missing_transformer_capability', 'The Blocks Engine php-transformer does not support staged URL batch plans.' );
}
$compiled = call_user_func( $compose, $staged['shared_plan'], $staged['page_plans'] );
if ( ! is_object( $compiled ) || ! is_callable( array( $compiled, 'toArray' ) ) ) {
return new WP_Error( 'static_site_importer_invalid_staged_compile', 'The Blocks Engine php-transformer returned an invalid staged URL batch plan.' );
}
return call_user_func( array( $compiled, 'toArray' ) );
} catch ( Throwable $error ) {
return new WP_Error( 'static_site_importer_staged_compose_failed', $error->getMessage() );
}
}
private static function staged_compiler(): mixed {
$compiler_class = 'Automattic\\BlocksEngine\\PhpTransformer\\ArtifactCompiler\\ArtifactCompiler';
if ( ! class_exists( $compiler_class ) ) {
return new WP_Error( 'static_site_importer_missing_transformer', 'Blocks Engine php-transformer is required to prepare staged URL batch plans.' );
}
return new $compiler_class();
}
private static function cached_fetcher( Static_Site_Importer_Artifact_Byte_Cache $cache, ?callable $fetcher ): callable {
$fetcher = $fetcher ?? static fn ( string $url, array $args ) => Static_Site_Importer_URL_Fetcher::fetch( $url, $args );
return static function ( string $url, array $args ) use ( $cache, $fetcher ) {
Expand Down Expand Up @@ -681,7 +760,8 @@ private static function continuation_result( array $manifest, string $path, int
static fn ( $value ): bool => null !== $value
);
$completed_batches = count( array_filter( $manifest['batches'], static fn ( array $batch ): bool => 'completed' === $batch['state'] ) );
$completed_routes = array_sum( array_column( $manifest['batches'], 'completed_routes' ) );
$completed_routes = self::materialized_routes( $manifest['batches'] );
$page_ready_routes = array_sum( array_map( static fn( array $batch ): int => 'page_ready' === ( $batch['state'] ?? '' ) ? count( $batch['route_indexes'] ?? array() ) : 0, $manifest['batches'] ) );
return array(
'success' => true,
'continuation' => true,
Expand All @@ -701,6 +781,7 @@ private static function continuation_result( array $manifest, string $path, int
'per_batch_limits' => $manifest['per_batch_limits'] ?? array(),
'total_routes' => $manifest['total_routes'],
'completed_routes' => $completed_routes,
'page_ready_routes' => $page_ready_routes,
'total_batches' => count( $manifest['batches'] ),
'completed_batches' => $completed_batches,
'effective_batches_processed' => $effective_batches,
Expand All @@ -712,6 +793,12 @@ private static function continuation_result( array $manifest, string $path, int
'batch_materialization' => $manifest['batches'],
);
}
private static function materialized_routes( array $batches ): int {
return array_sum( array_map( static function ( array $batch ): int {
$completed_routes = (int) ( $batch['completed_routes'] ?? 0 );
return 0 !== $completed_routes ? $completed_routes : ( 'page_ready' === ( $batch['state'] ?? '' ) ? count( $batch['route_indexes'] ?? array() ) : 0 );
}, $batches ) );
}
private static function contract( string $url, array $input, array $args, int $batch_pages ): array {
foreach ( array_keys( $args ) as $key ) {
if ( str_starts_with( (string) $key, '_static_site_importer_' ) ) {
Expand Down
57 changes: 51 additions & 6 deletions includes/class-static-site-importer-url-site-collector.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,10 +69,18 @@ public static function collect( string $url, array $args = array(), ?callable $f
$asset_failure_policy = $args['asset_failure_policy'] ?? '';
$preserve_failed_assets = in_array( $asset_failure_policy, array( 'preserve_external', 'preserve_failed_external_assets' ), true );
$preserve_asset_limits = 'preserve_external' === $asset_failure_policy;
$script_policy = self::script_policy( $args );
$script_exclusions = array();
$entry_resource_url = $entry_url;
$site_url = $entry_url;
$page_ready = 'page_ready' === ( $args['hydration_mode'] ?? '' );
$critical_assets = array();
if ( $page_ready ) {
// A ready checkpoint may retain optional resources externally, but never a
// stylesheet or font dependency needed to render the collected page.
$preserve_failed_assets = true;
$preserve_asset_limits = true;
}
$script_policy = self::script_policy( $args );
$script_exclusions = array();
$entry_resource_url = $entry_url;
$site_url = $entry_url;

$sitemap_urls = isset( $args['_route_set'] ) && is_array( $args['_route_set'] ) ? array_values( $args['_route_set'] ) : self::sitemap_urls( $entry_url, $fetcher, $fetch_args );
if ( is_wp_error( $sitemap_urls ) ) {
Expand Down Expand Up @@ -161,10 +169,17 @@ public static function collect( string $url, array $args = array(), ?callable $f
$page_queue[] = $discovered_url;
}

foreach ( self::critical_html_asset_urls( $body, $document_base_url ) as $asset_url ) {
$critical_assets[ $asset_url ] = true;
}
foreach ( self::html_asset_urls( $body, $document_base_url, $scripts['asset_urls'] ) as $asset_url ) {
if ( isset( $queued_assets[ $asset_url ] ) || isset( $resources[ $asset_url ] ) ) {
continue;
}
if ( $page_ready && ! isset( $critical_assets[ $asset_url ] ) ) {
$external_assets[ $asset_url ] = 'optional_pending';
continue;
}
if ( count( $asset_queue ) + self::resource_count( $resources, 'asset' ) >= $max_assets ) {
if ( $preserve_asset_limits ) {
$external_assets[ $asset_url ] = 'asset_limit';
Expand All @@ -181,10 +196,14 @@ public static function collect( string $url, array $args = array(), ?callable $f
$asset_fetcher = self::prefetched_fetcher( array_slice( $asset_queue, 0, $max_assets ), array_merge( $fetch_args, array( 'content_types' => array() ) ), $fetcher, $use_many_fetcher, $args );
while ( $asset_queue && self::resource_count( $resources, 'asset' ) < $max_assets ) {
$asset_url = array_shift( $asset_queue );
$critical = $page_ready && isset( $critical_assets[ $asset_url ] );
$response = $asset_fetcher( $asset_url, array_merge( $fetch_args, array( 'content_types' => array() ) ) );
$response = self::without_cache_marker( $response );
if ( is_wp_error( $response ) ) {
if ( $preserve_failed_assets ) {
if ( 'static_site_importer_invocation_deadline_exceeded' === $response->get_error_code() ) {
return $response;
}
if ( $preserve_failed_assets && ! $critical ) {
$external_assets[ $asset_url ] = $response->get_error_code();
continue;
}
Expand All @@ -203,7 +222,7 @@ public static function collect( string $url, array $args = array(), ?callable $f
$body = (string) $response['body'];
$bytes = strlen( $body );
if ( $total_bytes + $bytes > $max_total_bytes ) {
if ( $preserve_asset_limits ) {
if ( $preserve_asset_limits && ! $critical ) {
$external_assets[ $asset_url ] = 'byte_limit';
continue;
}
Expand All @@ -224,6 +243,7 @@ public static function collect( string $url, array $args = array(), ?callable $f
if ( isset( $queued_assets[ $nested_url ] ) || isset( $resources[ $nested_url ] ) ) {
continue;
}
$critical_assets[ $nested_url ] = true;
if ( count( $asset_queue ) + self::resource_count( $resources, 'asset' ) >= $max_assets ) {
if ( $preserve_asset_limits ) {
$external_assets[ $nested_url ] = 'asset_limit';
Expand Down Expand Up @@ -371,6 +391,12 @@ public static function collect( string $url, array $args = array(), ?callable $f
50
),
),
'readiness' => array(
'mode' => $page_ready ? 'page_ready' : 'complete_snapshot',
'html' => 'complete',
'critical_assets' => 'complete',
'optional_assets' => $page_ready && in_array( 'optional_pending', $external_assets, true ) ? 'pending' : 'complete',
),
),
),
);
Expand Down Expand Up @@ -623,6 +649,25 @@ static function ( array $matches ) use ( $base_url, $policy, &$asset_urls, &$exc
);
}

/** @return array<int,string> */
private static function critical_html_asset_urls( string $html, string $base_url ): array {
$urls = array();
preg_match_all( '#<link\b[^>]*>#is', $html, $matches );
foreach ( $matches[0] as $tag ) {
$relation = strtolower( (string) self::tag_attribute_value( $tag, 'rel' ) );
$href = self::tag_attribute_value( $tag, 'href' );
$as = strtolower( (string) self::tag_attribute_value( $tag, 'as' ) );
if ( null === $href || ( ! str_contains( $relation, 'stylesheet' ) && ( ! str_contains( $relation, 'preload' ) || ! in_array( $as, array( 'style', 'font' ), true ) ) ) ) {
continue;
}
$url = self::resolve_url( $href, $base_url );
if ( '' !== $url ) {
$urls[] = $url;
}
}
return array_values( array_unique( $urls ) );
}

/** @return array<int,string> */
private static function html_css_asset_urls( string $html, string $base_url ): array {
$css = array();
Expand Down
Loading
Loading