Skip to content

Commit 1915448

Browse files
faisalahammadfaisalahammad
committed
feat: classify imported documents as posts or pages
SSI imported every static document as a page. Add a document type classifier that reads plan signals to decide post vs page instead. - New Static_Site_Importer_Document_Type_Classifier keys off dated head meta (article:published_time, pubdate, dc.date, ...), a /YYYY/MM/ URL, or an explicit metadata.post_type from markdown frontmatter. - The plan materializer honors the classified post_type, applies the detected publish date to post_date and post_date_gmt, and reconciles across both posts and pages so re-imports stay idempotent. - The site entrypoint and undated documents stay pages, matching today's behavior, so existing imports are unchanged. Foundation for a follow-up that turns a dated index into a Query Loop. Fixes #513
1 parent 2e05210 commit 1915448

4 files changed

Lines changed: 264 additions & 3 deletions

‎includes/class-static-site-importer-document-type-classifier.php‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
<?php
2+
/**
3+
* Post-vs-page classification for imported site plan documents.
4+
*
5+
* @package StaticSiteImporter
6+
*/
7+
8+
if ( ! defined( 'ABSPATH' ) ) {
9+
exit;
10+
}
11+
12+
/**
13+
* Classifies plan page rows as WordPress posts or pages.
14+
*
15+
* A document is a post when it carries an article/temporal signal that
16+
* survives the Blocks Engine plan projection: a dated head meta tag or a
17+
* hierarchical date URL (YYYY/MM). The site entrypoint is always a page;
18+
* everything else without date evidence stays a page to preserve existing
19+
* import behavior.
20+
*/
21+
final class Static_Site_Importer_Document_Type_Classifier {
22+
23+
/** Head meta keys that carry a publish date. */
24+
private const PUBLISH_DATE_META_KEYS = array(
25+
'article:published_time',
26+
'article:published',
27+
'pubdate',
28+
'publishdate',
29+
'date',
30+
'dc.date.issued',
31+
'dc.date',
32+
'parsely-pub-date',
33+
'releasedate',
34+
);
35+
36+
/**
37+
* Classify one plan page row.
38+
*
39+
* @param array<string,mixed> $page One row from the resolved plan pages.
40+
* @return array{post_type:string,date:?string,signal:string}
41+
*/
42+
public static function classify( array $page ): array {
43+
if ( ! empty( $page['entrypoint'] ) ) {
44+
return self::result( 'page', null, 'page_default' );
45+
}
46+
47+
// The compiler defaults metadata.post_type to 'page' for every HTML
48+
// document, so only a non-page value counts as an explicit override.
49+
$explicit = sanitize_key( (string) ( $page['metadata']['post_type'] ?? '' ) );
50+
if ( '' !== $explicit && 'page' !== $explicit && self::post_type_registered( $explicit ) ) {
51+
return self::result( $explicit, self::publish_date( $page ), 'explicit_metadata' );
52+
}
53+
54+
$date = self::publish_date( $page );
55+
$route = (string) ( $page['route']['path'] ?? '' );
56+
if ( null !== $date ) {
57+
return self::result( 'post', $date, 'dated_meta' );
58+
}
59+
if ( self::route_is_dated_hierarchy( $route ) ) {
60+
return self::result( 'post', null, 'dated_route' );
61+
}
62+
63+
// No date evidence and no /YYYY/MM/ URL: stay a page. This preserves
64+
// about / contact / nav-linked sources untouched.
65+
return self::result( 'page', null, 'page_default' );
66+
}
67+
68+
/**
69+
* Build the classification result.
70+
*
71+
* @param string $post_type Classified post type.
72+
* @param string|null $date Detected publish date, if any.
73+
* @param string $signal Signal that drove the classification.
74+
* @return array{post_type:string,date:?string,signal:string}
75+
*/
76+
private static function result( string $post_type, ?string $date, string $signal ): array {
77+
return array( 'post_type' => $post_type, 'date' => $date, 'signal' => $signal );
78+
}
79+
80+
/**
81+
* Detect a parseable publish date from the page head meta.
82+
*
83+
* @param array<string,mixed> $page Plan page row.
84+
* @return string|null MySQL datetime, or null when no dated meta parses.
85+
*/
86+
private static function publish_date( array $page ): ?string {
87+
$metadata_date = isset( $page['metadata']['date'] ) ? self::normalize_date( (string) $page['metadata']['date'] ) : null;
88+
if ( null !== $metadata_date ) {
89+
return $metadata_date;
90+
}
91+
$meta = isset( $page['document_metadata']['meta'] ) && is_array( $page['document_metadata']['meta'] ) ? $page['document_metadata']['meta'] : array();
92+
foreach ( $meta as $row ) {
93+
if ( ! is_array( $row ) ) {
94+
continue;
95+
}
96+
$key = (string) ( $row['name'] ?? $row['property'] ?? $row['http_equiv'] ?? '' );
97+
if ( ! in_array( strtolower( $key ), self::PUBLISH_DATE_META_KEYS, true ) ) {
98+
continue;
99+
}
100+
$timestamp = strtotime( (string) ( $row['content'] ?? '' ) );
101+
if ( false === $timestamp ) {
102+
continue;
103+
}
104+
return date( 'Y-m-d H:i:s', $timestamp );
105+
}
106+
return null;
107+
}
108+
109+
/**
110+
* Normalize a date string into MySQL datetime or null.
111+
*
112+
* @param string $value Raw date string.
113+
* @return string|null
114+
*/
115+
private static function normalize_date( string $value ): ?string {
116+
$timestamp = strtotime( $value );
117+
if ( false === $timestamp ) {
118+
return null;
119+
}
120+
return date( 'Y-m-d H:i:s', $timestamp );
121+
}
122+
123+
/**
124+
* Whether the route path follows a hierarchical date URL (YYYY/MM).
125+
*
126+
* @param string $route Route path.
127+
* @return bool
128+
*/
129+
private static function route_is_dated_hierarchy( string $route ): bool {
130+
return 1 === preg_match( '#(?:^|/)\d{4}/(?:0?[1-9]|1[0-2])(?:/|$)#', $route );
131+
}
132+
133+
/**
134+
* Whether a post type is registered on the runtime.
135+
*
136+
* Falls back to the built-in types in standalone tests where WP is not
137+
* fully bootstrapped.
138+
*
139+
* @param string $post_type Post type name.
140+
* @return bool
141+
*/
142+
private static function post_type_registered( string $post_type ): bool {
143+
if ( function_exists( 'get_post_type_object' ) ) {
144+
return null !== get_post_type_object( $post_type );
145+
}
146+
return in_array( $post_type, array( 'page', 'post' ), true );
147+
}
148+
}

‎includes/class-static-site-importer-wordpress-site-plan-materializer.php‎

Lines changed: 39 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -187,13 +187,20 @@ private static function preflight_state( array &$state, bool $overwrite ): void
187187
throw new InvalidArgumentException( 'duplicate_page_route' );
188188
}
189189
$pages_by_route[ $route ] = true;
190+
191+
$classification = Static_Site_Importer_Document_Type_Classifier::classify( $page );
192+
$page['post_type'] = $classification['post_type'];
193+
$page['metadata']['post_type'] = $classification['post_type'];
194+
$page['metadata']['detected_date'] = $classification['date'];
195+
$page['metadata']['classification_signal'] = $classification['signal'];
196+
$state['resolved']['pages'][ array_search( $page['source_path'], array_column( $state['resolved']['pages'], 'source_path' ), true ) ] = $page;
190197
$existing = self::reconciled_post( $page['reconciliation_identity'] );
191198
if ( $existing ) {
192199
$page = self::plan_existing_page( $state, $page, $existing, 'reconciliation_identity_match' );
193200
$state['resolved']['pages'][ array_search( $page['source_path'], array_column( $state['resolved']['pages'], 'source_path' ), true ) ] = $page;
194201
continue;
195202
}
196-
$conflict = '' === trim( $route, '/' ) ? null : get_page_by_path( trim( $route, '/' ), OBJECT, 'page' );
203+
$conflict = '' === trim( $route, '/' ) ? null : get_page_by_path( trim( $route, '/' ), OBJECT, $page['post_type'] );
197204
if ( $conflict && ! $overwrite ) {
198205
throw new InvalidArgumentException( 'post_conflict' );
199206
}
@@ -240,15 +247,25 @@ private static function materialize_page( array $page, array $source_ids ) {
240247
if ( false === $parent ) {
241248
return new WP_Error( 'missing_parent_page' );
242249
}
250+
$post_type = sanitize_key( (string) ( $page['post_type'] ?? 'page' ) );
251+
if ( ! self::is_valid_post_type( $post_type ) ) {
252+
$post_type = 'page';
253+
}
243254
$post = array(
244255
'ID' => (int) ( $page['planned_existing_id'] ?? 0 ),
245-
'post_type' => 'page',
256+
'post_type' => $post_type,
246257
'post_status' => 'publish',
247258
'post_title' => $page['title'],
248259
'post_name' => $page['slug'],
249260
'post_parent' => $parent,
250261
'post_content' => wp_slash( $page['materialized_block_markup'] ?? $page['resolved_block_markup'] ),
251262
);
263+
if ( ! empty( $page['metadata']['detected_date'] ) ) {
264+
// The classifier normalizes dates to UTC, so both fields carry the
265+
// same value. Only dated documents set post_date.
266+
$post['post_date'] = (string) $page['metadata']['detected_date'];
267+
$post['post_date_gmt'] = (string) $page['metadata']['detected_date'];
268+
}
252269
$id = wp_insert_post( $post, true );
253270
if ( is_wp_error( $id ) ) {
254271
return $id;
@@ -460,10 +477,29 @@ private static function apply_operation( array $operation, array $page_ids ) {
460477
}
461478

462479
private static function reconciled_post( string $identity ) {
463-
$posts = get_posts( array( 'post_type' => 'page', 'post_status' => 'any', 'meta_key' => self::RECONCILIATION_META_KEY, 'meta_value' => $identity, 'numberposts' => 1 ) );
480+
// The reconciliation meta key is unique per document, so no post_type
481+
// filter is needed; 'any' covers posts, pages, and custom import types.
482+
$posts = get_posts( array( 'post_type' => 'any', 'post_status' => 'any', 'meta_key' => self::RECONCILIATION_META_KEY, 'meta_value' => $identity, 'numberposts' => 1 ) );
464483
return isset( $posts[0] ) ? $posts[0] : null;
465484
}
466485

486+
/**
487+
* Whether a post type is acceptable as an import target.
488+
*
489+
* Internal types (revision, nav_menu_item, wp_template_part) are not
490+
* valid import targets even when registered.
491+
*
492+
* @param string $post_type Post type name.
493+
* @return bool
494+
*/
495+
private static function is_valid_post_type( string $post_type ): bool {
496+
if ( function_exists( 'get_post_type_object' ) ) {
497+
$object = get_post_type_object( $post_type );
498+
return $object instanceof WP_Post_Type && ! empty( $object->public );
499+
}
500+
return in_array( $post_type, array( 'page', 'post' ), true );
501+
}
502+
467503
/** @param array<int,array<string,mixed>> $pages */
468504
private static function page_exists_in_plan( array $pages, string $identity ): bool {
469505
foreach ( $pages as $page ) {

‎static-site-importer.php‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@
6868
require_once STATIC_SITE_IMPORTER_PATH . 'includes/class-static-site-importer-validation-runtime.php';
6969
require_once STATIC_SITE_IMPORTER_PATH . 'includes/class-static-site-importer-report-diagnostics.php';
7070
require_once STATIC_SITE_IMPORTER_PATH . 'includes/class-static-site-importer-font-materializer.php';
71+
require_once STATIC_SITE_IMPORTER_PATH . 'includes/class-static-site-importer-document-type-classifier.php';
7172
require_once STATIC_SITE_IMPORTER_PATH . 'includes/class-static-site-importer-wordpress-site-plan-materializer.php';
7273
require_once STATIC_SITE_IMPORTER_PATH . 'includes/class-static-site-importer-figma-import.php';
7374
require_once STATIC_SITE_IMPORTER_PATH . 'includes/class-static-site-importer-theme-exporter.php';

‎tests/smoke-wordpress-site-plan-materializer.php‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,8 +84,17 @@ function wp_insert_post( array $post, bool $wp_error ) {
8484
$GLOBALS['ssi_plan_posts'][ $id ] = $post;
8585
return $id;
8686
}
87+
class WP_Post_Type {
88+
public string $name;
89+
public bool $public;
90+
public function __construct( string $name, bool $public = true ) { $this->name = $name; $this->public = $public; }
91+
}
92+
function get_post_type_object( string $post_type ): ?object {
93+
return in_array( $post_type, array( 'page', 'post' ), true ) ? new WP_Post_Type( $post_type ) : null;
94+
}
8795

8896
require dirname( __DIR__ ) . '/includes/class-static-site-importer-font-materializer.php';
97+
require dirname( __DIR__ ) . '/includes/class-static-site-importer-document-type-classifier.php';
8998
require dirname( __DIR__ ) . '/includes/class-static-site-importer-wordpress-site-plan-materializer.php';
9099
require dirname( __DIR__ ) . '/includes/class-static-site-importer-woo-product-seeder.php';
91100
require dirname( __DIR__ ) . '/includes/class-static-site-importer-form-seeder.php';
@@ -442,6 +451,73 @@ function wp_insert_post( array $post, bool $wp_error ) {
442451
$assert( 'about' === ( $GLOBALS['ssi_plan_posts'][ $about_id ]['post_name'] ?? null ) && 0 === ( $GLOBALS['ssi_plan_posts'][ $about_id ]['post_parent'] ?? null ), 'nested index page slug matches its top-level canonical route' );
443452
$assert( 'team' === ( $GLOBALS['ssi_plan_posts'][ $team_id ]['post_name'] ?? null ) && $about_id === ( $GLOBALS['ssi_plan_posts'][ $team_id ]['post_parent'] ?? null ), 'deeper nested index page preserves canonical slug and WordPress parent identity' );
444453

454+
$GLOBALS['ssi_plan_posts'] = array();
455+
$GLOBALS['ssi_plan_meta'] = array();
456+
$classify_artifact = array(
457+
'entrypoint' => 'index.html',
458+
'files' => array(
459+
'index.html' => '<main><h1>Home</h1></main>',
460+
'blog/hello.html' => '<html><head><meta property="article:published_time" content="2024-03-12T10:00:00Z"></head><body><main><h1>Hello</h1></main></body></html>',
461+
'2024/03/dated-post.html' => '<main><h1>Dated by URL</h1></main>',
462+
'blog/about-the-blog.html' => '<main><h1>About the blog</h1></main>',
463+
),
464+
);
465+
$classify_plan = ( new ArtifactCompiler() )->compile( $classify_artifact )->toArray()['source_reports']['wordpress_site_plan'];
466+
$classify_receipt = Static_Site_Importer_WordPress_Site_Plan_Materializer::materialize( $classify_plan, array( 'slug' => 'classify-plan' ) );
467+
$classify_ids = $classify_receipt['completed']['pages'] ?? array();
468+
$home_id = (int) ( $classify_ids['index.html'] ?? 0 );
469+
$post_id = (int) ( $classify_ids['blog/hello.html'] ?? 0 );
470+
$url_dated_id = (int) ( $classify_ids['2024/03/dated-post.html'] ?? 0 );
471+
$blog_about_id = (int) ( $classify_ids['blog/about-the-blog.html'] ?? 0 );
472+
$assert( 'page' === ( $GLOBALS['ssi_plan_posts'][ $home_id ]['post_type'] ?? null ), 'undated entrypoint stays a page by default' );
473+
$assert( 'post' === ( $GLOBALS['ssi_plan_posts'][ $post_id ]['post_type'] ?? null ) && '2024-03-12 10:00:00' === ( $GLOBALS['ssi_plan_posts'][ $post_id ]['post_date'] ?? null ), 'dated article meta classifies a document as a post with its publish date' );
474+
$assert( 'post' === ( $GLOBALS['ssi_plan_posts'][ $url_dated_id ]['post_type'] ?? null ), 'hierarchical YYYY/MM route classifies a document as a post' );
475+
$assert( 'page' === ( $GLOBALS['ssi_plan_posts'][ $blog_about_id ]['post_type'] ?? null ), 'a post-like URL without date evidence stays a page' );
476+
477+
// Re-import the same plan without resetting the post store: reconciliation
478+
// must reuse existing rows and keep both count and post types stable.
479+
$classify_repeat = Static_Site_Importer_WordPress_Site_Plan_Materializer::materialize( $classify_plan, array( 'slug' => 'classify-plan' ) );
480+
$assert( 'completed' === $classify_repeat['status'] && count( $classify_ids ) === count( $classify_repeat['completed']['pages'] ?? array() ), 'classification re-import completes with the same page count' );
481+
foreach ( array( 'index.html', 'blog/hello.html', '2024/03/dated-post.html', 'blog/about-the-blog.html' ) as $real_source ) {
482+
$id = (int) ( $classify_ids[ $real_source ] ?? 0 );
483+
$expected_type = in_array( $real_source, array( 'blog/hello.html', '2024/03/dated-post.html' ), true ) ? 'post' : 'page';
484+
$repeat_id = (int) ( $classify_repeat['completed']['pages'][ $real_source ] ?? 0 );
485+
// Synthetic compiler route pages are recreated on re-import; only the
486+
// real source documents must reuse the same post id.
487+
$assert( $id === $repeat_id && $expected_type === ( $GLOBALS['ssi_plan_posts'][ $id ]['post_type'] ?? null ), 'classification re-import reuses real document post ids with stable post types' );
488+
}
489+
490+
$GLOBALS['ssi_plan_posts'] = array();
491+
$GLOBALS['ssi_plan_meta'] = array();
492+
$parented_artifact = array(
493+
'entrypoint' => 'index.html',
494+
'files' => array(
495+
'index.html' => '<main><h1>Home</h1></main>',
496+
'about/index.html' => '<main><h1>About</h1></main>',
497+
'about/blog/index.html' => '<html><head><meta property="article:published_time" content="2024-01-05T08:00:00Z"></head><body><main><h1>Blog</h1></main></body></html>',
498+
),
499+
);
500+
$parented_plan = ( new ArtifactCompiler() )->compile( $parented_artifact )->toArray()['source_reports']['wordpress_site_plan'];
501+
$parented_receipt = Static_Site_Importer_WordPress_Site_Plan_Materializer::materialize( $parented_plan, array( 'slug' => 'parented-plan' ) );
502+
$parented_ids = $parented_receipt['completed']['pages'] ?? array();
503+
$parented_blog_id = (int) ( $parented_ids['about/blog/index.html'] ?? 0 );
504+
$assert( 'post' === ( $GLOBALS['ssi_plan_posts'][ $parented_blog_id ]['post_type'] ?? null ), 'a dated article nested under a page hierarchy still classifies as a post' );
505+
506+
$GLOBALS['ssi_plan_posts'] = array();
507+
$GLOBALS['ssi_plan_meta'] = array();
508+
$explicit_artifact = array(
509+
'entrypoint' => 'index.html',
510+
'files' => array(
511+
'index.html' => '<main><h1>Home</h1></main>',
512+
'notes/ideas.md' => "---\ntitle: Ideas\ntype: post\n---\n\n# Ideas\nBody",
513+
),
514+
);
515+
$explicit_plan = ( new ArtifactCompiler() )->compile( $explicit_artifact )->toArray()['source_reports']['wordpress_site_plan'];
516+
$explicit_receipt = Static_Site_Importer_WordPress_Site_Plan_Materializer::materialize( $explicit_plan, array( 'slug' => 'explicit-plan' ) );
517+
$explicit_ids = $explicit_receipt['completed']['pages'] ?? array();
518+
$ideas_id = (int) ( $explicit_ids['notes/ideas.md'] ?? 0 );
519+
$assert( 'post' === ( $GLOBALS['ssi_plan_posts'][ $ideas_id ]['post_type'] ?? null ), 'explicit markdown frontmatter post_type overrides signal-free detection' );
520+
445521
$GLOBALS['ssi_plan_posts'] = array();
446522
$GLOBALS['ssi_plan_meta'] = array();
447523
$GLOBALS['ssi_plan_fail_after'] = 1;

0 commit comments

Comments
 (0)