Merge branch 'main' into fail-if-mismatch-ts-moose-lib

Author: phiSgr

apps/framework-cli/src/cli/local_webserver.rs

@@ -3502,7 +3502,6 @@ async fn get_admin_reconciled_inframap(
 ) -> Result<InfrastructureMap, crate::framework::core::plan::PlanningError> {
     use crate::framework::core::state_storage::StateStorageBuilder;
     use crate::infrastructure::olap::clickhouse;
-    use std::collections::HashSet;
 
     // Build state storage from project configuration.
     // This provides access to the persisted infrastructure map (stored in Redis or ClickHouse).
@@ -3530,21 +3529,10 @@ async fn get_admin_reconciled_inframap(
     };
 
     // For admin endpoints, reconcile all currently managed tables and SQL resources only.
-    // Pass the managed table IDs as target_table_ids - this ensures that
-    // reconcile_with_reality only operates on resources that are already managed by Moose.
-    let target_table_ids: HashSet<String> = current_map
-        .tables
-        .values()
-        .map(|t| t.id(&current_map.default_database))
-        .collect();
-
-    let target_sql_resource_ids: HashSet<String> =
-        current_map.sql_resources.keys().cloned().collect();
-
-    let target_materialized_view_ids: HashSet<String> =
-        current_map.materialized_views.keys().cloned().collect();
-
-    let target_view_ids: HashSet<String> = current_map.views.keys().cloned().collect();
+    // Use the current map's resource IDs as the filter—this ensures that
+    // reconcile_with_reality only operates on resources already managed by Moose,
+    // not external tables that happen to exist in the same database.
+    let filter = crate::framework::core::plan::ReconciliationFilter::from_infra_map(&current_map);
 
     // Reconcile the loaded map with actual database state (single load, no race condition).
     // reconcile_with_reality handles the OLAP-disabled case internally, and in the future
@@ -3555,10 +3543,7 @@ async fn get_admin_reconciled_inframap(
         crate::framework::core::plan::reconcile_with_reality(
             project,
             &current_map,
-            &target_table_ids,
-            &target_sql_resource_ids,
-            &target_materialized_view_ids,
-            &target_view_ids,
+            &filter,
             clickhouse_client,
         )
         .await?

apps/framework-cli/src/cli/routines/migrate.rs

@@ -5,7 +5,7 @@ use crate::cli::routines::RoutineFailure;
 use crate::framework::core::infrastructure::table::Table;
 use crate::framework::core::infrastructure_map::InfrastructureMap;
 use crate::framework::core::migration_plan::MigrationPlan;
-use crate::framework::core::plan::reconcile_with_reality;
+use crate::framework::core::plan::{reconcile_with_reality, ReconciliationFilter};
 use crate::framework::core::state_storage::{StateStorage, StateStorageBuilder};
 use crate::infrastructure::olap::clickhouse::config::{ClickHouseConfig, ClusterConfig};
 use crate::infrastructure::olap::clickhouse::IgnorableOperation;
@@ -570,85 +570,59 @@ pub async fn execute_migration(
 
     // Wrap all operations to ensure lock cleanup on any error
     let result = async {
-        // Load current state from ClickHouse state table and reconcile with reality
+        // Load target state from current code first—we need its resource IDs to
+        // correctly filter which unmapped ClickHouse objects to adopt during
+        // reconciliation.  Without this, a fresh Redis (no stored state) would
+        // produce an empty filter, causing reconciliation to ignore every
+        // pre-existing table and the drift check to report them all as "removed".
+        let target_infra_map = InfrastructureMap::load_from_user_code(project, true)
+            .await
+            .map_err(|e| {
+                RoutineFailure::new(
+                    Message::new(
+                        "Code".to_string(),
+                        "Failed to load infrastructure from code".to_string(),
+                    ),
+                    e,
+                )
+            })?;
+
+        // Load current state from state storage and reconcile with reality
         let current_infra_map = state_storage
             .load_infrastructure_map()
             .await
             .map_err(|e| {
                 RoutineFailure::new(
                     Message::new(
                         "State".to_string(),
-                        "Failed to load infrastructure state from ClickHouse".to_string(),
+                        "Failed to load infrastructure state".to_string(),
                     ),
                     e,
                 )
             })?
             .unwrap_or_else(|| InfrastructureMap::empty_from_project(project));
 
         let current_infra_map = if project.features.olap {
-            use std::collections::HashSet;
-
-            // current_infra_map should have gone through fixup_default_db, but better safe than sorry
-            let target_table_ids: HashSet<String> = current_infra_map
-                .tables
-                .values()
-                .map(|t| t.id(&current_infra_map.default_database))
-                .collect();
-
-            let target_sql_resource_ids: HashSet<String> =
-                current_infra_map.sql_resources.keys().cloned().collect();
-
-            let target_materialized_view_ids: HashSet<String> = current_infra_map
-                .materialized_views
-                .keys()
-                .cloned()
-                .collect();
-
-            let target_view_ids: HashSet<String> =
-                current_infra_map.views.keys().cloned().collect();
-
+            let filter = ReconciliationFilter::from_infra_map(&target_infra_map);
             let olap_client = create_client(clickhouse_config.clone());
 
-            // We already have the current_infra_map loaded, so reconcile it directly
-            // instead of reloading from storage via load_reconciled_infrastructure
-            reconcile_with_reality(
-                project,
-                &current_infra_map,
-                &target_table_ids,
-                &target_sql_resource_ids,
-                &target_materialized_view_ids,
-                &target_view_ids,
-                olap_client,
-            )
-            .await
-            .map_err(|e| {
-                RoutineFailure::new(
-                    Message::new(
-                        "Reconciliation".to_string(),
-                        "Failed to reconcile state with ClickHouse reality".to_string(),
-                    ),
-                    anyhow::anyhow!("{:?}", e),
-                )
-            })?
+            reconcile_with_reality(project, &current_infra_map, &filter, olap_client)
+                .await
+                .map_err(|e| {
+                    RoutineFailure::new(
+                        Message::new(
+                            "Reconciliation".to_string(),
+                            "Failed to reconcile state with ClickHouse reality".to_string(),
+                        ),
+                        e,
+                    )
+                })?
         } else {
             current_infra_map
         };
 
         let current_tables = &current_infra_map.tables;
 
-        // Load target state from current code with credentials resolved for migration DDL
-        let target_infra_map = InfrastructureMap::load_from_user_code(project, true)
-            .await
-            .map_err(|e| {
-                RoutineFailure::new(
-                    Message::new(
-                        "Code".to_string(),
-                        "Failed to load infrastructure from code".to_string(),
-                    ),
-                    e,
-                )
-            })?;
-
         // Execute migration
         execute_migration_plan(
             project,

apps/framework-cli/src/cli/routines/mod.rs

@@ -120,6 +120,7 @@ use super::{Message, MessageType};
 use crate::framework::core::partial_infrastructure_map::LifeCycle;
 use crate::framework::core::plan::plan_changes;
 use crate::framework::core::plan::InfraPlan;
+use crate::framework::core::plan::ReconciliationFilter;
 use crate::framework::core::state_storage::StateStorageBuilder;
 use crate::framework::languages::SupportedLanguages;
 use crate::infrastructure::olap::clickhouse::diff_strategy::ClickHouseTableDiffStrategy;
@@ -1268,30 +1269,9 @@ pub async fn remote_plan(
             );
         }
 
-        let table_names: HashSet<String> = local_infra_map
-            .tables
-            .values()
-            .map(|t| t.id(&local_infra_map.default_database))
-            .collect();
-
-        let sql_resource_ids: HashSet<String> =
-            local_infra_map.sql_resources.keys().cloned().collect();
+        let filter = ReconciliationFilter::from_infra_map(&local_infra_map);
 
-        let materialized_view_ids: HashSet<String> =
-            local_infra_map.materialized_views.keys().cloned().collect();
-
-        let view_ids: HashSet<String> = local_infra_map.views.keys().cloned().collect();
-
-        get_remote_inframap_serverless(
-            project,
-            clickhouse_url,
-            None,
-            &table_names,
-            &sql_resource_ids,
-            &materialized_view_ids,
-            &view_ids,
-        )
-        .await?
+        get_remote_inframap_serverless(project, clickhouse_url, None, &filter).await?
     } else {
         // Moose server flow
         if !json {
@@ -1478,27 +1458,10 @@ pub async fn remote_gen_migration(
                 },
             );
 
-            let target_table_ids: HashSet<String> = local_infra_map
-                .tables
-                .values()
-                .map(|t| t.id(&local_infra_map.default_database))
-                .collect();
-            let target_sql_resource_ids: HashSet<String> =
-                local_infra_map.sql_resources.keys().cloned().collect();
-            let target_materialized_view_ids: HashSet<String> =
-                local_infra_map.materialized_views.keys().cloned().collect();
-            let target_view_ids: HashSet<String> = local_infra_map.views.keys().cloned().collect();
-
-            get_remote_inframap_serverless(
-                project,
-                clickhouse_url,
-                redis_url.as_deref(),
-                &target_table_ids,
-                &target_sql_resource_ids,
-                &target_materialized_view_ids,
-                &target_view_ids,
-            )
-            .await?
+            let filter = ReconciliationFilter::from_infra_map(&local_infra_map);
+
+            get_remote_inframap_serverless(project, clickhouse_url, redis_url.as_deref(), &filter)
+                .await?
         }
     };
 
@@ -1561,10 +1524,7 @@ async fn get_remote_inframap_serverless(
     project: &Project,
     clickhouse_url: &str,
     redis_url: Option<&str>,
-    target_table_ids: &HashSet<String>,
-    target_sql_resource_ids: &HashSet<String>,
-    target_materialized_view_ids: &HashSet<String>,
-    target_view_ids: &HashSet<String>,
+    filter: &ReconciliationFilter,
 ) -> anyhow::Result<InfrastructureMap> {
     use crate::infrastructure::olap::clickhouse::config::parse_clickhouse_connection_string;
     use crate::infrastructure::olap::clickhouse::create_client;
@@ -1584,10 +1544,7 @@ async fn get_remote_inframap_serverless(
         project,
         &*state_storage,
         olap_client,
-        target_table_ids,
-        target_sql_resource_ids,
-        target_materialized_view_ids,
-        target_view_ids,
+        filter,
     )
     .await?;