address comments; remove redundant condition

Author: themisvaltinos

sqlmesh/core/engine_adapter/base.py

@@ -1758,11 +1758,8 @@ def remove_managed_columns(
                     if truncate
                     else existing_rows_query.where(
                         exp.and_(
-                            valid_from_col <= cleanup_ts,
-                            exp.and_(
-                                valid_to_col.is_(exp.Null().not_()),
-                                valid_to_col < cleanup_ts,
-                            ),
+                            valid_to_col.is_(exp.Null().not_()),
+                            valid_to_col < cleanup_ts,
                         ),
                     ),
                 )

sqlmesh/core/snapshot/definition.py

@@ -800,13 +800,14 @@ def get_removal_interval(
         # SCD Type 2 validation that end date is the latest interval if it was provided
         if not is_preview and self.is_scd_type_2 and self.intervals:
             requested_start, requested_end = removal_interval
-            latest_end = max(interval[1] for interval in self.intervals)
-            if requested_end != latest_end:
+            latest_end = self.intervals[-1][1]
+            if requested_end < latest_end:
                 from sqlmesh.core.console import get_console
 
                 get_console().log_warning(
                     f"SCD Type 2 model '{self.model.name}' does not support end date in restatements.\n"
-                    f"Requested end date [{to_ts(requested_end)}] doesn't match latest interval end date [{to_ts(latest_end)}].\n"
+                    f"Requested end date [{to_ts(requested_end)}] is less than the latest interval end date.\n"
+                    f"The requested end date will be ignored. Using the latest interval end instead: [{to_ts(latest_end)}]"
                 )
 
                 removal_interval = self.inclusive_exclusive(requested_start, latest_end, strict)

tests/core/engine_adapter/test_base.py

@@ -1255,8 +1255,7 @@ def test_scd_type_2_by_time(make_mocked_engine_adapter: t.Callable):
     "test_valid_to",
     TRUE AS "_exists"
   FROM "target"
-  WHERE "test_valid_from" <= CAST('2020-01-01 00:00:00' AS TIMESTAMP)
-    AND (NOT "test_valid_to" IS NULL AND "test_valid_to" < CAST('2020-01-01 00:00:00' AS TIMESTAMP))
+  WHERE NOT "test_valid_to" IS NULL AND "test_valid_to" < CAST('2020-01-01 00:00:00' AS TIMESTAMP)
 ), "latest" AS (
   SELECT
     "id",
@@ -1455,8 +1454,7 @@ def test_scd_type_2_by_time_no_invalidate_hard_deletes(make_mocked_engine_adapte
     "test_valid_to",
     TRUE AS "_exists"
   FROM "target"
-  WHERE "test_valid_from" <= CAST('2020-01-01 00:00:00' AS TIMESTAMP)
-    AND (NOT "test_valid_to" IS NULL AND "test_valid_to" < CAST('2020-01-01 00:00:00' AS TIMESTAMP))
+  WHERE NOT "test_valid_to" IS NULL AND "test_valid_to" < CAST('2020-01-01 00:00:00' AS TIMESTAMP)
 ), "latest" AS (
   SELECT
     "id",
@@ -1651,8 +1649,7 @@ def test_merge_scd_type_2_pandas(make_mocked_engine_adapter: t.Callable):
     "test_valid_to",
     TRUE AS "_exists"
     FROM "target"
-    WHERE "test_valid_from" <= CAST('2020-01-01 00:00:00+00:00' AS TIMESTAMPTZ)
-    AND (NOT "test_valid_to" IS NULL AND "test_valid_to" < CAST('2020-01-01 00:00:00+00:00' AS TIMESTAMPTZ))
+    WHERE NOT "test_valid_to" IS NULL AND "test_valid_to" < CAST('2020-01-01 00:00:00+00:00' AS TIMESTAMPTZ)
   ), "latest" AS (
     SELECT
     "id1",
@@ -1831,8 +1828,7 @@ def test_scd_type_2_by_column(make_mocked_engine_adapter: t.Callable):
     "test_valid_to",
     TRUE AS "_exists"
     FROM "target"
-    WHERE "test_VALID_from" <= CAST('2020-01-01 00:00:00' AS TIMESTAMP)
-    AND (NOT "test_valid_to" IS NULL AND "test_valid_to" < CAST('2020-01-01 00:00:00' AS TIMESTAMP))
+    WHERE NOT "test_valid_to" IS NULL AND "test_valid_to" < CAST('2020-01-01 00:00:00' AS TIMESTAMP)
   ), "latest" AS (
     SELECT
     "id",
@@ -2025,8 +2021,7 @@ def test_scd_type_2_by_column_composite_key(make_mocked_engine_adapter: t.Callab
     "test_valid_to",
     TRUE AS "_exists"
     FROM "target"
-    WHERE "test_VALID_from" <= CAST('2020-01-01 00:00:00' AS TIMESTAMP)
-    AND (NOT "test_valid_to" IS NULL AND "test_valid_to" < CAST('2020-01-01 00:00:00' AS TIMESTAMP))
+    WHERE NOT "test_valid_to" IS NULL AND "test_valid_to" < CAST('2020-01-01 00:00:00' AS TIMESTAMP)
   ), "latest" AS (
     SELECT
     "id_a",
@@ -2386,8 +2381,7 @@ def test_scd_type_2_by_column_star_check(make_mocked_engine_adapter: t.Callable)
     "test_valid_to",
     TRUE AS "_exists"
     FROM "target"
-    WHERE "test_valid_from" <= CAST('2020-01-01 00:00:00' AS TIMESTAMP)
-    AND (NOT "test_valid_to" IS NULL AND "test_valid_to" < CAST('2020-01-01 00:00:00' AS TIMESTAMP))
+    WHERE NOT "test_valid_to" IS NULL AND "test_valid_to" < CAST('2020-01-01 00:00:00' AS TIMESTAMP)
   ), "latest" AS (
     SELECT
     "id",
@@ -2562,8 +2556,7 @@ def test_scd_type_2_by_column_no_invalidate_hard_deletes(make_mocked_engine_adap
     "test_valid_to",
     TRUE AS "_exists"
   FROM "target"
-  WHERE "test_valid_from" <= CAST('2020-01-01 00:00:00' AS TIMESTAMP)
-    AND (NOT "test_valid_to" IS NULL AND "test_valid_to" < CAST('2020-01-01 00:00:00' AS TIMESTAMP))
+  WHERE NOT "test_valid_to" IS NULL AND "test_valid_to" < CAST('2020-01-01 00:00:00' AS TIMESTAMP)
 ), "latest" AS (
   SELECT
     "id",