dbt 1.9 migration

dbt/include/dremio/macros/adapters/snapshot.sql

@@ -14,22 +14,33 @@ limitations under the License.*/
 
 
 {% macro dremio__snapshot_merge_sql(target, source, insert_cols) -%}
-    {%- set insert_cols_csv = insert_cols | join(', ') -%}
-
-    merge into {{ target }} as DBT_INTERNAL_DEST
-    using {{ source }} as DBT_INTERNAL_SOURCE
-    on DBT_INTERNAL_SOURCE.dbt_scd_id = DBT_INTERNAL_DEST.dbt_scd_id
-
-    when matched
-        then update
-        set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to
-
-    when not matched
-        then insert ({{ insert_cols_csv }})
-        values
-            ({% for column_name in insert_cols -%}
-                DBT_INTERNAL_SOURCE.{{ column_name }}
-                {%- if not loop.last %}, {%- endif %}
-            {%- endfor %})
-
-{% endmacro %}
+  {%- set columns = config.get("snapshot_table_column_names") or get_snapshot_table_column_names() -%}
+  {%- set insert_cols_csv = insert_cols | join(', ') -%}
+
+  merge into {{ target }} as DBT_INTERNAL_DEST
+    using (
+      select * from {{ source }}
+      where dbt_change_type in ('update', 'delete', 'insert')
+    ) as DBT_INTERNAL_SOURCE
+    on (DBT_INTERNAL_SOURCE.{{ columns.dbt_scd_id }} = DBT_INTERNAL_DEST.{{ columns.dbt_scd_id }}
+        and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')
+        {%- if config.get("dbt_valid_to_current") %}
+        and (
+          DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }}
+          or DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null
+        )
+        {%- else %}
+        and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null
+        {%- endif %})
+
+    when matched then update set {{ columns.dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ columns.dbt_valid_to }}
+
+    when not matched then insert ({{ insert_cols_csv }})
+      values
+      (
+        {%- for column_name in insert_cols -%}
+          DBT_INTERNAL_SOURCE.{{ column_name }}{% if not loop.last %}, {% endif %}
+        {%- endfor %}
+      )
+
+{% endmacro %}

dbt/include/dremio/macros/builtins/builtins.sql

@@ -24,8 +24,8 @@ limitations under the License.*/
       and model.config.format is defined
       else none -%}
     {%- set format_clause = format_clause_from_node(model.config) if format is not none else none -%}
-    {%- set relation2 = api.Relation.create(database=relation.database, schema=relation.schema, identifier=relation.identifier, format=format, format_clause=format_clause) -%}
-    {{ return (relation2) }}
+    {%- set relation2 = api.Relation.create(database=relation.database, schema=relation.schema, identifier=relation.identifier, format=format, format_clause=format_clause, limit=relation.limit) -%}
+      {{ return (relation2) }}
   {%- else -%}
     {{ return (relation) }}
   {%- endif -%}
@@ -40,8 +40,8 @@ limitations under the License.*/
       and source.external.format is defined
       else none -%}
     {%- set format_clause = format_clause_from_node(source.external) if format is not none else none -%}
-    {%- set relation2 = api.Relation.create(database=relation.database, schema=relation.schema, identifier=relation.identifier, format=format, format_clause=format_clause) -%}
-    {{ return (relation2) }}
+    {%- set relation2 = api.Relation.create(database=relation.database, schema=relation.schema, identifier=relation.identifier, format=format, format_clause=format_clause, limit=relation.limit) -%}
+      {{ return (relation2) }}
   {%- else -%}
     {{ return (relation) }}
   {%- endif -%}

dev_requirements.txt

@@ -3,8 +3,8 @@ Babel==2.12.1
 betterproto==1.2.5
 certifi==2023.7.22
 charset-normalizer==3.1.0
-dbt-core==1.8.8
-dbt-tests-adapter==1.8.0
+dbt-core==1.9.1
+dbt-tests-adapter==1.10.1
 python-dotenv==1.0.1
 exceptiongroup==1.1.1
 future==0.18.3

setup.py

@@ -22,7 +22,7 @@
 
 package_name = "dbt-dremio"
 
-package_version = "1.8.4"
+package_version = "1.9.0"
 
 description = """The Dremio adapter plugin for dbt"""
 
@@ -37,8 +37,9 @@
     packages=find_namespace_packages(include=["dbt", "dbt.*"]),
     include_package_data=True,
     install_requires=[
-        "dbt-core>=1.8",
-        "dbt-adapters>=1.0.0, <2.0.0",
+        "dbt-core>=1.9",
+        "dbt-common>=1.11,<2.0",
+        "dbt-adapters>=1.10.1, <2.0",
         "requests>=2.31.0",
     ],
     classifiers=[

tests/functional/adapter/empty/test_empty.py

@@ -41,15 +41,3 @@ def models(self):
             "model.sql": _models.model_sql,
             "sources.yml": schema_sources_yml,
         }
-
-    def test_run_with_empty(self, project):
-        # create source from seed
-        run_dbt(["seed"])
-
-        # run without empty - 3 expected rows in output - 1 from each input
-        # run_dbt(["run"])
-        # self.assert_row_count(project, "model", 3)
-
-        # run with empty - 0 expected rows in output
-        run_dbt(["build", "--empty"])
-        self.assert_row_count(project, "model", 0)

tests/hooks/test_run_hooks.py

@@ -13,22 +13,21 @@
 import pytest
 
 from pathlib import Path
+from dbt_common.exceptions.base import DbtRuntimeError
 
 from tests.hooks.fixtures import (
     macros__hook,
     macros__before_and_after,
     models__hooks,
-    seeds__example_seed_csv,
-    macros_missing_column,
-    models__missing_column,
+    seeds__example_seed_csv
 )
 
 from dbt.tests.util import (
     check_table_does_not_exist,
     run_dbt,
 )
 
-from dbt.tests.adapter.hooks.test_run_hooks import TestAfterRunHooks
+from dbt.tests.adapter.hooks.test_run_hooks import BaseAfterRunHooks
 
 from tests.utils.util import BUCKET, SOURCE
 
@@ -195,11 +194,9 @@ def test_pre_and_post_seed_hooks(self, setUp, project, dbt_profile_target):
         self.assert_used_schemas(project)
 
 
-class TestAfterRunHooksDremio(TestAfterRunHooks):
-    @pytest.fixture(scope="class")
-    def macros(self):
-        return {"temp_macro.sql": macros_missing_column}
-
-    @pytest.fixture(scope="class")
-    def models(self):
-        return {"test_column.sql": models__missing_column}
+class TestAfterRunHooksDremio(BaseAfterRunHooks):
+    def test_missing_column_pre_hook(self, project):
+        # Changing DbtDatabaseError to DbtRuntimeError because our exception_handler
+        # only raises DbtRuntimeError
+        with pytest.raises(DbtRuntimeError):
+            run_dbt(["run"], expect_pass=False)