dbt 1.9 migration

CHANGELOG.md

@@ -1,3 +1,13 @@
+# dbt-dremio v1.9.0
+
+## Changes
+
+- Updated dbt-dremio to match dbt-core v1.9 with the snapshots improvements
+
+## Dependency
+
+- Upgraded dbt-core to 1.9.0 and dbt-tests-adapter to 1.11.0
+
 # dbt-dremio v1.8.4
 
 ## Changes

dbt/adapters/dremio/__version__.py

@@ -9,4 +9,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-version = "1.8.4"
+version = "1.9.0"

dbt/include/dremio/macros/adapters/snapshot.sql

@@ -14,22 +14,33 @@ limitations under the License.*/
 
 
 {% macro dremio__snapshot_merge_sql(target, source, insert_cols) -%}
-    {%- set insert_cols_csv = insert_cols | join(', ') -%}
-
-    merge into {{ target }} as DBT_INTERNAL_DEST
-    using {{ source }} as DBT_INTERNAL_SOURCE
-    on DBT_INTERNAL_SOURCE.dbt_scd_id = DBT_INTERNAL_DEST.dbt_scd_id
-
-    when matched
-        then update
-        set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to
-
-    when not matched
-        then insert ({{ insert_cols_csv }})
-        values
-            ({% for column_name in insert_cols -%}
-                DBT_INTERNAL_SOURCE.{{ column_name }}
-                {%- if not loop.last %}, {%- endif %}
-            {%- endfor %})
-
-{% endmacro %}
+  {%- set columns = config.get("snapshot_table_column_names") or get_snapshot_table_column_names() -%}
+  {%- set insert_cols_csv = insert_cols | join(', ') -%}
+
+  merge into {{ target }} as DBT_INTERNAL_DEST
+    using (
+      select * from {{ source }}
+      where dbt_change_type in ('update', 'delete', 'insert')
+    ) as DBT_INTERNAL_SOURCE
+    on (DBT_INTERNAL_SOURCE.{{ columns.dbt_scd_id }} = DBT_INTERNAL_DEST.{{ columns.dbt_scd_id }}
+        and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')
+        {%- if config.get("dbt_valid_to_current") %}
+        and (
+          DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }}
+          or DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null
+        )
+        {%- else %}
+        and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null
+        {%- endif %})
+
+    when matched then update set {{ columns.dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ columns.dbt_valid_to }}
+
+    when not matched then insert ({{ insert_cols_csv }})
+      values
+      (
+        {%- for column_name in insert_cols -%}
+          DBT_INTERNAL_SOURCE.{{ column_name }}{% if not loop.last %}, {% endif %}
+        {%- endfor %}
+      )
+
+{% endmacro %}

dbt/include/dremio/macros/builtins/builtins.sql

@@ -24,8 +24,8 @@ limitations under the License.*/
       and model.config.format is defined
       else none -%}
     {%- set format_clause = format_clause_from_node(model.config) if format is not none else none -%}
-    {%- set relation2 = api.Relation.create(database=relation.database, schema=relation.schema, identifier=relation.identifier, format=format, format_clause=format_clause) -%}
-    {{ return (relation2) }}
+    {%- set relation2 = api.Relation.create(database=relation.database, schema=relation.schema, identifier=relation.identifier, format=format, format_clause=format_clause, limit=relation.limit) -%}
+      {{ return (relation2) }}
   {%- else -%}
     {{ return (relation) }}
   {%- endif -%}
@@ -40,8 +40,8 @@ limitations under the License.*/
       and source.external.format is defined
       else none -%}
     {%- set format_clause = format_clause_from_node(source.external) if format is not none else none -%}
-    {%- set relation2 = api.Relation.create(database=relation.database, schema=relation.schema, identifier=relation.identifier, format=format, format_clause=format_clause) -%}
-    {{ return (relation2) }}
+    {%- set relation2 = api.Relation.create(database=relation.database, schema=relation.schema, identifier=relation.identifier, format=format, format_clause=format_clause, limit=relation.limit) -%}
+      {{ return (relation2) }}
   {%- else -%}
     {{ return (relation) }}
   {%- endif -%}

dev_requirements.txt

@@ -3,8 +3,8 @@ Babel==2.12.1
 betterproto==1.2.5
 certifi==2023.7.22
 charset-normalizer==3.1.0
-dbt-core==1.8.8
-dbt-tests-adapter==1.8.0
+dbt-core==1.9.0
+dbt-tests-adapter==1.11.0
 python-dotenv==1.0.1
 exceptiongroup==1.1.1
 future==0.18.3

setup.py

@@ -22,7 +22,7 @@
 
 package_name = "dbt-dremio"
 
-package_version = "1.8.4"
+package_version = "1.9.0"
 
 description = """The Dremio adapter plugin for dbt"""
 
@@ -37,8 +37,9 @@
     packages=find_namespace_packages(include=["dbt", "dbt.*"]),
     include_package_data=True,
     install_requires=[
-        "dbt-core>=1.8",
-        "dbt-adapters>=1.0.0, <2.0.0",
+        "dbt-core>=1.9",
+        "dbt-common>=1.11,<2.0",
+        "dbt-adapters>=1.10.1, <2.0",
         "requests>=2.31.0",
     ],
     classifiers=[

tests/functional/adapter/empty/test_empty.py

@@ -41,15 +41,3 @@ def models(self):
             "model.sql": _models.model_sql,
             "sources.yml": schema_sources_yml,
         }
-
-    def test_run_with_empty(self, project):
-        # create source from seed
-        run_dbt(["seed"])
-
-        # run without empty - 3 expected rows in output - 1 from each input
-        # run_dbt(["run"])
-        # self.assert_row_count(project, "model", 3)
-
-        # run with empty - 0 expected rows in output
-        run_dbt(["build", "--empty"])
-        self.assert_row_count(project, "model", 0)