Merge pull request #245 from microsoft/v1.8.0

V1.8.0 release

Author: prdpsvs

dbt/adapters/synapse/__version__.py

@@ -1 +1 @@
-version = "1.8.0rc3"
+version = "1.8.0"

dbt/include/synapse/macros/adapters/columns.sql

@@ -7,7 +7,7 @@
       path={"identifier": relation.identifier.replace("#", "") ~ '_tmp_tbl_hack'},
       type='table')-%}
 
-    {% do  drop_relation(tmp_tbl_hack) %}
+    {% do adapter.drop_relation(tmp_tbl_hack) %}
     {% set sql_create %}
         CREATE TABLE {{ tmp_tbl_hack }}
         WITH(
@@ -23,7 +23,7 @@
     {% call statement() -%} {{ sql_create }} {%- endcall %}
 
     {% set output = get_columns_in_relation(tmp_tbl_hack) %}
-    {% do  drop_relation(tmp_tbl_hack) %}
+    {% do adapter.drop_relation(tmp_tbl_hack) %}
     {{ return(output) }}
   {% endif %}
 

dbt/include/synapse/macros/adapters/metadata.sql

@@ -1,3 +1,10 @@
+{%- macro synapse__get_use_database_sql(database) -%}
+{%- endmacro -%}
+
+{%- macro default__get_use_database_sql(database) -%}
+  {{ return('') }}
+{%- endmacro -%}
+
 {% macro synapse__list_schemas(database) %}
   {% call statement('list_schemas', fetch_result=True, auto_begin=False) -%}
     select  name as [schema]
@@ -8,6 +15,7 @@
 
 {% macro synapse__list_relations_without_caching(schema_relation) %}
   {% call statement('list_relations_without_caching', fetch_result=True) -%}
+    {{ get_use_database_sql(schema_relation.database) }}
     select
       table_catalog as [database],
       table_name as [name],
@@ -26,6 +34,7 @@
 
 {% macro synapse__get_relation_without_caching(schema_relation) -%}
   {% call statement('list_relations_without_caching', fetch_result=True) -%}
+    {{ get_use_database_sql(schema_relation.database) }}
     select
       table_catalog as [database],
       table_name as [name],

dbt/include/synapse/macros/adapters/relation.sql

@@ -1,31 +1,12 @@
-{% macro synapse__drop_relation(relation) -%}
-  {% call statement('drop_relation', auto_begin=False) -%}
-    {{ synapse__drop_relation_script(relation) }}
-  {%- endcall %}
-{% endmacro %}
-
-{% macro synapse__drop_relation_script(relation) -%}
-  {% if relation is not none %}
+{% macro synapse__get_drop_sql(relation) -%}
   {% if relation.type == 'view' or relation.type == 'materialized_view' -%}
     {% set object_id_type = 'V' %}
   {% elif relation.type == 'table'%}
     {% set object_id_type = 'U' %}
   {%- else -%} invalid target name
   {% endif %}
-    if object_id ('{{ relation }}','{{ object_id_type }}') is not null
-    {% if relation.type == 'view' or relation.type == 'materialized_view' -%}
-      begin
-      drop view {{ relation }}
-      end
-    {% elif relation.type == 'table' %}
-      begin
-      drop {{ relation.type }} {{ relation }}
-      end
-    {% endif %}
-  {% else %}
-    -- no object to drop
-    select 1 as nothing
-  {% endif %}
+  if object_id ('{{ relation }}','{{ object_id_type }}') is not null
+    drop {{ relation.type }} {{ relation }}
 {% endmacro %}
 
 {% macro synapse__rename_relation(from_relation, to_relation) -%}

dbt/include/synapse/macros/adapters/replace.sql

@@ -26,23 +26,24 @@
         {{ get_create_intermediate_sql(target_relation, sql) }};
         {{ get_create_backup_sql(existing_relation) }};
         {{ get_rename_intermediate_sql(target_relation) }};
-        {{ synapse__drop_relation(existing_relation) }}
+        {% do adapter.drop_relation(existing_relation) %}
 
     {# /* create target_relation as an intermediate relation, then swap it out with the existing one without using a backup */ #}
     {%- elif target_relation.can_be_renamed -%}
         {{ get_create_intermediate_sql(target_relation, sql) }};
-        {{ synapse__drop_relation(existing_relation) }};
+        {% do adapter.drop_relation(existing_relation) %}
         {{ get_rename_intermediate_sql(target_relation) }}
 
     {# /* create target_relation in place by first backing up the existing relation */ #}
     {%- elif existing_relation.can_be_renamed -%}
         {{ get_create_backup_sql(existing_relation) }};
         {{ get_create_sql(target_relation, sql) }};
-        {{ synapse__drop_relation(existing_relation) }}
+        {% do adapter.drop_relation(existing_relation) %}
 
     {# /* no renaming is allowed, so just drop and create */ #}
     {%- else -%}
         {{ synapse__drop_relation(existing_relation) }};
+        {% do adapter.drop_relation(existing_relation) %}
         {{ get_create_sql(target_relation, sql) }}
 
     {%- endif -%}

dbt/include/synapse/macros/adapters/schema.sql

@@ -9,6 +9,7 @@
 
 {% macro synapse__create_schema_with_authorization(relation, schema_authorization) -%}
   {% call statement('create_schema') -%}
+    
     IF NOT EXISTS (SELECT * FROM sys.schemas WHERE name = '{{ relation.schema }}')
     BEGIN
     EXEC('CREATE SCHEMA [{{ relation.schema }}] AUTHORIZATION [{{ schema_authorization }}]')
@@ -25,7 +26,7 @@
                                                identifier=row[1],
                                                type=row[3]
                                                ) -%}
-    {% do drop_relation(schema_relation) %}
+    {% do adapter.drop_relation(schema_relation) %}
   {%- endfor %}
 
   {% call statement('drop_schema') -%}

dbt/include/synapse/macros/materializations/models/materialized_view/materialized_view.sql

@@ -32,8 +32,9 @@
     they return none
 
     -- drop the temp relations if they exist already in the database
-    {{ synapse__drop_relation(preexisting_backup_relation) }}
-    {{ synapse__drop_relation(preexisting_intermediate_relation) }}
+    {% do adapter.drop_relation(preexisting_backup_relation) %}
+    {% do adapter.drop_relation(preexisting_intermediate_relation) %}
+    
 
     {{ run_hooks(pre_hooks, inside_transaction=False) }}
 
@@ -43,8 +44,9 @@
 {% macro materialized_view_teardown(backup_relation, intermediate_relation, post_hooks) %}
 
     -- drop the temp relations if they exist to leave the database clean for the next run
-    {{ synapse__drop_relation_script(backup_relation) }}
-    {{ synapse__drop_relation_script(intermediate_relation) }}
+
+    {% do adapter.drop_relation(backup_relation) %}
+    {% do adapter.drop_relation(intermediate_relation) %}
 
     {{ run_hooks(post_hooks, inside_transaction=False) }}
 

dbt/include/synapse/macros/materializations/models/table/create_table_as.sql

@@ -1,31 +1,25 @@
+-- Need to check why model contract are not enforced.
+-- TODO: Is it because Synapse uses Fabric table materialization and usage of this macro build model constraints?
 {% macro synapse__create_table_as(temporary, relation, sql) -%}
-   {%- set index = config.get('index', default="CLUSTERED COLUMNSTORE INDEX") -%}
-   {%- set dist = config.get('dist', default="ROUND_ROBIN") -%}
-   {% set tmp_relation = relation.incorporate(
-   path={"identifier": relation.identifier.replace("#", "") ~ '_temp_view'},
-   type='view')-%}
-   {%- set temp_view_sql = sql.replace("'", "''") -%}
+    {%- set index = config.get('index', default="CLUSTERED COLUMNSTORE INDEX") -%}
+    {%- set dist = config.get('dist', default="ROUND_ROBIN") -%}
+    {% set tmp_relation = relation.incorporate(path={"identifier": relation.identifier ~ '__dbt_tmp_vw'}, type='view')-%}
+    {%- set temp_view_sql = sql.replace("'", "''") -%}
 
-   {{ synapse__drop_relation_script(tmp_relation) }}
+    {{ get_create_view_as_sql(tmp_relation, sql) }}
+    {% set contract_config = config.get('contract') %}
 
-   {{ synapse__drop_relation_script(relation) }}
-
-   {{ synapse__create_view_as(tmp_relation, sql) }}
-
-   {% set contract_config = config.get('contract') %}
-
-   {% if contract_config.enforced %}
+    {% if contract_config.enforced %}
 
         {{exceptions.warn("Model contracts cannot be enforced by <adapter>!")}}
 
         CREATE TABLE [{{relation.schema}}].[{{relation.identifier}}]
         {{ synapse__build_columns_constraints(tmp_relation) }}
         WITH(
-          DISTRIBUTION = {{dist}},
-          {{index}}
+            DISTRIBUTION = {{dist}},
+            {{index}}
         )
         {{ get_assert_columns_equivalent(sql)  }}
-
         {% set listColumns %}
             {% for column in model['columns'] %}
                 {{ "["~column~"]" }}{{ ", " if not loop.last }}
@@ -34,11 +28,8 @@
 
         INSERT INTO [{{relation.schema}}].[{{relation.identifier}}]
         ({{listColumns}}) SELECT {{listColumns}} FROM [{{tmp_relation.schema}}].[{{tmp_relation.identifier}}]
-
     {%- else %}
         EXEC('CREATE TABLE [{{relation.database}}].[{{relation.schema}}].[{{relation.identifier}}]WITH(DISTRIBUTION = {{dist}},{{index}}) AS (SELECT * FROM [{{tmp_relation.database}}].[{{tmp_relation.schema}}].[{{tmp_relation.identifier}}]);');
     {% endif %}
-
-   {{ synapse__drop_relation_script(tmp_relation) }}
-
+    {% do adapter.drop_relation(tmp_relation)%}
 {% endmacro %}

dbt/include/synapse/macros/materializations/models/view/create_view_as.sql

@@ -1,12 +1,10 @@
 {% macro synapse__create_view_as(relation, sql) -%}
 
   {%- set temp_view_sql = sql.replace("'", "''") -%}
-
   {% set contract_config = config.get('contract') %}
 
-
   {% if contract_config.enforced %}
-      {{ exceptions.warn("Model contracts cannot be enforced by <adapter>!" }}
+      {{ exceptions.warn("Model contracts cannot be enforced by <adapter>!") }}
       {{ get_assert_columns_equivalent(sql) }}
   {%- endif %}
 

dbt/include/synapse/macros/materializations/tests/helpers.sql

@@ -0,0 +1,41 @@
+{% macro synapse__get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}
+
+  -- Create target schema in synapse db if it does not
+  IF NOT EXISTS (SELECT * FROM sys.schemas WHERE name = '{{ target.schema }}')
+  BEGIN
+    EXEC('CREATE SCHEMA [{{ target.schema }}]')
+  END
+
+  {% if main_sql.strip().lower().startswith('with') %}
+    {% set testview %}
+      {{ target.schema }}.testview_{{ range(1300, 19000) | random }}
+    {% endset %}
+
+    {% set sql = main_sql.replace("'", "''")%}
+    EXEC('create view {{testview}} as {{ sql }};')
+    select
+      {{ "top (" ~ limit ~ ')' if limit != none }}
+      {{ fail_calc }} as failures,
+      case when {{ fail_calc }} {{ warn_if }}
+        then 'true' else 'false' end as should_warn,
+      case when {{ fail_calc }} {{ error_if }}
+        then 'true' else 'false' end as should_error
+    from (
+      select * from {{testview}}
+    ) dbt_internal_test;
+
+    EXEC('drop view {{testview}};')
+
+  {% else -%}
+    select
+      {{ "top (" ~ limit ~ ')' if limit != none }}
+      {{ fail_calc }} as failures,
+      case when {{ fail_calc }} {{ warn_if }}
+        then 'true' else 'false' end as should_warn,
+      case when {{ fail_calc }} {{ error_if }}
+        then 'true' else 'false' end as should_error
+    from (
+      {{ main_sql }}
+    ) dbt_internal_test
+  {%- endif -%}
+{%- endmacro %}

dev_requirements.txt

@@ -1,8 +1,9 @@
 # install latest changes in dbt-core
 # TODO: how to automate switching from develop to version branches?
-git+https://github.com/dbt-labs/dbt-core.git@fc431010ef0bd11ee6a502fc6c9e5e3e75c5d72d#egg=dbt-core&subdirectory=core
-git+https://github.com/dbt-labs/dbt-adapters.git@4c289b150853b94beb67921f2a8dd203abe53cbe
-git+https://github.com/dbt-labs/dbt-adapters.git@4c289b150853b94beb67921f2a8dd203abe53cbe#subdirectory=dbt-tests-adapter
+git+https://github.com/dbt-labs/[email protected]#egg=dbt-core&subdirectory=core
+git+https://github.com/dbt-labs/dbt-adapters.git
+git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter
+git+https://github.com/dbt-labs/dbt-common.git
 
 pytest==8.0.1
 twine==5.0.0
@@ -11,7 +12,8 @@ pre-commit==3.5.0;python_version<"3.9"
 pre-commit==3.6.2;python_version>="3.9"
 pytest-dotenv==0.5.2
 aiohttp==3.8.3
-azure-mgmt-synapse==2.0.0
+#azure-mgmt-synapse==2.0.0
 flaky==3.7.0
 pytest-xdist==3.5.0
 -e .
+#-e /mnt/c/users/pvenkat/repos/dbt-fabric

setup.py

@@ -16,7 +16,7 @@
     "Sam Debruyn",
 ]
 dbt_version = "1.8"
-dbt_fabric_requirement = "dbt-fabric==1.8.2"
+dbt_fabric_requirement = "dbt-fabric==1.8.6"
 description = """An Azure Synapse adapter plugin for dbt"""
 
 this_directory = os.path.abspath(os.path.dirname(__file__))

tests/functional/adapter/test_constraints.py

@@ -648,29 +648,34 @@ def models(self):
     @pytest.fixture(scope="class")
     def expected_sql(self):
         return """
-        if object_id <model_identifier> is not null begin drop view <model_identifier> end
-        if object_id <model_identifier> is not null begin drop table <model_identifier> end
-        exec(\'create view <model_identifier> as select \'\'blue\'\' as "from",1 as id,\'\'2019-01-01\'\' as date_day;\');
-        create table <model_identifier>([id] int not null,[from] varchar(100)not null,[date_day] varchar(100))
-        with(distribution = round_robin,heap)
-        insert into <model_identifier>([id],[from],[date_day])
-        select [id],[from],[date_day] from <model_identifier>
-        if object_id <model_identifier> is not null begin drop view <model_identifier> end
+        exec(\'create view <model_identifier> as select \'\'blue\'\' as "from",1 as id,\'\'2019-01-01\'\' as date_day;\'); create table <model_identifier>([id] int not null,[from] varchar(100)not null,[date_day] varchar(100))with(distribution = round_robin,heap)insert into <model_identifier>([id],[from],[date_day])select [id],[from],[date_day] from <model_identifier>
         """
 
 
 class TestTableConstraintsRuntimeDdlEnforcementSynapse(BaseConstraintsRuntimeDdlEnforcement):
-    pass
+    @pytest.fixture(scope="class")
+    def expected_sql(self):
+        return """
+        exec('create view <model_identifier> as -- depends_on: <foreign_key_model_identifier> select ''blue'' as color,1 as id,''2019-01-01'' as date_day;'); create table <model_identifier>([id] int not null,[color] varchar(100),[date_day] varchar(100))with(distribution = round_robin,heap)insert into <model_identifier>([id],[color],[date_day])select [id],[color],[date_day] from <model_identifier>
+        """
 
 
 class TestIncrementalConstraintsRuntimeDdlEnforcementSynapse(
     BaseIncrementalConstraintsRuntimeDdlEnforcement
 ):
-    pass
+    @pytest.fixture(scope="class")
+    def expected_sql(self):
+        return """
+        exec('create view <model_identifier> as -- depends_on: <foreign_key_model_identifier> select ''blue'' as color,1 as id,''2019-01-01'' as date_day;'); create table <model_identifier>([id] int not null,[color] varchar(100),[date_day] varchar(100))with(distribution = round_robin,heap)insert into <model_identifier>([id],[color],[date_day])select [id],[color],[date_day] from <model_identifier>
+        """
 
 
 class TestModelConstraintsRuntimeEnforcementSynapse(BaseModelConstraintsRuntimeEnforcement):
-    pass
+    @pytest.fixture(scope="class")
+    def expected_sql(self):
+        return """
+        exec('create view <model_identifier> as -- depends_on: <foreign_key_model_identifier> select ''blue'' as color,1 as id,''2019-01-01'' as date_day;'); create table <model_identifier>([id] int not null,[color] varchar(100),[date_day] varchar(100))with(distribution = round_robin,heap)insert into <model_identifier>([id],[color],[date_day])select [id],[color],[date_day] from <model_identifier>
+        """
 
 
 class TestTableConstraintsColumnsEqualSynapse(BaseTableConstraintsColumnsEqual):