Merge pull request #192 from microsoft/v1.8.7

Addressing issues 189, 188, 181, 179, #197

Author: prdpsvs

.github/workflows/publish-docker.yml

@@ -24,7 +24,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Log in to the Container registry
-        uses: docker/login-action@v3.0.0
+        uses: docker/login-action@v3.3.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}

CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+### V1.8.7
+* Improving table materialization to minimize downtime #189
+* Handling temp tables in incremental models #188
+* Add label support to filter queries #181
+* Addressed bug - incremental models cannot full refresh #179 
+* Addressed bug - #197, dbt test incorrect syntax with macro helpers.sql
+
 ### v1.8.0rc2
 
 ## Bug Fixes

dbt/adapters/fabric/__version__.py

@@ -1 +1 @@
-version = "1.8.6"
+version = "1.8.7"

dbt/include/fabric/macros/adapters/catalog.sql

@@ -1,131 +1,132 @@
 {% macro fabric__get_catalog(information_schemas, schemas) -%}
+    {% set query_label = apply_label() %}
+    {%- call statement('catalog', fetch_result=True) -%}
 
-  {%- call statement('catalog', fetch_result=True) -%}
-
-    with
-    principals as (
-        select
-            name as principal_name,
-            principal_id as principal_id
-        from
-            sys.database_principals {{ information_schema_hints() }}
-    ),
-
-    schemas as (
-        select
-            name as schema_name,
-            schema_id as schema_id,
-            principal_id as principal_id
-        from
-            sys.schemas {{ information_schema_hints() }}
-    ),
-
-    tables as (
-        select
-            object_id,
-            name as table_name,
-            schema_id as schema_id,
-            principal_id as principal_id,
-            'BASE TABLE' as table_type
-        from
-            sys.tables {{ information_schema_hints() }}
-    ),
-
-    tables_with_metadata as (
-        select
-            object_id,
-            table_name,
-            schema_name,
-            coalesce(tables.principal_id, schemas.principal_id) as owner_principal_id,
-            table_type
-        from
-            tables
-        join schemas on tables.schema_id = schemas.schema_id
-    ),
-
-    views as (
-        select
-            object_id,
-            name as table_name,
-            schema_id as schema_id,
-            principal_id as principal_id,
-            'VIEW' as table_type
-        from
-            sys.views {{ information_schema_hints() }}
-    ),
-
-    views_with_metadata as (
-        select
-            object_id,
-            table_name,
-            schema_name,
-            coalesce(views.principal_id, schemas.principal_id) as owner_principal_id,
-            table_type
-        from
-            views
-        join schemas on views.schema_id = schemas.schema_id
-    ),
-
-    tables_and_views as (
-        select
-            object_id,
-            table_name,
-            schema_name,
-            principal_name,
-            table_type
-        from
-            tables_with_metadata
-        join principals on tables_with_metadata.owner_principal_id = principals.principal_id
-        union all
-        select
-            object_id,
-            table_name,
-            schema_name,
-            principal_name,
-            table_type
-        from
-            views_with_metadata
-        join principals on views_with_metadata.owner_principal_id = principals.principal_id
-    ),
-
-    cols as (
+        with
+        principals as (
+            select
+                name as principal_name,
+                principal_id as principal_id
+            from
+                sys.database_principals {{ information_schema_hints() }}
+        ),
+
+        schemas as (
+            select
+                name as schema_name,
+                schema_id as schema_id,
+                principal_id as principal_id
+            from
+                sys.schemas {{ information_schema_hints() }}
+        ),
+
+        tables as (
+            select
+                object_id,
+                name as table_name,
+                schema_id as schema_id,
+                principal_id as principal_id,
+                'BASE TABLE' as table_type
+            from
+                sys.tables {{ information_schema_hints() }}
+        ),
+
+        tables_with_metadata as (
+            select
+                object_id,
+                table_name,
+                schema_name,
+                coalesce(tables.principal_id, schemas.principal_id) as owner_principal_id,
+                table_type
+            from
+                tables
+            join schemas on tables.schema_id = schemas.schema_id
+        ),
+
+        views as (
+            select
+                object_id,
+                name as table_name,
+                schema_id as schema_id,
+                principal_id as principal_id,
+                'VIEW' as table_type
+            from
+                sys.views {{ information_schema_hints() }}
+        ),
+
+        views_with_metadata as (
+            select
+                object_id,
+                table_name,
+                schema_name,
+                coalesce(views.principal_id, schemas.principal_id) as owner_principal_id,
+                table_type
+            from
+                views
+            join schemas on views.schema_id = schemas.schema_id
+        ),
+
+        tables_and_views as (
+            select
+                object_id,
+                table_name,
+                schema_name,
+                principal_name,
+                table_type
+            from
+                tables_with_metadata
+            join principals on tables_with_metadata.owner_principal_id = principals.principal_id
+            union all
+            select
+                object_id,
+                table_name,
+                schema_name,
+                principal_name,
+                table_type
+            from
+                views_with_metadata
+            join principals on views_with_metadata.owner_principal_id = principals.principal_id
+        ),
+
+        cols as (
+
+            select
+                c.object_id,
+                c.name as column_name,
+                c.column_id as column_index,
+                t.name as column_type
+            from sys.columns as c {{ information_schema_hints() }}
+            left join sys.types as t on c.system_type_id = t.system_type_id {{ information_schema_hints() }}
+        )
 
         select
-            c.object_id,
-            c.name as column_name,
-            c.column_id as column_index,
-            t.name as column_type
-        from sys.columns as c {{ information_schema_hints() }}
-        left join sys.types as t on c.system_type_id = t.system_type_id {{ information_schema_hints() }}
-    )
-
-    select
-        DB_NAME() as table_database,
-        tv.schema_name as table_schema,
-        tv.table_name,
-        tv.table_type,
-        null as table_comment,
-        tv.principal_name as table_owner,
-        cols.column_name,
-        cols.column_index,
-        cols.column_type,
-        null as column_comment
-    from tables_and_views tv
-    join cols on tv.object_id = cols.object_id
-    where ({%- for schema in schemas -%}
-        upper(tv.schema_name) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%}
-    {%- endfor -%})
-
-    order by column_index
-
-    {%- endcall -%}
-
-  {{ return(load_result('catalog').table) }}
+            DB_NAME() as table_database,
+            tv.schema_name as table_schema,
+            tv.table_name,
+            tv.table_type,
+            null as table_comment,
+            tv.principal_name as table_owner,
+            cols.column_name,
+            cols.column_index,
+            cols.column_type,
+            null as column_comment
+        from tables_and_views tv
+        join cols on tv.object_id = cols.object_id
+        where ({%- for schema in schemas -%}
+            upper(tv.schema_name) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%}
+        {%- endfor -%})
+
+        order by column_index
+        {{ query_label }}
+
+        {%- endcall -%}
+
+    {{ return(load_result('catalog').table) }}
 
 {%- endmacro %}
 
 {% macro fabric__get_catalog_relations(information_schema, relations) -%}
-
+    {% set query_label = apply_label() %}
     {%- call statement('catalog', fetch_result=True) -%}
 
     with
@@ -260,7 +261,7 @@
     )
 
     order by column_index
-
+    {{ query_label }}
     {%- endcall -%}
 
   {{ return(load_result('catalog').table) }}

dbt/include/fabric/macros/adapters/columns.sql

@@ -11,42 +11,46 @@
 {% endmacro %}
 
 {% macro fabric__get_columns_in_relation(relation) -%}
-  {% call statement('get_columns_in_relation', fetch_result=True) %}
+    {% set query_label = apply_label() %}
+    {% call statement('get_columns_in_relation', fetch_result=True) %}
+
+        with mapping as (
+            select
+                row_number() over (partition by object_name(c.object_id) order by c.column_id) as ordinal_position,
+                c.name collate database_default as column_name,
+                t.name as data_type,
+                c.max_length as character_maximum_length,
+                c.precision as numeric_precision,
+                c.scale as numeric_scale
+            from [{{ 'tempdb' if '#' in relation.identifier else relation.database }}].sys.columns c {{ information_schema_hints() }}
+            inner join sys.types t {{ information_schema_hints() }}
+            on c.user_type_id = t.user_type_id
+            where c.object_id = object_id('{{ 'tempdb..' ~ relation.include(database=false, schema=false) if '#' in relation.identifier else relation }}')
+        )
 
-    with mapping as (
         select
-            row_number() over (partition by object_name(c.object_id) order by c.column_id) as ordinal_position,
-            c.name collate database_default as column_name,
-            t.name as data_type,
-            c.max_length as character_maximum_length,
-            c.precision as numeric_precision,
-            c.scale as numeric_scale
-        from [{{ 'tempdb' if '#' in relation.identifier else relation.database }}].sys.columns c {{ information_schema_hints() }}
-        inner join sys.types t {{ information_schema_hints() }}
-        on c.user_type_id = t.user_type_id
-        where c.object_id = object_id('{{ 'tempdb..' ~ relation.include(database=false, schema=false) if '#' in relation.identifier else relation }}')
-    )
-
-    select
-        column_name,
-        data_type,
-        character_maximum_length,
-        numeric_precision,
-        numeric_scale
-    from mapping
-    order by ordinal_position
-
-  {% endcall %}
-  {% set table = load_result('get_columns_in_relation').table %}
-  {{ return(sql_convert_columns_in_relation(table)) }}
+            column_name,
+            data_type,
+            character_maximum_length,
+            numeric_precision,
+            numeric_scale
+        from mapping
+        order by ordinal_position
+        {{ query_label }}
+
+    {% endcall %}
+    {% set table = load_result('get_columns_in_relation').table %}
+    {{ return(sql_convert_columns_in_relation(table)) }}
 {% endmacro %}
 
 {% macro fabric__get_columns_in_query(select_sql) %}
+    {% set query_label = apply_label() %}
     {% call statement('get_columns_in_query', fetch_result=True, auto_begin=False) -%}
         select TOP 0 * from (
             {{ select_sql }}
         ) as __dbt_sbq
         where 0 = 1
+        {{ query_label }}
     {% endcall %}
 
     {{ return(load_result('get_columns_in_query').table.columns | map(attribute='name') | list) }}
@@ -84,6 +88,7 @@
     {% set tempTable %}
         CREATE TABLE {{tempTableName}}
         AS SELECT {{query_result_text}}, CAST({{ column_name }} AS {{new_column_type}}) AS {{column_name}} FROM {{ relation.schema }}.{{ relation.identifier }}
+        {{ apply_label() }}
     {% endset %}
 
     {% call statement('create_temp_table') -%}
@@ -100,7 +105,7 @@
 
     {% set createTable %}
         CREATE TABLE {{ relation.schema }}.{{ relation.identifier }}
-        AS SELECT * FROM {{tempTableName}}
+        AS SELECT * FROM {{tempTableName}} {{ apply_label() }}
     {% endset %}
 
     {% call statement('create_Table') -%}