Add Microsoft Fabric Data Warehouse support (+ SQL Server and Synapse)

dbt_project.yml

@@ -26,17 +26,17 @@ dispatch:
 
 models:
   dbt_project_evaluator:
-    +materialized: "{{ 'table' if target.type in ['duckdb'] else 'view' }}"
+    +materialized: "{{ 'table' if target.type in ['duckdb', 'fabric'] else 'view' }}"
     marts:
       core:
         int_all_graph_resources:
           +materialized: table   
         int_direct_relationships:
-          # required for BigQuery and Redshift for performance/memory reasons
-          +materialized: "{{ 'table' if target.type in ['bigquery', 'redshift', 'databricks'] else 'view' }}"
+          # required for BigQuery, Redshift, Databricks, and Fabric for performance/memory reasons
+          +materialized: "{{ 'table' if target.type in ['bigquery', 'redshift', 'databricks', 'fabric'] else 'view' }}"
         int_all_dag_relationships:
-          # required for BigQuery, Redshift, and Databricks for performance/memory reasons
-          +materialized: "{{ 'table' if target.type in ['bigquery', 'redshift', 'databricks', 'clickhouse'] else 'view' }}"
+          # required for BigQuery, Redshift, Databricks, Clickhouse, and Fabric for performance/memory reasons
+          +materialized: "{{ 'table' if target.type in ['bigquery', 'redshift', 'databricks', 'clickhouse', 'fabric'] else 'view' }}"
       dag:
         +materialized: table
     staging:
@@ -45,11 +45,11 @@ models:
           +materialized: table
       variables:
         stg_naming_convention_folders:
-          # required for Redshift because listagg runs only on tables
-          +materialized: "{{ 'table' if target.type == 'redshift' else 'view' }}"
+          # required for Redshift and Fabric because listagg runs only on tables
+          +materialized: "{{ 'table' if target.type in ['redshift', 'fabric'] else 'view' }}"
         stg_naming_convention_prefixes:
-          # required for Redshift because listagg runs only on tables
-          +materialized: "{{ 'table' if target.type == 'redshift' else 'view' }}"
+          # required for Redshift and Fabric because listagg runs only on tables
+          +materialized: "{{ 'table' if target.type in ['redshift', 'fabric'] else 'view' }}"
 
 
 vars:
@@ -89,7 +89,7 @@ vars:
 
   # -- Execution variables --
   insert_batch_size: "{{ 500 if target.type in ['athena', 'bigquery'] else 10000 }}"
-  max_depth_dag: "{{ 9 if target.type in ['bigquery', 'spark', 'databricks'] else 4 if target.type in ['athena', 'trino', 'clickhouse'] else -1 }}"
+  max_depth_dag: "{{ 9 if target.type in ['bigquery', 'spark', 'databricks', 'fabric'] else 4 if target.type in ['athena', 'trino', 'clickhouse'] else -1 }}"
 
   # -- Code complexity variables --
   comment_chars: ["--"]

macros/cross_db_shim/bool_literal.sql

@@ -0,0 +1,8 @@
+{# Convert a Python boolean to a SQL boolean literal appropriate for the target adapter #}
+{% macro bool_literal(value) %}
+    {{ return(adapter.dispatch('bool_literal', 'dbt_project_evaluator')(value)) }}
+{% endmacro %}
+
+{% macro default__bool_literal(value) %}{{ value | trim }}{% endmacro %}
+
+{% macro fabric__bool_literal(value) %}{% if value %}1{% else %}0{% endif %}{% endmacro %}

macros/cross_db_shim/escape_single_quotes.sql

@@ -0,0 +1,7 @@
+{% macro spark__escape_single_quotes(expression) -%}
+    {{ expression | replace("'","\\'") }}
+{%- endmacro %}
+
+{% macro fabric__escape_single_quotes(expression) -%}
+    {{ expression | replace("'","''") }}
+{%- endmacro %}

macros/cross_db_shim/quote_identifier.sql

@@ -0,0 +1,7 @@
+{% macro quote_identifier(name) %}
+    {{ return(adapter.dispatch('quote_identifier', 'dbt_project_evaluator')(name)) }}
+{% endmacro %}
+
+{% macro default__quote_identifier(name) %}{{ name }}{% endmacro %}
+
+{% macro fabric__quote_identifier(name) %}[{{ name }}]{% endmacro %}

macros/cross_db_shim/type_string.sql

@@ -9,3 +9,7 @@
 {%- macro redshift__type_string_dpe() -%}
     {{ return(api.Column.string_type(600)) }}
 {%- endmacro -%}
+
+{%- macro fabric__type_string_dpe() -%}
+    {{ return("varchar(8000)") }}
+{%- endmacro -%}

macros/get_directory_pattern.sql

@@ -24,11 +24,15 @@
 
 {% macro get_dbtreplace_directory_pattern() %}
   {% if execute %}
-    {%- set on_mac_or_linux = dbt_project_evaluator.is_os_mac_or_linux() -%}
-    {%- if on_mac_or_linux -%}
-      {{ dbt.replace("file_path", "regexp_replace(file_path,'.*/','')", "''") }}
-    {% else %}
-      {{ dbt.replace("file_path", "regexp_replace(file_path,'.*\\\\\\\\','')", "''") }}
-    {% endif %}
+    {%- if target.type == 'fabric' -%}
+      left(file_path, len(file_path) - charindex('/', reverse(file_path)))
+    {%- else -%}
+      {%- set on_mac_or_linux = dbt_project_evaluator.is_os_mac_or_linux() -%}
+      {%- if on_mac_or_linux -%}
+        {{ dbt.replace("file_path", "regexp_replace(file_path,'.*/','')", "''") }}
+      {% else %}
+        {{ dbt.replace("file_path", "regexp_replace(file_path,'.*\\\\\\\\','')", "''") }}
+      {% endif %}
+    {%- endif -%}
   {% endif %}
-{% endmacro %} 
+{% endmacro %}

macros/is_not_empty_string.sql

@@ -10,4 +10,8 @@
     {{ false }}
     {% endif %}
 
+{% endmacro %}
+
+{% macro fabric__is_not_empty_string(str) %}
+    {% if str %}1{% else %}0{% endif %}
 {% endmacro %}

macros/recursive_dag.sql

@@ -257,6 +257,125 @@ with direct_relationships as (
 {% endmacro %}
 
 
+{% macro fabric__recursive_dag() %}
+
+{% set max_depth = var('max_depth_dag') | int %}
+{% if max_depth < 2 or max_depth < var('chained_views_threshold') | int %}
+    {% do exceptions.raise_compiler_error(
+        'Variable max_depth_dag must be at least 2 and must be greater or equal to than chained_views_threshold.'
+        ) %}
+{% endif %}
+
+with direct_relationships as (
+    select
+        *
+    from {{ ref('int_direct_relationships') }}
+    where resource_type <> 'test'
+)
+
+, get_distinct as (
+    select distinct
+        resource_id as parent_id,
+        resource_id as child_id,
+        resource_name,
+        materialized as child_materialized,
+        is_public as child_is_public,
+        access as child_access,
+        is_excluded as child_is_excluded
+
+    from direct_relationships
+)
+
+, cte_0 as (
+    select
+        parent_id,
+        child_id,
+        child_materialized,
+        child_is_public,
+        child_access,
+        child_is_excluded,
+        0 as distance,
+        cast({{ dbt.array_construct(['resource_name']) }} as varchar(max)) as path,
+        cast(null as {{ dbt.type_boolean() }}) as is_dependent_on_chain_of_views
+    from get_distinct
+)
+
+{% for i in range(1, max_depth) %}
+{% set prev_cte_path %}cte_{{ i - 1 }}.path{% endset %}
+, cte_{{ i }} as (
+    select
+        cte_{{ i - 1 }}.parent_id as parent_id,
+        direct_relationships.resource_id as child_id,
+        direct_relationships.materialized as child_materialized,
+        direct_relationships.is_public as child_is_public,
+        direct_relationships.access as child_access,
+        direct_relationships.is_excluded as child_is_excluded,
+        cte_{{ i - 1 }}.distance+1 as distance,
+        cast({{ dbt.array_append(prev_cte_path, 'direct_relationships.resource_name') }} as varchar(max)) as path,
+        case
+            when
+                cte_{{ i - 1 }}.child_materialized in ('view', 'ephemeral')
+                and coalesce(cte_{{ i - 1 }}.is_dependent_on_chain_of_views, cast(1 as bit)) = cast(1 as bit)
+                then cast(1 as bit)
+            else cast(0 as bit)
+        end as is_dependent_on_chain_of_views
+
+        from direct_relationships
+            inner join cte_{{ i - 1 }}
+            on cte_{{ i - 1 }}.child_id = direct_relationships.direct_parent_id
+)
+{% endfor %}
+
+, all_relationships_unioned as (
+    {% for i in range(max_depth) %}
+    select * from cte_{{ i }}
+    {% if not loop.last %}union all{% endif %}
+    {% endfor %}
+)
+
+, resource_info as (
+    select * from {{ ref('int_all_graph_resources') }}
+)
+
+, all_relationships as (
+    select
+        parent.resource_id as parent_id,
+        parent.resource_name as parent,
+        parent.resource_type as parent_resource_type,
+        parent.model_type as parent_model_type,
+        parent.materialized as parent_materialized,
+        parent.is_public as parent_is_public,
+        parent.access as parent_access,
+        parent.source_name as parent_source_name,
+        parent.file_path as parent_file_path,
+        parent.directory_path as parent_directory_path,
+        parent.file_name as parent_file_name,
+        parent.is_excluded as parent_is_excluded,
+        child.resource_id as child_id,
+        child.resource_name as child,
+        child.resource_type as child_resource_type,
+        child.model_type as child_model_type,
+        child.materialized as child_materialized,
+        child.is_public as child_is_public,
+        child.access as child_access,
+        child.source_name as child_source_name,
+        child.file_path as child_file_path,
+        child.directory_path as child_directory_path,
+        child.file_name as child_file_name,
+        child.is_excluded as child_is_excluded,
+        cast(all_relationships_unioned.distance as {{ dbt.type_int() }}) as distance,
+        all_relationships_unioned.path,
+        case when all_relationships_unioned.is_dependent_on_chain_of_views = cast(1 as bit) then cast(1 as bit) else cast(0 as bit) end as is_dependent_on_chain_of_views
+    from all_relationships_unioned
+    left join resource_info as parent
+        on all_relationships_unioned.parent_id = parent.resource_id
+    left join resource_info as child
+        on all_relationships_unioned.child_id = child.resource_id
+)
+
+{% endmacro %}
+
+
 {% macro clickhouse__recursive_dag() %}
     {{ return(bigquery__recursive_dag()) }}
 {% endmacro %}

macros/unpack/get_column_values.sql

@@ -25,7 +25,7 @@
                         wrap_string_with_quotes(dbt.escape_single_quotes(column.description | replace("\\","\\\\"))),
                         wrap_string_with_quotes(dbt.escape_single_quotes(column.data_type)),
                         wrap_string_with_quotes(dbt.escape_single_quotes(tojson(column.constraints))),
-                        column.constraints | selectattr('type', 'equalto', 'not_null') | list | length > 0,
+                        "cast(" ~ dbt_project_evaluator.bool_literal(column.constraints | selectattr('type', 'equalto', 'not_null') | list | length > 0) | trim ~ " as " ~ dbt.type_boolean() ~ ")",
                         column.constraints | length,
                         wrap_string_with_quotes(dbt.escape_single_quotes(column.quote))
                     ]

macros/unpack/get_node_values.sql

@@ -23,15 +23,15 @@
                 wrap_string_with_quotes(node.name),
                 wrap_string_with_quotes(node.resource_type),
                 wrap_string_with_quotes(node.original_file_path | replace("\\","\\\\")),
-                "cast(" ~ node.config.enabled | trim ~ " as " ~ dbt.type_boolean() ~ ")",
+                "cast(" ~ dbt_project_evaluator.bool_literal(node.config.enabled) | trim ~ " as " ~ dbt.type_boolean() ~ ")",
                 wrap_string_with_quotes(node.config.materialized),
                 wrap_string_with_quotes(node.config.on_schema_change),
                 wrap_string_with_quotes(node.group),
                 wrap_string_with_quotes(node.access),
                 wrap_string_with_quotes(node.latest_version),
                 wrap_string_with_quotes(node.version),
                 wrap_string_with_quotes(node.deprecation_date),
-                "cast(" ~ contract | trim  ~ " as " ~ dbt.type_boolean() ~ ")",
+                "cast(" ~ dbt_project_evaluator.bool_literal(contract) | trim ~ " as " ~ dbt.type_boolean() ~ ")",
                 node.columns.values() | list | length,
                 node.columns.values() | list | selectattr('description') | list | length,
                 wrap_string_with_quotes(node.database),
@@ -46,7 +46,7 @@
                 sql_complexity,
                 wrap_string_with_quotes(node.get('depends_on',{}).get('macros',[]) | tojson),
                 "cast(" ~ dbt_project_evaluator.is_not_empty_string(node.test_metadata) | trim ~ " as " ~ dbt.type_boolean() ~ ")",
-                "cast(" ~ exclude_node ~ " as " ~ dbt.type_boolean() ~ ")",
+                "cast(" ~ dbt_project_evaluator.bool_literal(exclude_node) | trim ~ " as " ~ dbt.type_boolean() ~ ")",
             ]
         %}
 

macros/unpack/get_relationship_values.sql

@@ -21,12 +21,12 @@
 
             {%- if node.get('depends_on',{}).get('nodes',[]) |length == 0 -%}
 
-                {%- set values_line = 
+                {%- set values_line =
                   [
                     "cast('" ~ node.unique_id ~ "' as " ~ dbt_project_evaluator.type_string_dpe() ~ ")",
                     "cast(NULL as " ~ dbt_project_evaluator.type_string_dpe() ~ ")",
-                    "FALSE",
-                  ] 
+                    "cast(" ~ dbt_project_evaluator.bool_literal(false) | trim ~ " as " ~ dbt.type_boolean() ~ ")",
+                  ]
                 %}
 
                 {%- do values.append(values_line) -%}
@@ -42,7 +42,7 @@
                         [
                             "cast('" ~ node.unique_id ~ "' as " ~ dbt_project_evaluator.type_string_dpe() ~ ")",
                             "cast('" ~ parent ~ "' as " ~ dbt_project_evaluator.type_string_dpe() ~ ")",
-                            "" ~ is_primary ~ "" if node.unique_id.split('.')[0] == 'test' else "FALSE"
+                            "cast(" ~ dbt_project_evaluator.bool_literal(is_primary if node.unique_id.split('.')[0] == 'test' else false) | trim ~ " as " ~ dbt.type_boolean() ~ ")"
                         ]
                     %}
 

macros/unpack/get_source_values.sql

@@ -22,23 +22,23 @@
               wrap_string_with_quotes(node.source_name),
               "cast(" ~ dbt_project_evaluator.is_not_empty_string(node.source_description) | trim ~ " as " ~ dbt.type_boolean() ~ ")",
               "cast(" ~ dbt_project_evaluator.is_not_empty_string(node.description) | trim ~ " as " ~ dbt.type_boolean() ~ ")",
-              "cast(" ~ node.config.enabled ~ " as " ~ dbt.type_boolean() ~ ")",
+              "cast(" ~ dbt_project_evaluator.bool_literal(node.config.enabled) | trim ~ " as " ~ dbt.type_boolean() ~ ")",
               wrap_string_with_quotes(node.loaded_at_field | replace("'", "_")),
 
-              "cast(" ~ (
-                ((node.config.freshness != None) and (dbt_project_evaluator.is_not_empty_string(node.config.freshness.warn_after.count) 
-                  or dbt_project_evaluator.is_not_empty_string(node.config.freshness.error_after.count))) 
-                or ((node.freshness != None) and (dbt_project_evaluator.is_not_empty_string(node.freshness.warn_after.count) 
+              "cast(" ~ dbt_project_evaluator.bool_literal(
+                ((node.config.freshness != None) and (dbt_project_evaluator.is_not_empty_string(node.config.freshness.warn_after.count)
+                  or dbt_project_evaluator.is_not_empty_string(node.config.freshness.error_after.count)))
+                or ((node.freshness != None) and (dbt_project_evaluator.is_not_empty_string(node.freshness.warn_after.count)
                   or dbt_project_evaluator.is_not_empty_string(node.freshness.error_after.count)))
-                ) | trim ~ " as boolean)",
+                ) | trim ~ " as " ~ dbt.type_boolean() ~ ")",
 
               wrap_string_with_quotes(node.database),
               wrap_string_with_quotes(node.schema),
               wrap_string_with_quotes(node.package_name),
               wrap_string_with_quotes(node.loader),
               wrap_string_with_quotes(node.identifier),
               wrap_string_with_quotes(node.meta | tojson),
-              "cast(" ~ exclude_source ~ " as " ~ dbt.type_boolean() ~ ")",
+              "cast(" ~ dbt_project_evaluator.bool_literal(exclude_source) | trim ~ " as " ~ dbt.type_boolean() ~ ")",
             ]
         %}
 

models/marts/core/int_all_graph_resources.sql

@@ -47,12 +47,16 @@ unioned_with_calc as (
         end as resource_name,
         case
             when resource_type = 'source' then null
-            else {{ dbt.split_part('name', "'_'", 1) }}||'_' 
+            else {{ dbt.concat([dbt.split_part('name', "'_'", 1), "'_'"]) }}
         end as prefix,
         {{ get_dbtreplace_directory_pattern() }} as directory_path,
+        {% if target.type == 'fabric' %}
+        right(file_path, charindex('/', reverse(file_path)) - 1) as file_name
+        {% else %}
         regexp_replace(file_path,'.*{{ get_regexp_directory_pattern() }}','') as file_name
+        {% endif %}
     from unioned
-    where coalesce(is_enabled, True) = True and package_name != 'dbt_project_evaluator'
+    where coalesce(is_enabled, cast(1 as {{ dbt.type_boolean() }})) = cast(1 as {{ dbt.type_boolean() }}) and package_name != 'dbt_project_evaluator'
 ), 
 
 joined as (
@@ -78,19 +82,19 @@ joined as (
         {{ dbt.position(dbt.concat([quoted_directory_pattern, 'naming_convention_folders.folder_name_value', quoted_directory_pattern]),'unioned_with_calc.directory_path') }} as position_folder,  
         nullif(unioned_with_calc.column_name, '') as column_name,
         {% for test in test_macro_set %}
-        unioned_with_calc.macro_dependencies like '%macro.{{ test }}%' and unioned_with_calc.resource_type = 'test' as is_{{ test.split('.')[1] }},  
+        case when unioned_with_calc.macro_dependencies like '%macro.{{ test }}%' and unioned_with_calc.resource_type = 'test' then cast(1 as {{ dbt.type_boolean() }}) else cast(0 as {{ dbt.type_boolean() }}) end as is_{{ test.split('.')[1] }},
         {% endfor %}
         unioned_with_calc.is_enabled, 
         unioned_with_calc.materialized, 
         unioned_with_calc.on_schema_change, 
-        unioned_with_calc.database, 
-        unioned_with_calc.schema, 
+        unioned_with_calc.{{ dbt_project_evaluator.quote_identifier('database') }},
+        unioned_with_calc.{{ dbt_project_evaluator.quote_identifier('schema') }},
         unioned_with_calc.package_name, 
         unioned_with_calc.alias, 
         unioned_with_calc.is_described, 
         unioned_with_calc.model_group, 
         unioned_with_calc.access, 
-        unioned_with_calc.access = 'public' as is_public, 
+        case when unioned_with_calc.access = 'public' then cast(1 as {{ dbt.type_boolean() }}) else cast(0 as {{ dbt.type_boolean() }}) end as is_public,
         unioned_with_calc.latest_version, 
         unioned_with_calc.version, 
         unioned_with_calc.deprecation_date, 

models/marts/core/int_direct_relationships.sql

@@ -64,10 +64,7 @@ direct_relationships as (
             when all_graph_resources.resource_type in ('model', 'snapshot', 'test') then models.direct_parent_id
             else null
         end as direct_parent_id,
-        (
-            all_graph_resources.resource_type = 'test'
-            and models.is_primary_relationship
-        ) as is_primary_test_relationship
+        case when all_graph_resources.resource_type = 'test' and models.is_primary_relationship = cast(1 as {{ dbt.type_boolean() }}) then cast(1 as {{ dbt.type_boolean() }}) else cast(0 as {{ dbt.type_boolean() }}) end as is_primary_test_relationship
     from all_graph_resources
     left join direct_model_relationships as models
         on all_graph_resources.resource_id = models.resource_id