Merge pull request #576 from sdebruyn/feature/fabric-support

Add Microsoft Fabric Data Warehouse support (+ SQL Server and Synapse)

Author: b-per

README.md

@@ -24,14 +24,16 @@ Currently, the following adapters are supported:
 - AWS Athena (tested manually)
 - Greenplum (tested manually)
 - ClickHouse (tested manually)
+- Microsoft Fabric Data Warehouse (tested manually)
+- Microsoft Fabric Spark (tested manually)
 
 ## Using This Package
 
 ### Cloning via dbt Package Hub
 
 Check [dbt Hub](https://hub.getdbt.com/dbt-labs/dbt_project_evaluator/latest/) for the latest installation instructions, or [read the docs](https://docs.getdbt.com/docs/package-management) for more information on installing packages.
 
-### Additional setup for Databricks/Spark/DuckDB/Redshift/ClickHouse
+### Additional setup for Databricks/Spark/DuckDB/Redshift/ClickHouse/Fabric
 
 In your `dbt_project.yml`, add the following config:
 

dbt_project.yml

@@ -26,17 +26,17 @@ dispatch:
 
 models:
   dbt_project_evaluator:
-    +materialized: "{{ 'table' if target.type in ['duckdb'] else 'view' }}"
+    +materialized: "{{ 'table' if target.type in ['duckdb', 'fabric'] else 'view' }}"
     marts:
       core:
         int_all_graph_resources:
           +materialized: table   
         int_direct_relationships:
-          # required for BigQuery and Redshift for performance/memory reasons
-          +materialized: "{{ 'table' if target.type in ['bigquery', 'redshift', 'databricks'] else 'view' }}"
+          # required for BigQuery, Redshift, Databricks, and Fabric for performance/memory reasons
+          +materialized: "{{ 'table' if target.type in ['bigquery', 'redshift', 'databricks', 'fabric'] else 'view' }}"
         int_all_dag_relationships:
-          # required for BigQuery, Redshift, and Databricks for performance/memory reasons
-          +materialized: "{{ 'table' if target.type in ['bigquery', 'redshift', 'databricks', 'clickhouse'] else 'view' }}"
+          # required for BigQuery, Redshift, Databricks, Clickhouse, and Fabric for performance/memory reasons
+          +materialized: "{{ 'table' if target.type in ['bigquery', 'redshift', 'databricks', 'clickhouse', 'fabric'] else 'view' }}"
       dag:
         +materialized: table
     staging:
@@ -45,11 +45,11 @@ models:
           +materialized: table
       variables:
         stg_naming_convention_folders:
-          # required for Redshift because listagg runs only on tables
-          +materialized: "{{ 'table' if target.type == 'redshift' else 'view' }}"
+          # required for Redshift and Fabric because listagg runs only on tables
+          +materialized: "{{ 'table' if target.type in ['redshift', 'fabric'] else 'view' }}"
         stg_naming_convention_prefixes:
-          # required for Redshift because listagg runs only on tables
-          +materialized: "{{ 'table' if target.type == 'redshift' else 'view' }}"
+          # required for Redshift and Fabric because listagg runs only on tables
+          +materialized: "{{ 'table' if target.type in ['redshift', 'fabric'] else 'view' }}"
 
 
 vars:
@@ -89,7 +89,7 @@ vars:
 
   # -- Execution variables --
   insert_batch_size: "{{ 500 if target.type in ['athena', 'bigquery'] else 10000 }}"
-  max_depth_dag: "{{ 9 if target.type in ['bigquery', 'spark', 'databricks'] else 4 if target.type in ['athena', 'trino', 'clickhouse'] else -1 }}"
+  max_depth_dag: "{{ 9 if target.type in ['bigquery', 'spark', 'databricks', 'fabric'] else 4 if target.type in ['athena', 'trino', 'clickhouse'] else -1 }}"
 
   # -- Code complexity variables --
   comment_chars: ["--"]

docs/customization/overriding-variables.md

@@ -103,14 +103,14 @@ vars:
 
 | variable    | description | default     |
 | ----------- | ----------- | ----------- |
-| `max_depth_dag` | limits the maximum distance between nodes calculated in `int_all_dag_relationships` | 9 for bigquery and spark, -1 for other adatpters |
+| `max_depth_dag` | limits the maximum distance between nodes calculated in `int_all_dag_relationships` | 9 for bigquery, spark, and fabric, -1 for other adapters |
 | `insert_batch_size` | number of records inserted per batch when unpacking the graph into models | 10000 |
 
 **Note on max_depth_dag**
 
 The default behavior for limiting the relationships calculated in the `int_all_dag_relationships` model differs depending on your adapter.
 
-- For Bigquery & Spark/Databricks the maximum distance between two nodes in your DAG, calculated in `int_all_dag_relationships`, is set by the `max_depth_dag` variable, which is defaulted to 9. So by default, `int_all_dag_relationships` contains a row for every path less than or equal to 9 nodes in length between two nodes in your DAG. This is because these adapters do not currently support recursive SQL, and queries often fail on more than 9 recursive joins.
+- For BigQuery, Spark/Databricks, and Microsoft Fabric Data Warehouse the maximum distance between two nodes in your DAG, calculated in `int_all_dag_relationships`, is set by the `max_depth_dag` variable, which is defaulted to 9. So by default, `int_all_dag_relationships` contains a row for every path less than or equal to 9 nodes in length between two nodes in your DAG. This is because these adapters do not currently support recursive SQL, and queries often fail on more than 9 recursive joins.
 - For all other adapters `int_all_dag_relationships` by default contains a row for every single path between two nodes in your DAG. If you experience long runtimes for the `int_all_dag_relationships` model, you may consider limiting the length of your generated DAG paths. To do this, set `max_depth_dag: {{ whatever limit you want to enforce }}`. The value of `max_depth_dag` must be greater than 2 for all DAG tests to work, and greater than `chained_views_threshold` to ensure your performance tests to work. By default, the value of this variable for these adapters is -1, which the package interprets as "no limit".
 
 ```yaml title="dbt_project.yml"

docs/index.md

@@ -25,14 +25,16 @@ Currently, the following adapters are supported:
 - AWS Athena (tested manually)
 - Greenplum (tested manually)
 - ClickHouse (tested manually)
+- Microsoft Fabric Data Warehouse (tested manually)
+- Microsoft Fabric Spark (tested manually)
 
 ## Using This Package
 
 ### Cloning via dbt Package Hub
 
 Check [dbt Hub](https://hub.getdbt.com/dbt-labs/dbt_project_evaluator/latest/) for the latest installation instructions, or [read the docs](https://docs.getdbt.com/docs/package-management) for more information on installing packages.
 
-### Additional setup for Databricks/Spark/DuckDB/Redshift
+### Additional setup for Databricks/Spark/DuckDB/Redshift/Fabric
 
 In your `dbt_project.yml`, add the following config:
 
@@ -64,8 +66,8 @@ Each test warning indicates the presence of a type of misalignment. To troublesh
 
 ## Limitations
 
-### BigQuery and Databricks
+### BigQuery, Databricks, and Microsoft Fabric Data Warehouse
 
-BigQuery current support for recursive CTEs is limited and Databricks SQL doesn't support recursive CTEs.
+BigQuery has limited support for recursive CTEs, while Databricks SQL and Microsoft Fabric Data Warehouse do not support them.
 
 For those Data Warehouses, the model `int_all_dag_relationships` needs to be created by looping CTEs instead. The number of loops is configured with `max_depth_dag` and defaulted to 9. This means that dependencies between models of more than 9 levels of separation won't show in the model `int_all_dag_relationships` but tests on the DAG will still be correct. With a number of loops higher than 9 BigQuery sometimes raises an error saying the query is too complex.

macros/cross_db_shim/bool_literal.sql

@@ -0,0 +1,8 @@
+{# Convert a Python boolean to a SQL boolean literal appropriate for the target adapter #}
+{% macro bool_literal(value) %}
+    {{ return(adapter.dispatch('bool_literal', 'dbt_project_evaluator')(value)) }}
+{% endmacro %}
+
+{% macro default__bool_literal(value) %}{{ value | trim }}{% endmacro %}
+
+{% macro fabric__bool_literal(value) %}{% if value %}1{% else %}0{% endif %}{% endmacro %}

macros/cross_db_shim/escape_single_quotes.sql

@@ -0,0 +1,7 @@
+{% macro spark__escape_single_quotes(expression) -%}
+    {{ expression | replace("'","\\'") }}
+{%- endmacro %}
+
+{% macro fabric__escape_single_quotes(expression) -%}
+    {{ expression | replace("'","''") }}
+{%- endmacro %}

macros/cross_db_shim/quote_identifier.sql

@@ -0,0 +1,7 @@
+{% macro quote_identifier(name) %}
+    {{ return(adapter.dispatch('quote_identifier', 'dbt_project_evaluator')(name)) }}
+{% endmacro %}
+
+{% macro default__quote_identifier(name) %}{{ name }}{% endmacro %}
+
+{% macro fabric__quote_identifier(name) %}[{{ name }}]{% endmacro %}

macros/cross_db_shim/spark_shims.sql

@@ -9,3 +9,7 @@
 {%- macro redshift__type_string_dpe() -%}
     {{ return(api.Column.string_type(600)) }}
 {%- endmacro -%}
+
+{%- macro fabric__type_string_dpe() -%}
+    {{ return("varchar(8000)") }}
+{%- endmacro -%}

macros/cross_db_shim/type_string.sql

@@ -23,6 +23,10 @@
 {% endmacro %}
 
 {% macro get_dbtreplace_directory_pattern() %}
+  {{ return(adapter.dispatch('get_dbtreplace_directory_pattern', 'dbt_project_evaluator')()) }}
+{% endmacro %}
+
+{% macro default__get_dbtreplace_directory_pattern() %}
   {% if execute %}
     {%- set on_mac_or_linux = dbt_project_evaluator.is_os_mac_or_linux() -%}
     {%- if on_mac_or_linux -%}
@@ -31,4 +35,15 @@
       {{ dbt.replace("file_path", "regexp_replace(file_path,'.*\\\\\\\\','')", "''") }}
     {% endif %}
   {% endif %}
-{% endmacro %} 
+{% endmacro %}
+
+{% macro fabric__get_dbtreplace_directory_pattern() %}
+  {% if execute %}
+    {%- set on_mac_or_linux = dbt_project_evaluator.is_os_mac_or_linux() -%}
+    {%- if on_mac_or_linux -%}
+      left(file_path, len(file_path) - charindex('/', reverse(file_path)))
+    {%- else -%}
+      left(file_path, len(file_path) - charindex('\', reverse(file_path)))
+    {%- endif -%}
+  {% endif %}
+{% endmacro %}