Merge pull request #265 from dbt-labs/insert-materialization-for-large-projects

Insert materialization for large projects

Author: dave-connors-3

README.md

@@ -23,18 +23,18 @@ Currently, the following adapters are supported:
 ### Cloning via dbt Package Hub
 
 Check [dbt Hub](https://hub.getdbt.com/dbt-labs/dbt_project_evaluator/latest/) for the latest installation instructions, or [read the docs](https://docs.getdbt.com/docs/package-management) for more information on installing packages.
-### Additional setup for Databricks/Spark
+### Additional setup for Databricks/Spark/DuckDB/Redshift
 
 In your `dbt_project.yml`, add the following config:
 ```yml
 # dbt_project.yml
 
 dispatch:
-  - macro_namespace: dbt_utils
-    search_order: ['dbt_project_evaluator', 'spark_utils', 'dbt_utils']
+  - macro_namespace: dbt
+    search_order: ['dbt_project_evaluator', 'dbt']
 ```
 
-This is required because the project currently provides limited support for arrays macros for Databricks/Spark which is not part of `spark_utils` yet.
+This is required because the project currently overrides a small number of dbt core macros in order to ensure the project can run across the listed adapters. The overridden macros are in the [cross_db_shim directory](macros/cross_db_shim/). 
   
 ### How It Works
 
@@ -1055,14 +1055,17 @@ vars:
 | variable    | description | default     |
 | ----------- | ----------- | ----------- |
 | `chained_views_threshold` | threshold for unacceptable length of chain of views for `fct_chained_views_dependencies` | 4 |
+| `insert_batch_size` | number of records inserted per batch when unpacking the graph into models | 10000 |
 
 ```yml
 # dbt_project.yml
-# set your chained views threshold to 8 instead of 4
 
 vars:
   dbt_project_evaluator:
+    # set your chained views threshold to 8 instead of 4
     chained_views_threshold: 8
+    # update the number of records inserted from the graph from 10,000 to 500 to reduce query size
+    insert_batch_size: 500
 ```
 </details>
 

dbt_project.yml

@@ -78,6 +78,7 @@ vars:
 
   # -- Performance variables --
   chained_views_threshold: 5
+  insert_batch_size: "{{ 500 if target.type == 'bigquery' else 10000 }}"
 
   # -- Warehouse specific variables --
   max_depth_dag: 9

integration_tests/dbt_project.yml

@@ -63,4 +63,5 @@ vars:
   # dummy variable used for testing fct_hard_coded_references
   my_table_reference: 'grace_table'
   new_model_type_folder_name: 'my_new_models'
-  new_model_type_prefixes: 'nwmdl_'
+  new_model_type_prefixes: 'nwmdl_'
+  insert_batch_size: 100

integration_tests_2/README.md

@@ -6,3 +6,4 @@ Currently, this project is used to test the package behavior when:
 - there is no exposure
 - there is no metric
 - people don't override the default seed for `dbt_project_evaluator_exceptions`
+- people don't override the default value of `insert_batch_size`

macros/cross_db_shim/redshift_shims.sql

@@ -0,0 +1,3 @@
+{%- macro redshift__type_string() -%}
+  {{ "VARCHAR(600)" }}
+{%- endmacro %}

macros/insert_resources_from_graph.sql

@@ -0,0 +1,21 @@
+{% macro insert_resources_from_graph(relation, resource_type='nodes', relationships=False, batch_size=var('insert_batch_size') | int) %}
+  {%- set values = get_resource_values(resource_type, relationships) -%}
+  {%- set values_length = values | length -%}
+  {%- set loop_count = (values_length / batch_size) | round(0, 'ceil') | int -%}
+  
+    {%- for loop_number in range(loop_count) -%}
+        {%- set lower_bound = loop.index0 * batch_size -%}
+        {%- set upper_bound = loop.index * batch_size -%}
+        {%- set values_subset = values[lower_bound : upper_bound] %}
+        {%- set values_list_of_strings = [] -%}
+        {%- for indiv_values in values_subset %}
+            {%- do values_list_of_strings.append( indiv_values | join(", \n")) -%}
+        {%- endfor -%}
+        {%- set values_string = '(' ~ values_list_of_strings | join("), \n\n(") ~ ')' %}
+        {%- set insert_statement = "insert into " ~ relation ~ " values \n" ~  values_string ~ ";"%}
+        {% call statement('insert') -%}
+            {{ insert_statement }}
+        {%- endcall %}
+    {% endfor %}
+    
+{% endmacro %}

macros/select_from_values.sql

@@ -1,8 +1,8 @@
-{%- macro get_exposures() -%}
-    {{ return(adapter.dispatch('get_exposures', 'dbt_project_evaluator')()) }}
+{%- macro get_exposure_values() -%}
+    {{ return(adapter.dispatch('get_exposure_values', 'dbt_project_evaluator')()) }}
 {%- endmacro -%}
 
-{%- macro default__get_exposures() -%}
+{%- macro default__get_exposure_values() -%}
 
     {%- if execute -%}
 
@@ -33,24 +33,6 @@
     {%- endfor -%}
     {%- endif -%}
 
-    {{ return(
-        dbt_project_evaluator.select_from_values(
-            values = values,
-            columns = [
-              'unique_id', 
-              'name', 
-              'resource_type',
-              'file_path', 
-              ('is_described', 'boolean'),
-              'exposure_type', 
-              'maturity', 
-              'package_name', 
-              'url',
-              'owner_name',
-              'owner_email',
-              'meta'
-            ]
-         )
-    ) }}
+    {{ return(values) }}
 
 {%- endmacro -%}

macros/unpack/get_exposures.sql macros/unpack/get_exposure_values.sqlmacros/unpack/get_exposures.sql renamed to macros/unpack/get_exposure_values.sql

@@ -1,8 +1,8 @@
-{%- macro get_metrics() -%}
-    {{ return(adapter.dispatch('get_metrics', 'dbt_project_evaluator')()) }}
+{%- macro get_metric_values() -%}
+    {{ return(adapter.dispatch('get_metric_values', 'dbt_project_evaluator')()) }}
 {%- endmacro -%}
 
-{%- macro default__get_metrics() -%}
+{%- macro default__get_metric_values() -%}
 
     {%- if execute -%}
     {%- set nodes_list = graph.metrics.values() -%}
@@ -45,26 +45,6 @@
     {%- endfor -%}
     {%- endif -%}
 
-    {{ return(
-        dbt_project_evaluator.select_from_values(
-            values = values,
-            columns = [
-              'unique_id', 
-              'name', 
-              'resource_type', 
-              'file_path', 
-              ('is_described', 'boolean'),
-              'metric_type', 
-              'model',
-              'label', 
-              'sql', 
-              'timestamp', 
-              'package_name',
-              'dimensions',
-              'filters',
-              'meta'
-            ]
-         )
-    ) }}
+    {{ return(values) }}
 
 {%- endmacro -%}

macros/unpack/get_metrics.sql macros/unpack/get_metric_values.sqlmacros/unpack/get_metrics.sql renamed to macros/unpack/get_metric_values.sql

@@ -1,8 +1,8 @@
-{%- macro get_nodes() -%}
-    {{ return(adapter.dispatch('get_nodes', 'dbt_project_evaluator')()) }}
+{%- macro get_node_values() -%}
+    {{ return(adapter.dispatch('get_node_values', 'dbt_project_evaluator')()) }}
 {%- endmacro -%}
 
-{%- macro default__get_nodes() -%}
+{%- macro default__get_node_values() -%}
 
     {%- if execute -%}
     {%- set nodes_list = graph.nodes.values() -%}
@@ -39,29 +39,6 @@
     {%- endfor -%}
     {%- endif -%}
 
-    {{ return(
-        dbt_project_evaluator.select_from_values(
-            values = values,
-            columns = [
-              'unique_id',
-              'name',
-              'resource_type',
-              'file_path',
-              ('is_enabled', 'boolean'),
-              'materialized',
-              'on_schema_change',
-              'database',
-              'schema',
-              'package_name',
-              'alias',
-              ('is_described', 'boolean'),
-              'column_name',
-              'meta',
-              'hard_coded_references',
-              'macro_dependencies',
-              ('is_generic_test', 'boolean')
-            ]
-         )
-    ) }}
+    {{ return(values) }}
 
 {%- endmacro -%}