Merge pull request #573 from dbt-labs/feat/new-sl-spec-support

Author: b-per

.github/workflows/fusion.yml

@@ -26,7 +26,42 @@ env:
     PYTHON_VERSION: "3.11"
 
 jobs:
-    run-tests:
+    run-duckdb-tests:
+        runs-on: ubuntu-latest
+        permissions:
+            contents: read
+
+        steps:
+            - name: "Checkout PR code"
+              uses: actions/checkout@v4
+              with:
+                  ref: ${{ github.event.pull_request.head.sha || github.ref }}
+
+            - name: "Set up Python ${{ env.PYTHON_VERSION }}"
+              uses: actions/setup-python@v5
+              with:
+                  python-version: ${{ env.PYTHON_VERSION }}
+
+            - name: "Install dbt Fusion"
+              run: |
+                  curl -fsSL https://public.cdn.getdbt.com/fs/install/install.sh | sh
+                  echo "$HOME/.local/bin" >> $GITHUB_PATH
+
+            - name: "Verify Fusion installation"
+              run: |
+                  dbt --version
+
+            - name: "Install tox"
+              run: |
+                  python -m pip install --upgrade pip
+                  pip install tox
+
+            - name: "Run Fusion integration tests on duckdb"
+              run: |
+                  tox -e dbt_integration_fusion_duckdb
+
+    run-cloud-tests:
+        needs: run-duckdb-tests
         runs-on: ubuntu-latest
         environment: ${{ (github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name != github.repository) && 'cloud-tests' || '' }}
         strategy:

integration_tests_sl/.envrc

@@ -0,0 +1,7 @@
+export SNOWFLAKE_ACCOUNT=zna84829
+export SNOWFLAKE_USER=benoit_p
+export DBT_ENV_SECRET_SNOWFLAKE_PASS=zcq4wrv@VNJ8jvh2unv
+export SNOWFLAKE_ROLE=TRANSFORMER
+export SNOWFLAKE_DATABASE=analytics
+export SNOWFLAKE_WAREHOUSE=TRANSFORMING
+    

integration_tests_sl/.user.yml

@@ -0,0 +1 @@
+id: 86a16d1b-e324-4f8f-b626-bb8e31b1cd1a

integration_tests_sl/README.md

@@ -0,0 +1,42 @@
+# Test dbt Project
+
+The models within this folder (barring those in models/audit_schema_tests) represent a dbt project with poor DAG modeling. Error detection tools within this package are tested on this dbt project.
+
+<img width="1377" alt="DAG of the test dbt project" src="https://user-images.githubusercontent.com/73915542/170353654-58ad303c-adaa-49f6-86b8-723543eb2d3d.png">
+
+## Adding an Integration Test
+Create a seed which matches the intended output of your model and add equality tests comparing the output to your seed to the output of your model.
+
+## Local tests
+
+### AWS Athena
+
+To run tests locally, please follow instructions:
+
+* Set up environment variables:
+
+```bash
+ATHENA_S3_STAGING_DIR=
+ATHENA_S3_DATA_DIR=
+ATHENA_REGION=
+ATHENA_SCHEMA=
+ATHENA_WORKGROUP=
+```
+
+* Add `profiles.yml` file based on [sample](ci/sample.profiles.yml):
+
+```yaml
+    athena: # for local tests only
+      type: athena
+      s3_staging_dir: {{ env_var('ATHENA_S3_STAGING_DIR') }}
+      s3_data_dir: {{ env_var('ATHENA_S3_DATA_DIR') }}
+      s3_data_naming: schema_table_unique
+      region_name: {{ env_var('ATHENA_REGION') }}
+      schema: {{ env_var('ATHENA_SCHEMA') }}
+      database: awsdatacatalog
+      work_group: {{ env_var('ATHENA_WORKGROUP') }}
+      num_retries: 2
+      threads: 4
+```
+
+* Now you can run integration tests, see details [here](../run_test.sh) with `--target athena` flag for dbt commands.

integration_tests_sl/dbt_project.yml

@@ -0,0 +1,79 @@
+
+# Name your project! Project names should contain only lowercase characters
+# and underscores. A good package name should reflect your organization's
+# name or the intended use of these models
+name: 'dbt_project_evaluator_integration_tests'
+version: '1.0.0'
+config-version: 2
+
+# This setting configures which "profile" dbt uses for this project.
+profile: 'integration_tests'
+
+# These configurations specify where dbt should look for different types of files.
+# The `source-paths` config, for example, states that models in this project can be
+# found in the "models/" directory. You probably won't need to change these!
+model-paths: ["models"]
+analysis-paths: ["analysis"]
+test-paths: ["tests"]
+seed-paths: ["seeds"]
+macro-paths: ["macros"]
+snapshot-paths: ["snapshots"]
+
+target-path: "target"  # directory which will store compiled SQL files
+clean-targets:         # directories to be removed by `dbt clean`
+  - "target"
+  - "dbt_packages"
+
+dispatch:
+  - macro_namespace: dbt
+    search_order: ['dbt_project_evaluator', 'dbt']
+
+flags:
+  require_nested_cumulative_type_params: True
+  require_yaml_configuration_for_mf_time_spines: True
+  require_generic_test_arguments_property: True
+  require_ref_prefers_node_package_to_root: True
+
+models:
+  dbt_project_evaluator_integration_tests:
+    # materialize as ephemeral to prevent the fake models from executing, but keep them enabled
+    +materialized: ephemeral
+  dbt_project_evaluator:
+    marts:
+      tests:
+        fct_test_coverage:
+          # materialize as a table to ensure SQL query runs successfully
+          +materialized: table
+      documentation:
+        fct_documentation_coverage:
+          # materialize as a table to ensure SQL query runs successfully
+          +materialized: table
+
+tests:
+  dbt_project_evaluator:
+    +enabled: false
+  dbt_project_evaluator_integration_tests:
+    dbt_project_evaluator_schema_tests:
+      unique_int_all_dag_relationships_dpe_path:
+        # Grouping by expressions of type ARRAY is not allowed for BigQuery
+        +enabled: "{{ false if target.type in ['bigquery'] else true }}"
+
+seeds:
+  dbt_project_evaluator:
+    dbt_project_evaluator_exceptions:
+      +enabled: false
+
+vars:
+  # Set to true when running with dbt Fusion to disable tests that are not compatible
+  deactivate_for_fusion: false
+  # ensure integration tests run successfully when there are 0 of a given model type (extra)
+  exclude_packages: ['exclude_package']
+  exclude_paths_from_project: ["/to_exclude/","source_3.table_6","/dbt_project_evaluator_schema_tests/"]
+  model_types: ['base','staging', 'intermediate', 'marts', 'other', 'extra', 'new_model_type']
+  # dummy variable used for testing fct_hard_coded_references
+  my_table_reference: 'grace_table'
+  new_model_type_folder_name: 'my_new_models'
+  new_model_type_prefixes: 'nwmdl_'
+  insert_batch_size: 100
+  too_many_joins_threshold: 3
+  chained_views_threshold: 2

integration_tests_sl/exclude_package/dbt_project.yml

@@ -0,0 +1,30 @@
+
+# Name your project! Project names should contain only lowercase characters
+# and underscores. A good package name should reflect your organization's
+# name or the intended use of these models
+name: 'exclude_package'
+version: '1.0.0'
+config-version: 2
+
+# This setting configures which "profile" dbt uses for this project.
+profile: 'integration_tests'
+
+# These configurations specify where dbt should look for different types of files.
+# The `source-paths` config, for example, states that models in this project can be
+# found in the "models/" directory. You probably won't need to change these!
+model-paths: ["models"]
+analysis-paths: ["analysis"]
+test-paths: ["tests"]
+seed-paths: ["seeds"]
+macro-paths: ["macros"]
+snapshot-paths: ["snapshots"]
+
+target-path: "target"  # directory which will store compiled SQL files
+clean-targets:         # directories to be removed by `dbt clean`
+  - "target"
+  - "dbt_packages"
+
+models:
+  exclude_package:
+    # materialize as ephemeral to prevent the fake models from executing, but keep them enabled
+    +materialized: view

integration_tests_sl/exclude_package/models/staging/_models.yml

@@ -0,0 +1,9 @@
+models:
+  - name: excluded_model
+    config: 
+      access: public
+      contract: 
+        enforced: true
+    columns: 
+      - name: id
+        data_type: "{{ 'UInt8' if target.type in ['clickhouse'] else 'integer' }}"

integration_tests_sl/exclude_package/models/staging/_sources.yml

@@ -0,0 +1,4 @@
+sources:
+  - name: fake_source
+    tables:
+      - name: fake_source

integration_tests_sl/exclude_package/models/staging/excluded_model.sql

@@ -0,0 +1,2 @@
+-- {{ source('fake_source', 'fake_source') }}
+select 1 as id

integration_tests_sl/models/dbt_project_evaluator_schema_tests/core.yml

@@ -0,0 +1,40 @@
+version: 2
+
+models:
+  - name: int_all_dag_relationships_dpe
+    description: "This table shows one record for every resource and each of its downstream children (including itself). It includes models, snapshots, exposures, metrics and seeds"
+    columns:
+      - name: path
+        description: unique identifier representing a distinct path from each resource to another resource
+        data_tests:
+          - unique
+          - not_null
+
+  - name: int_all_graph_resources_dpe
+    description: "This table shows one record for each enabled resource in the graph and information about that resource."
+    columns:
+      - name: resource_id
+        data_tests:
+          - unique
+          - not_null
+      - name: on_schema_change
+        description: this contains the on_schema_change setting for incremental models. This column was sometimes an empty string, so should be tested to detect regressions
+        data_tests:
+          - not_null:
+              config:
+                where: "resource_type = 'model' and materialized = 'incremental'"
+      - name: model_type
+        data_tests:
+          - dbt_utils.expression_is_true:
+              arguments:
+                expression: "= 'base'"
+              config:
+                where: "resource_name = 'base_model_10'"
+
+  - name: int_direct_relationships_dpe
+    description: "This table shows one record for each direct parent/child pair in the graph."
+    columns:
+      - name: unique_id
+        data_tests:
+          - unique
+          - not_null