Feature/redshift model large (#129)

fivetran-catfritz · github-actions[bot] · web-flow · commit 6411a755e270 · 2026-02-19T10:38:42.000-06:00
* feature/redshift-model-large * add seed * revise * formatting * fix * update logic with new reqs * fix test schema * try different seed method * try different seed method * chagenlog * chagenlog * Generate dbt docs via GitHub Actions * adjust * changelog * Apply suggestion from @fivetran-catfritz * Generate dbt docs via GitHub Actions --------- Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh
@@ -19,4 +19,6 @@ dbt deps
 dbt seed --target "$db" --full-refresh
 dbt run --target "$db" --full-refresh
 dbt test --target "$db"
+dbt run --vars '{using_quoted_identifiers: true}' --target "$db" --full-refresh
+dbt test --vars '{using_quoted_identifiers: true}' --target "$db"
 dbt run-operation fivetran_utils.drop_schemas_automation --target "$db"
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,13 @@
+# dbt_salesforce_formula_utils v0.11.1
+[PR #129](https://github.com/fivetran/dbt_salesforce_formula_utils/pull/129) includes the following updates:
+
+## Feature Update
+- Applies **only** to Redshift: adds support for Redshift's `SUPER` data type in the `sfdc_formula_view` macro, enabling proper handling of large model configurations stored as JSON in the `fivetran_formula_model.model_large` column.
+
+## Under the Hood
+- Refactors the handling of the `using_quoted_identifiers` argument of the `sfdc_formula_view` macro for readability and maintainability.
+    - Adds integration tests for when `using_quoted_identifiers` is true. 
+
 # dbt_salesforce_formula_utils v0.11.0
 
 [PR #126](https://github.com/fivetran/dbt_salesforce_formula_utils/pull/126) includes the following updates:
diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,4 +1,4 @@
 name: 'salesforce_formula_utils'
-version: '0.11.0'
+version: '0.11.1'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<3.0.0"]
diff --git a/docs/catalog.json b/docs/catalog.json
diff --git a/docs/manifest.json b/docs/manifest.json
diff --git a/integration_tests/.gitignore b/integration_tests/.gitignore
@@ -3,4 +3,5 @@ target/
 dbt_modules/
 logs/
 env/
-package-lock.yml
+package-lock.yml
+dbt_internal_packages/
diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -1,5 +1,5 @@
 name: 'salesforce_formula_integration_tests'
-version: '0.11.0'
+version: '0.11.1'
 profile: 'integration_tests'
 config-version: 2
 vars:
@@ -15,12 +15,24 @@ seeds:
   salesforce_formula_integration_tests:
     +quote_columns: "{{ true if target.type == 'redshift' else false }}"
     opportunity:
-      +columns_types:
+      +column_types:
         amount: float
         probability: float
     fivetran_formula:
       +column_types:
         view_sql: "{{ 'varchar(256)'  if target.type in ('redshift','postgres') else 'string'}}"
+    fivetran_formula_model: 
+      +enabled: "{{ false if target.type == 'redshift' else true }}"
+    fivetran_formula_model_redshift:
+      +enabled: "{{ true if target.type == 'redshift' else false }}"
+      +alias: fivetran_formula_model
+      +column_types:
+        model_string: varchar(65535)
+      # Load as VARCHAR first, then convert to SUPER using JSON_PARSE() in a post-hook.
+      # This is necessary for testing the SUPER datatype with seed data.
+      +post-hook:
+        - "alter table {{ this }} add column model_large super default null"
+        - "update {{ this }} set model_large = json_parse(model_string)"
 
 dispatch:
   - macro_namespace: dbt_utils
diff --git a/integration_tests/models/account_view.sql b/integration_tests/models/account_view.sql
@@ -1,3 +1,4 @@
 {{ salesforce_formula_utils.sfdc_formula_view(
     source_table = 'account',
+    using_quoted_identifiers = var('using_quoted_identifiers', false),
     full_statement_version=true) }} --leaving full_statement_version=true in this model to make sure it doesn't error
diff --git a/integration_tests/models/opportunity_view.sql b/integration_tests/models/opportunity_view.sql
@@ -1,2 +1,3 @@
 {{ salesforce_formula_utils.sfdc_formula_view(
-    source_table = 'opportunity') }}
+    source_table = 'opportunity',
+    using_quoted_identifiers = var('using_quoted_identifiers', false))  }}
diff --git a/integration_tests/seeds/fivetran_formula_model_redshift.csv b/integration_tests/seeds/fivetran_formula_model_redshift.csv
@@ -0,0 +1,5 @@
+object,model_string
+opportunity,"""select \nopportunity__table.*,\n(amount * probability) as opportunity_test_formula \n from salesforce_formula_integrations_tests.opportunity as opportunity__table"""
+account,"""select \naccount__table.*,\n(number_of_employees * 2) as account_test_formula \n from salesforce_formula_integrations_tests.account as account__table"""
+user,"""select \nuser__table.*,\ncase when is_active = true then 'active' else 'not active' end as is_active_user \n from salesforce_formula_integrations_tests.user as user__table"""
+user_role,"""select \nuser_role__table.*,\nconcat(developer_name,id) as developer_and_id \n from salesforce_formula_integrations_tests.user_role as user_role__table"""
diff --git a/macros/sfdc_formula_view.sql b/macros/sfdc_formula_view.sql
@@ -7,23 +7,53 @@
     )
 }}
 
--- Raise a warning if users are trying to use full_statement_version=false. We are keeping the variable in the macro, however, since we don't want errors if they previously set it to true.
+{# Raise a warning if users are trying to use full_statement_version=false. We are keeping the variable in the macro, however, since we don't want errors if they previously set it to true. #}
 {% if not full_statement_version %}
     {{ exceptions.warn("\nERROR: The full_statement_version=false, reserved_table_name, and fields_to_include parameters are no longer supported. Please update your " ~ this.identifier|upper ~ " model to remove these parameters.\n") }}
     See_full_model_error_in_log
 
 {% else %}
+    {% if target.type == 'redshift' %}
 
-    {% if using_quoted_identifiers %}
-    {%- set table_results = dbt_utils.get_column_values(table=source(source_name, 'fivetran_formula_model'), 
-                                                        column='"MODEL"' if target.type in ('snowflake') else '"model"' if target.type in ('postgres', 'redshift', 'snowflake') else '`model`', 
-                                                        where=("\"OBJECT\" = '" if target.type in ('snowflake') else "\"object\" = '" if target.type in ('postgres', 'redshift') else "`object` = '") ~ source_table ~ "'") -%}
+        {# Specifically for redshift we need to check if the model_large column exists and has non-null values. #}
+        {% set formula_model_columns = adapter.get_columns_in_relation(source(source_name, 'fivetran_formula_model')) %}
+        {%- set formula_model_column_names = formula_model_columns | map(attribute='name') | map('lower') | list -%}
+        {%- set model_large_col_exists = 'model_large' in formula_model_column_names -%}
 
-    {% else %}
-    {%- set table_results = dbt_utils.get_column_values(table=source(source_name, 'fivetran_formula_model'), column='model', where="object = '" ~ source_table ~ "'") -%}
+        {%- set run_query %}
+            select 'has_values'
+            from {{ source(source_name, 'fivetran_formula_model') }}
+            where model_large is not null
+            limit 1
+        {%- endset %}
+
+        {# Use the run_query only if model_large_col_exists #}
+        {%- set model_large_has_values = (dbt_utils.get_single_value(run_query) == 'has_values') if model_large_col_exists else false -%}
+        {%- set model_column_name = 'model_large' if model_large_has_values else 'model' -%}
 
+        {# Check datatype #}
+        {%- set ns = namespace(column_type='string') -%}
+        {%- for column in formula_model_columns if column.name|lower == model_column_name -%}
+            {%- set ns.column_type = column.dtype|lower -%}
+        {%- endfor -%}
+        {%- set model_column_datatype = ns.column_type -%}
+
+    {% else %}
+        {%- set model_column_name = 'MODEL' if target.type == 'snowflake' else 'model' -%}
+        {%- set model_column_datatype = 'string' -%}
     {% endif %}
 
-    {{ table_results[0] }}
+    {%- set object_column = adapter.quote('OBJECT' if target.type == 'snowflake' else 'object') if using_quoted_identifiers else 'object' -%}
+    {%- set model_col = adapter.quote(model_column_name) if using_quoted_identifiers else model_column_name -%}
+
+    {%- set table_results = dbt_utils.get_column_values(
+        table=source(source_name, 'fivetran_formula_model'),
+        column=model_col,
+        where=object_column ~ " = '" ~ source_table ~ "'"
+    ) -%}
+
+    {# Use dbt's built-in JSON parsing to handle all escape sequences for SUPER datatype #}
+    {{ fromjson(table_results[0]) if model_column_datatype == 'super' else table_results[0] }}
+
 {% endif %}
 {%- endmacro -%}
