Merge pull request #123 from dbt-labs/version_3_adding_order_by

Adding Order By and Fixing Case When

Author: callum-mcdata

macros/get_metric_sql.sql

@@ -87,16 +87,31 @@ metrics there are #}
 
     {{ metrics.gen_joined_metrics_cte(metric_tree["parent_set"], metric_tree["expression_set"], metric_tree["ordered_expression_set"], grain, non_calendar_dimensions, calendar_dimensions, secondary_calculations, relevant_periods) }}
     {{ metrics.gen_secondary_calculation_cte(metric_tree["base_set"], non_calendar_dimensions, grain, metric_tree["full_set"], secondary_calculations, calendar_dimensions) }}
-    {{ metrics.gen_final_cte(metric_tree["base_set"], grain, metric_tree["full_set"], secondary_calculations,where) }}
-    
+    {{ metrics.gen_final_cte(
+        metric_tree=metric_tree,
+        grain=grain, 
+        dimensions=non_calendar_dimensions, 
+        calendar_dimensions=calendar_dimensions, 
+        relevant_periods=relevant_periods,
+        secondary_calculations=secondary_calculations,
+        where=where) 
+        }}    
 
 {# If we're calling the develop macro then we don't need to loop through the metrics because we know 
 this is only a single metric and not an expression metric #}
 {%- elif is_develop_macro -%}
 
     {{ metrics.build_metric_sql(metric_name, metric_type, metric_sql, metric_timestamp, metric_filters, metric_model, grain, non_calendar_dimensions, secondary_calculations, start_date, end_date, calendar_tbl, relevant_periods, calendar_dimensions, dimensions_provided) }}
     {{ metrics.gen_secondary_calculation_cte(metric_tree["base_set"], non_calendar_dimensions, grain, metric_tree["full_set"], secondary_calculations, calendar_dimensions) }}
-    {{ metrics.gen_final_cte(metric_tree["base_set"], grain, metric_tree["full_set"], secondary_calculations,where) }}
+    {{ metrics.gen_final_cte(
+        metric_tree=metric_tree,
+        grain=grain, 
+        dimensions=non_calendar_dimensions, 
+        calendar_dimensions=calendar_dimensions, 
+        relevant_periods=relevant_periods,
+        secondary_calculations=secondary_calculations,
+        where=where) 
+        }}
 
 {# If it is NOT a composite metric, we run the baseline model #}
 {%- else -%}
@@ -123,8 +138,17 @@ this is only a single metric and not an expression metric #}
         {{ metrics.build_metric_sql(metric_name, metric_type, metric_sql, metric_timestamp, metric_filters, metric_model, grain, non_calendar_dimensions, secondary_calculations, start_date, end_date,calendar_tbl, relevant_periods, calendar_dimensions,dimensions_provided) }}
     {% endfor %}
     {{ metrics.gen_secondary_calculation_cte(metric_tree["base_set"], non_calendar_dimensions, grain, metric_tree["full_set"], secondary_calculations, calendar_dimensions) }}
-    {{ metrics.gen_final_cte(metric_tree["base_set"], grain, metric_tree["full_set"], secondary_calculations,where) }}
 
+    {{ metrics.gen_final_cte(
+        metric_tree=metric_tree,
+        grain=grain, 
+        dimensions=non_calendar_dimensions, 
+        calendar_dimensions=calendar_dimensions, 
+        relevant_periods=relevant_periods,
+        secondary_calculations=secondary_calculations,
+        where=where) 
+        }}
+
 {%- endif -%}
 
 {% endmacro %}

macros/sql_gen/gen_base_query.sql

@@ -24,7 +24,7 @@
             calendar_table.{{ calendar_dim }},
         {%- endfor %}
         {%- if metric_sql and metric_sql | replace('*', '') | trim != '' -%}
-            base_model.{{ metric_sql }} as property_to_aggregate
+            ({{ metric_sql }}) as property_to_aggregate
         {%- elif metric_type == 'count' -%}
             1 as property_to_aggregate /*a specific expression to aggregate wasn't provided, so this effectively creates count(*) */
         {%- else -%}

macros/sql_gen/gen_final_cte.sql

@@ -1,72 +1,49 @@
-{% macro gen_final_cte(base_set,grain,full_set,secondary_calculations, where) %}
-    {{ return(adapter.dispatch('gen_final_cte', 'metrics')(base_set,grain,full_set,secondary_calculations, where)) }}
+{%- macro gen_final_cte(metric_tree, grain, dimensions, calendar_dimensions, relevant_periods, secondary_calculations, where) -%}  
+    {{ return(adapter.dispatch('gen_final_cte', 'metrics')(metric_tree, grain, dimensions, calendar_dimensions, relevant_periods, secondary_calculations, where)) }}
 {% endmacro %}
 
-{% macro default__gen_final_cte(base_set,grain,full_set,secondary_calculations, where) %}
+{% macro default__gen_final_cte(metric_tree, grain, dimensions, calendar_dimensions, relevant_periods, secondary_calculations, where) %}
 
-{%- if full_set | length > 1 %}
 
-    {% if secondary_calculations | length > 0 %}
+{%- if secondary_calculations | length > 0 -%}
 
-        ,final as (
+    , final as (
 
-            select
-                *
-            from secondary_calculations
-        )
+        select
+            *
+        from secondary_calculations
+    )
 
-        select * from final 
+    select * from final 
 
-            -- metric where clauses...
-        {% if where %}
-        where {{ where }}
-        {% endif %}
-
-    {% else %}
-
-    select * from joined_metrics
-
-    -- metric where clauses...
-    {% if where %}
-        where {{ where }}
-    {% endif %}
-
-    {% endif %}
+    {# metric where clauses #}
+    {%- if where %}
+    where {{ where }}
+    {%- endif %}
+    {{ metrics.gen_order_by(grain, dimensions, calendar_dimensions, relevant_periods) }}
 
 {% else %}
 
-    {% if secondary_calculations | length > 0 %}
-
-        -- single metric with secondary calculations
-        
-        ,final as (
-
-            select
-                *
-            from secondary_calculations
-        )
+    {%- if metric_tree.full_set | length > 1 %}
 
-        select * from final 
-
-        -- metric where clauses...
-        {% if where %}
-        where {{ where }}
-        {% endif %}
-
-        {% else %}
-
-        -- single metric without secondary calculations
-
-        select * from {{base_set[0]}}__final 
+    select * from joined_metrics
+    {#- metric where clauses -#}
+        {%- if where %}
+    where {{ where }}
+        {%- endif -%}
+    {{ metrics.gen_order_by(grain, dimensions, calendar_dimensions, relevant_periods) }}
 
+    {% else %}
 
-        -- metric where clauses...
-        {% if where %}
-        where {{ where }}
-        {% endif %}
+    select * from {{metric_tree.base_set[0]}}__final 
+        {%- if where %}
+    where {{ where }}
+        {%- endif -%}
+    {{ metrics.gen_order_by(grain, dimensions, calendar_dimensions, relevant_periods) }}
+    
+    {%- endif %}
 
-    {% endif %}
 
-{% endif %}
+{%- endif %}
 
-{% endmacro %}
+{%- endmacro %}

macros/sql_gen/gen_order_by.sql

@@ -0,0 +1,24 @@
+{%- macro gen_order_by(grain, dimensions, calendar_dimensions, relevant_periods) -%}
+    {{ return(adapter.dispatch('gen_order_by', 'metrics')(grain, dimensions, calendar_dimensions, relevant_periods)) }}
+{%- endmacro -%}
+
+{% macro default__gen_order_by(grain, dimensions, calendar_dimensions, relevant_periods) %}
+
+{#- This macro exclusively exists because dynamic group by counts based on range 
+were too funky when we hardcoded values for 1+1. So we're getting around it by
+making it its own function -#}
+{#- The issue arises when we have an initial date column (ie date_month) where month 
+is also included in the relevent periods. This causes issues and so we need to
+remove the grain from the list of relevant periods so it isnt double counted -#}
+    {%- set dimension_length = dimensions | length -%}
+    {%- set calendar_dimension_length = calendar_dimensions | length -%}
+    {%- set cleaned_relevant_periods = [] -%}
+    {%- set period_length = relevant_periods | length -%}
+    {%- set total_length = dimension_length + period_length + calendar_dimension_length -%}
+    {% if grain != 'all_time' %}
+        order by
+        {% for number in range(1,total_length+2) -%}
+            {{ number }} desc {%- if not loop.last -%}, {% endif -%}
+        {% endfor -%}
+    {% endif %}
+{% endmacro %}

tests/functional/metric_options/case_when/test_case_when_metric.py

@@ -0,0 +1,106 @@
+from struct import pack
+import os
+import pytest
+from dbt.tests.util import run_dbt
+
+# our file contents
+from tests.functional.fixtures import (
+    fact_orders_source_csv,
+    fact_orders_sql,
+    fact_orders_yml,
+)
+
+# models/case_when_metric.sql
+case_when_metric_sql = """
+select *
+from 
+{{ metrics.calculate(metric('case_when_metric'), 
+    grain='month'
+    )
+}}
+"""
+
+# models/case_when_metric.yml
+case_when_metric_yml = """
+version: 2 
+models:
+  - name: case_when_metric
+    tests: 
+      - dbt_utils.equality:
+          compare_model: ref('case_when_metric__expected')
+metrics:
+  - name: case_when_metric
+    model: ref('fact_orders')
+    label: Total Discount ($)
+    timestamp: order_date
+    time_grains: [day, week, month]
+    type: sum
+    sql: case when had_discount = true then 1 else 0 end 
+    dimensions:
+      - order_country
+"""
+
+# seeds/case_when_metric__expected.csv
+case_when_metric__expected_csv = """
+date_month,case_when_metric
+2022-01-01,2
+2022-02-01,1
+""".lstrip()
+
+class TestCaseWhenMetric:
+
+    # configuration in dbt_project.yml
+    @pytest.fixture(scope="class")
+    def project_config_update(self):
+        return {
+          "name": "example",
+          "models": {"+materialized": "table"}
+        }
+
+    # install current repo as package
+    @pytest.fixture(scope="class")
+    def packages(self):
+        return {
+            "packages": [
+                {"local": os.getcwd()}
+                ]
+        }
+
+
+    # everything that goes in the "seeds" directory
+    @pytest.fixture(scope="class")
+    def seeds(self):
+        return {
+            "fact_orders_source.csv": fact_orders_source_csv,
+            "case_when_metric__expected.csv": case_when_metric__expected_csv,
+        }
+
+    # everything that goes in the "models" directory
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {
+            "fact_orders.sql": fact_orders_sql,
+            "fact_orders.yml": fact_orders_yml,
+            "case_when_metric.sql": case_when_metric_sql,
+            "case_when_metric.yml": case_when_metric_yml
+        }
+
+    def test_build_completion(self,project,):
+        # running deps to install package
+        results = run_dbt(["deps"])
+
+        # seed seeds
+        results = run_dbt(["seed"])
+        assert len(results) == 2
+
+        # initial run
+        results = run_dbt(["run"])
+        assert len(results) == 3
+
+        # test tests
+        results = run_dbt(["test"]) # expect passing test
+        assert len(results) == 1
+
+        # # # validate that the results include pass
+        result_statuses = sorted(r.status for r in results)
+        assert result_statuses == ["pass"]