Merge pull request #208 from dbt-labs/let-there-be-median

Let There Be Median

Author: callum-mcdata

.changes/unreleased/Features-20230109-145530.yaml

@@ -0,0 +1,7 @@
+kind: Features
+body: Adding median
+time: 2023-01-09T14:55:30.09271-06:00
+custom:
+  Author: callum-mcdata
+  Issue: "180"
+  PR: "208"

integration_tests/dbt_project.yml

@@ -6,7 +6,7 @@ version: "1.0.0"
 config-version: 2
 
 # This setting configures which "profile" dbt uses for this project.
-profile: "dbt_metrics_integration_tests_postgres"
+profile: "dbt_metrics_integration_tests_bigquery"
 
 model-paths: ["models"]
 analysis-paths: ["analyses"]

integration_tests/models/metric_definitions/base_median_metric.yml

@@ -0,0 +1,14 @@
+version: 2 
+metrics:
+  - name: base_median_metric
+    model: ref('fact_orders')
+    label: Total Discount ($)
+    timestamp: order_date
+    time_grains: [day, week, month, all_time]
+    calculation_method: median
+    expression: discount_total
+    dimensions:
+      - had_discount
+      - order_country
+
+    

integration_tests/models/metric_testing_models/base_median_metric.sql

@@ -0,0 +1,6 @@
+select *
+from 
+{{ metrics.calculate(metric('base_median_metric'), 
+    grain='month', 
+    dimensions=['had_discount']) 
+}}

integration_tests/models/metric_testing_models/base_median_metric_no_time_grain.sql

@@ -0,0 +1,4 @@
+select *
+from 
+{{ metrics.calculate(metric('base_median_metric')) 
+}}

macros/calculate.sql

@@ -7,7 +7,7 @@
     {#- Need this here, since the actual ref is nested within loops/conditions: -#}
     -- depends on: {{ ref(var('dbt_metrics_calendar_model', 'dbt_metrics_default_calendar')) }}
 
-    {# ############
+    {#- ############
     VARIABLE SETTING - Creating the metric tree and making sure metric list is a list!
     ############ -#}
 

macros/get_metric_sql.sql

@@ -48,13 +48,10 @@ metrics there are -#}
     start_date=start_date, 
     end_date=end_date) 
     }}
-
 {#- Next we check if it is a composite metric or single metric by checking the length of the list -#}
 {#- This filter forms the basis of how we construct the SQL -#}
-
 {#- If composite, we begin by looping through each of the metric names that make
 up the composite metric. -#}
-
 {%- for metric_name in metric_tree["parent_set"] -%}
 
     {{ metrics.build_metric_sql(

macros/sql_gen/build_metric_sql.sql

@@ -1,5 +1,5 @@
 {%- macro build_metric_sql(metric_dictionary, grain, dimensions, secondary_calculations, start_date, end_date, calendar_tbl, relevant_periods, calendar_dimensions, dimensions_provided, total_dimension_count) %}
-    
+
     {%- set treat_null_values_as_zero = metric_dictionary.get("config").get("treat_null_values_as_zero", True)  -%}
     {#- This is the SQL Gen part - we've broken each component out into individual macros -#}
     {#- We broke this out so it can loop for composite metrics -#}
@@ -28,7 +28,6 @@
 
         {%- endif -%}
 
-
         {{ metrics.gen_spine_time_cte(
             metric_name=metric_dictionary.name, 
             grain=grain, 
@@ -53,4 +52,4 @@
         treat_null_values_as_zero=treat_null_values_as_zero
     )}} 
 
-{% endmacro -%}
+{%- endmacro -%}

macros/sql_gen/gen_aggregate_cte.sql

@@ -9,6 +9,7 @@
     and THEN aggregating, we are instead aggregating from the beginning and then 
     joining downstream for performance. Additionally, we're using a subquery instead 
     of a CTE, which was significantly more performant during our testing. -#}
+    {#- #}
     select
 
         {%- if grain %}
@@ -37,12 +38,11 @@
 
         {%- if grain %}
         {{ bool_or('metric_date_day is not null') }} as has_data,
-        {% endif %}
+        {%- endif %}
 
         {#- This line performs the relevant aggregation by calling the 
         gen_primary_metric_aggregate macro. Take a look at that one if you're curious -#}
         {{ metrics.gen_primary_metric_aggregate(metric_dictionary.calculation_method, 'property_to_aggregate') }} as {{ metric_dictionary.name }}
-
     from ({{ metrics.gen_base_query(
                 metric_dictionary=metric_dictionary,
                 grain=grain, 
@@ -57,11 +57,9 @@
     ) as base_query
 
     where 1=1
-
-    {% if metric_dictionary.window is not none and grain %}
+    {%- if metric_dictionary.window is not none and grain %}
     and date_{{grain}} = window_filter_date
-    {% endif %}
-
+    {%- endif %}
     {{ metrics.gen_group_by(grain, dimensions, calendar_dimensions, relevant_periods) }}
 
 )

macros/sql_gen/gen_base_query.sql

@@ -3,12 +3,10 @@
 {% endmacro %}
 
 {% macro default__gen_base_query(metric_dictionary, grain, dimensions, secondary_calculations, start_date, end_date, calendar_tbl, relevant_periods, calendar_dimensions, total_dimension_count) %}
-
         {# This is the "base" CTE which selects the fields we need to correctly 
-        calculate the metric.  #}
+        calculate the metric.  -#}
         select 
-    
-            {% if grain %}
+            {% if grain -%}
             cast(base_model.{{metric_dictionary.timestamp}} as date) as metric_date_day,
             calendar_table.date_{{ grain }} as date_{{grain}},
             calendar_table.date_day as window_filter_date,
@@ -17,36 +15,23 @@
             calendar_table.date_{{ period }},
                     {% endfor -%}
                 {%- endif -%}
-            {%- endif -%}
-
-            {% for dim in dimensions %}
+            {%- endif %}
+            {#- -#}
+            {%- for dim in dimensions -%}
             base_model.{{ dim }},
-            {% endfor %}
-
-            {% for calendar_dim in calendar_dimensions %}
+            {%- endfor -%}
+            {%- for calendar_dim in calendar_dimensions -%}
             calendar_table.{{ calendar_dim }},
-            {% endfor %}
-
-
-            {%- if metric_dictionary.expression and metric_dictionary.expression | replace('*', '') | trim != '' %}
-
-            ({{ metric_dictionary.expression }}) as property_to_aggregate
-            {%- elif metric_dictionary.calculation_method == 'count' -%} 
-            {# We use 1 as the property to aggregate in count so that it matches count(*) #}
-            1 as property_to_aggregate 
-            {%- else -%}
-                {%- do exceptions.raise_compiler_error("Expression to aggregate is required for non-count aggregation in metric `" ~ metric_dictionary.name ~ "`") -%}  
-            {%- endif %}
-
-
+            {%- endfor -%}
+            {{ metrics.gen_property_to_aggregate(metric_dictionary, grain, dimensions, calendar_dimensions) }}
         from {{ metric_dictionary.metric_model }} base_model 
-        
-        {% if grain or calendar_dimensions|length > 0 %}
+        {# -#}
+        {%- if grain or calendar_dimensions|length > 0 -%}
         {{ metrics.gen_calendar_table_join(metric_dictionary, calendar_tbl) }} 
-        {% endif %}
-
+        {%- endif -%}
+        {# #}
         where 1=1
-        
+        {#- -#}
         {{ metrics.gen_filters(metric_dictionary, start_date, end_date) }}
 
 {%- endmacro -%}