Update opmon generation to use statistic tables (#523)

scholtzan · web-flow · commit 50de83fc0026 · 2022-09-20T10:23:19.000-07:00
diff --git a/generator/dashboards/operational_monitoring_dashboard.py b/generator/dashboards/operational_monitoring_dashboard.py
@@ -12,13 +12,6 @@ class OperationalMonitoringDashboard(Dashboard):
 
     type: str = "operational_monitoring_dashboard"
 
-    OPMON_DASH_EXCLUDED_FIELDS: List[str] = [
-        "branch",
-        "probe",
-        "histogram__VALUES__key",
-        "histogram__VALUES__value",
-    ]
-
     def __init__(
         self,
         title: str,
@@ -101,16 +94,17 @@ def to_lookml(self, bq_client):
                 series_colors = self._map_series_to_colours(
                     table_defn["branches"], explore
                 )
-                for metric in table_defn.get("probes", []):
+                for summary in table_defn.get("summaries", []):
                     if self.compact_visualization:
-                        title = "Probe"
+                        title = "Metric"
                     else:
-                        title = lookml_utils.slug_to_title(metric)
+                        title = lookml_utils.slug_to_title(summary["metric"])
 
                     kwargs["elements"].append(
                         {
                             "title": title,
-                            "metric": metric,
+                            "metric": summary["metric"],
+                            "statistic": summary["statistic"],
                             "explore": explore,
                             "series_colors": series_colors,
                             "xaxis": self.xaxis,
@@ -124,7 +118,8 @@ def to_lookml(self, bq_client):
                         kwargs["elements"].append(
                             {
                                 "title": f"{title} - By {self.group_by_dimension}",
-                                "metric": metric,
+                                "metric": summary["metric"],
+                                "statistic": summary["statistic"],
                                 "explore": explore,
                                 "series_colors": series_colors,
                                 "xaxis": self.xaxis,
diff --git a/generator/dashboards/templates/dashboard.lkml b/generator/dashboards/templates/dashboard.lkml
@@ -6,41 +6,53 @@
   elements:
   {% for element in elements -%}
   - title: {{element.title}}
-    name: {{element.title}}
+    name: {{element.title}}_{{element.statistic}}
+    note_state: expanded
+    note_display: above
+    note_text: {{element.statistic.title()}}
     explore: {{element.explore}}
+    {% if element.statistic == "percentile" -%}
     type: "ci-line-chart"
+    {% else -%}
+    type: looker_line
+    {% endif -%}
     fields: [
       {{element.explore}}.{{element.xaxis}},
       {{element.explore}}.branch,
-      {{element.explore}}.high,
-      {{element.explore}}.low,
-      {{element.explore}}.percentile
+      {% if element.statistic == "percentile" -%}
+      {{element.explore}}.upper,
+      {{element.explore}}.lower,
+      {% endif -%}
+      {{element.explore}}.point
     ]
     pivots: [
       {{element.explore}}.branch 
       {%- if group_by_dimension and element.title.endswith(group_by_dimension) %}, {{element.explore}}.{{group_by_dimension}} {% endif %}
     ]
     {% if not compact_visualization -%}
     filters:
-      {{element.explore}}.probe: {{element.metric}}
+      {{element.explore}}.metric: {{element.metric}}
+      {{element.explore}}.statistic: {{element.statistic}}
     {% endif -%}
     row: {{element.row}}
     col: {{element.col}}
     width: 12
     height: 8
     field_x: {{element.explore}}.{{element.xaxis}}
-    field_y: {{element.explore}}.percentile
+    field_y: {{element.explore}}.point
     log_scale: false
-    ci_lower: {{element.explore}}.low
-    ci_upper: {{element.explore}}.high
+    ci_lower: {{element.explore}}.lower
+    ci_upper: {{element.explore}}.upper
     show_grid: true
     listen:
-      Percentile: {{element.explore}}.percentile_conf
+      {%- if element.statistic == "percentile" %}
+      Percentile: {{element.explore}}.parameter
+      {%- endif %}
       {%- for dimension in dimensions %}
       {{dimension.title}}: {{element.explore}}.{{dimension.name}}
       {%- endfor %}
       {% if compact_visualization -%}
-      Probe: {{element.explore}}.probe
+      Metric: {{element.explore}}.metric
       {% endif -%}
     {%- for branch, color in element.series_colors.items() %}
     {{ branch }}: "{{ color }}"
@@ -54,7 +66,7 @@
     explore: {{alerts.explore}}
     type: looker_grid
     fields: [{{alerts.explore}}.submission_date,
-      {{alerts.explore}}.probe, {{alerts.explore}}.percentile,
+      {{alerts.explore}}.metric, {{alerts.explore}}.statistic, {{alerts.explore}}.percentile,
       {{alerts.explore}}.message, {{alerts.explore}}.branch, {{alerts.explore}}.errors]
     sorts: [{{alerts.explore}}.submission_date
         desc]
@@ -107,39 +119,45 @@
   filters:
   - name: Percentile
     title: Percentile
-    type: number_filter
+    type: field_filter
     default_value: '50'
     allow_multiple_values: false
     required: true
     ui_config:
-      type: dropdown_menu
+      type: slider
       display: inline
-      options:
-      - '10'
-      - '20'
-      - '30'
-      - '40'
-      - '50'
-      - '60'
-      - '70'
-      - '80'
-      - '90'
-      - '95'
-      - '99'
+      options: []
+    model: operational_monitoring
+    explore: {{ elements[0].explore }}
+    listens_to_filters: []
+    field: {{ elements[0].explore }}.parameter
   {% if compact_visualization -%}
-  - name: Probe
-    title: Probe
+  - name: Metric
+    title: Metric
     type: field_filter
     default_value: '{{ elements[0].metric }}'
-    allow_multiple_values: true
+    allow_multiple_values: false
     required: true
     ui_config:
       type: dropdown_menu
       display: popover
     model: operational_monitoring
     explore: {{ elements[0].explore }}
     listens_to_filters: []
-    field: {{ elements[0].explore }}.probe
+    field: {{ elements[0].explore }}.metric
+  - name: Statistic
+    title: Statistic
+    type: field_filter
+    default_value: '{{ elements[0].statistic }}'
+    allow_multiple_values: false
+    required: true
+    ui_config:
+      type: dropdown_menu
+      display: popover
+    model: operational_monitoring
+    explore: {{ elements[0].explore }}
+    listens_to_filters: []
+    field: {{ elements[0].explore }}.statistic
   {% endif -%}
 
   {% for dimension in dimensions -%}
@@ -171,5 +189,5 @@
       {% for option in dimension.options | sort -%}
       - '{{option}}'
       {% endfor %}
-    {% endif %}
+  {% endif %}
   {% endfor -%}
diff --git a/generator/explores/operational_monitoring_explore.py b/generator/explores/operational_monitoring_explore.py
@@ -28,7 +28,7 @@ def __init__(
             self.branches = ", ".join(defn["branches"])
             self.xaxis = defn.get("xaxis")
             self.dimensions = defn.get("dimensions", {})
-            self.probes = defn.get("probes", [])
+            self.summaries = defn.get("summaries", [])
 
     @staticmethod
     def from_views(views: List[View]) -> Iterator[Explore]:
@@ -56,7 +56,6 @@ def _to_lookml(
 
         filters = [
             {f"{base_view_name}.branch": self.branches},
-            {f"{base_view_name}.percentile_conf": "50"},
         ]
         for dimension, info in self.dimensions.items():
             if "default" in info:
diff --git a/generator/namespaces.py b/generator/namespaces.py
@@ -100,12 +100,11 @@ def _get_opmon(bq_client: bigquery.Client, namespaces: Dict[str, Any]):
         branches = project.get("branches", ["enabled", "disabled"])
 
         # append view and explore for data type
-        project_data_type_id = f"{table_prefix}"
-        table = f"{PROD_PROJECT}.{OPMON_DATASET}.{table_prefix}"
+        table = f"{PROD_PROJECT}.{OPMON_DATASET}.{table_prefix}_statistics"
         dimensions = operational_monitoring_utils.get_dimension_defaults(
             bq_client, table, project["dimensions"]
         )
-        om_content["views"][project_data_type_id] = {
+        om_content["views"][table_prefix] = {
             "type": "operational_monitoring_view",
             "tables": [
                 {
@@ -115,13 +114,13 @@ def _get_opmon(bq_client: bigquery.Client, namespaces: Dict[str, Any]):
                 }
             ],
         }
-        om_content["explores"][project_data_type_id] = {
+        om_content["explores"][table_prefix] = {
             "type": "operational_monitoring_explore",
             "views": {"base_view": f"{table_prefix}"},
             "branches": branches,
             "xaxis": project["xaxis"],
             "dimensions": dimensions,
-            "probes": [p["name"] for p in project["probes"]],
+            "summaries": project["summaries"],
         }
 
         if "alerting" in project and project["alerting"]:
@@ -145,15 +144,15 @@ def _get_opmon(bq_client: bigquery.Client, namespaces: Dict[str, Any]):
             "tables": [
                 {
                     "explore": f"{table_prefix}",
-                    "table": f"{PROD_PROJECT}.{OPMON_DATASET}.{table_prefix}",
+                    "table": f"{PROD_PROJECT}.{OPMON_DATASET}.{table_prefix}_statistics",
                     "branches": branches,
                     "xaxis": project["xaxis"],
                     "compact_visualization": project.get(
                         "compact_visualization", False
                     ),
                     "dimensions": dimensions,
                     "group_by_dimension": project.get("group_by_dimension", None),
-                    "probes": [p["name"] for p in project["probes"]],
+                    "summaries": project["summaries"],
                 }
             ],
         }
diff --git a/generator/views/__init__.py b/generator/views/__init__.py
@@ -14,10 +14,10 @@
     ClientCountsView.type: ClientCountsView,
     EventsView.type: EventsView,
     FunnelAnalysisView.type: FunnelAnalysisView,
+    OperationalMonitoringView.type: OperationalMonitoringView,
+    OperationalMonitoringAlertingView.type: OperationalMonitoringAlertingView,
     GleanPingView.type: GleanPingView,
     PingView.type: PingView,
     GrowthAccountingView.type: GrowthAccountingView,
-    OperationalMonitoringView.type: OperationalMonitoringView,
-    OperationalMonitoringAlertingView.type: OperationalMonitoringAlertingView,
     TableView.type: TableView,
 }
diff --git a/generator/views/operational_monitoring_view.py b/generator/views/operational_monitoring_view.py
@@ -1,7 +1,6 @@
 """Class to describe an Operational Monitoring View."""
 from __future__ import annotations
 
-from textwrap import dedent
 from typing import Any, Dict, List, Optional
 
 from . import lookml_utils
@@ -10,17 +9,19 @@
 
 ALLOWED_DIMENSIONS = {
     "branch",
-    "probe",
-    "value__VALUES__key",
-    "value__VALUES__value",
+    "metric",
+    "statistic",
+    "point",
+    "parameter",
+    "upper",
+    "lower",
 }
 
 
 class OperationalMonitoringView(PingView):
     """A view on a operational monitoring table."""
 
     type: str = "operational_monitoring_view"
-    percentile_ci_labels = ["percentile", "low", "high"]
 
     def __init__(self, namespace: str, name: str, tables: List[Dict[str, Any]]):
         """Create instance of a OperationalMonitoringView."""
@@ -40,34 +41,6 @@ def __init__(self, namespace: str, name: str, tables: List[Dict[str, Any]]):
                 "sql": xaxis_to_sql_mapping[xaxis],
             }
         ]
-        self.parameters: List[Dict[str, str]] = [
-            {
-                "name": "percentile_conf",
-                "type": "number",
-                "label": "Percentile",
-                "default_value": "50.0",
-            }
-        ]
-
-    def _percentile_measure(self, percentile_ci_label) -> Dict[str, str]:
-        return {
-            "name": percentile_ci_label,
-            "type": "number",
-            "sql": dedent(
-                f"""
-                `moz-fx-data-shared-prod`.udf_js.jackknife_percentile_ci(
-                    {{% parameter percentile_conf %}},
-                    STRUCT(
-                        mozfun.hist.merge(
-                          ARRAY_AGG(
-                            ${{TABLE}}.value IGNORE NULLS
-                          )
-                        ).values AS values
-                    )
-                ).{percentile_ci_label}
-            """
-            ),
-        }
 
     @classmethod
     def from_dict(
@@ -98,11 +71,6 @@ def to_lookml(self, bq_client, v1_name: Optional[str]) -> Dict[str, Any]:
                     "name": self.name,
                     "sql_table_name": reference_table,
                     "dimensions": self.dimensions,
-                    "parameters": self.parameters,
-                    "measures": [
-                        self._percentile_measure(label)
-                        for label in self.percentile_ci_labels
-                    ],
                 }
             ]
         }
diff --git a/tests/test_namespaces.py b/tests/test_namespaces.py
@@ -166,9 +166,9 @@ def result(self):
                             "name": "OpMon",
                             "branches": ["enabled", "disabled"],
                             "xaxis": "submission_date",
-                            "probes": [
-                                {"name": "GC_MS", "agg_type": "histogram"},
-                                {"name": "GC_MS_CONTENT", "agg_type": "histogram"},
+                            "summaries": [
+                                {"metric": "GC_MS", "statistic": "mean"},
+                                {"metric": "GC_MS_CONTENT", "statistic": "percentile"},
                             ],
                             "dimensions": {
                                 "cores_count": {"default": "4", "options": ["4", "1"]}
@@ -422,8 +422,14 @@ def test_namespaces_full(
                                     },
                                     "explore": "op_mon",
                                     "group_by_dimension": None,
-                                    "probes": ["GC_MS", "GC_MS_CONTENT"],
-                                    "table": "moz-fx-data-shared-prod.operational_monitoring.op_mon",
+                                    "summaries": [
+                                        {"metric": "GC_MS", "statistic": "mean"},
+                                        {
+                                            "metric": "GC_MS_CONTENT",
+                                            "statistic": "percentile",
+                                        },
+                                    ],
+                                    "table": "moz-fx-data-shared-prod.operational_monitoring.op_mon_statistics",
                                     "xaxis": "submission_date",
                                 }
                             ],
@@ -437,7 +443,10 @@ def test_namespaces_full(
                             "dimensions": {
                                 "cores_count": {"default": "4", "options": ["4", "1"]}
                             },
-                            "probes": ["GC_MS", "GC_MS_CONTENT"],
+                            "summaries": [
+                                {"metric": "GC_MS", "statistic": "mean"},
+                                {"metric": "GC_MS_CONTENT", "statistic": "percentile"},
+                            ],
                             "type": "operational_monitoring_explore",
                             "views": {"base_view": "op_mon"},
                             "xaxis": "submission_date",
@@ -457,7 +466,7 @@ def test_namespaces_full(
                                             "options": ["4", "1"],
                                         }
                                     },
-                                    "table": "moz-fx-data-shared-prod.operational_monitoring.op_mon",
+                                    "table": "moz-fx-data-shared-prod.operational_monitoring.op_mon_statistics",
                                     "xaxis": "submission_date",
                                 }
                             ],
diff --git a/tests/test_operational_monitoring.py b/tests/test_operational_monitoring.py
