diff --git a/cloud/temporal_cloud.json b/cloud/temporal_cloud.json index 238265d..85673d3 100644 --- a/cloud/temporal_cloud.json +++ b/cloud/temporal_cloud.json @@ -1,4 +1,41 @@ { + "__inputs": [ + { + "name": "DS_EXTERNAL_METRICS", + "label": "external metrics", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "11.4.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], "annotations": { "list": [ { @@ -26,32 +63,18 @@ "graphTooltip": 0, "id": null, "links": [], - "liveNow": false, "panels": [ { "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, - "id": 17, + "id": 29, "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "State Transitions", + "title": "Actions", "type": "row" }, { @@ -59,18 +82,20 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "", + "description": "Approximate count of Temporal Cloud Actions per-second. Use `namespace_mode` to filter for active/standby actions for multi-region namespaces. More information about [billable actions](https://docs.temporal.io/cloud/pricing#action).", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -79,6 +104,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -108,7 +134,8 @@ "value": 80 } ] - } + }, + "unit": "APS" }, "overrides": [] }, @@ -118,7 +145,7 @@ "x": 0, "y": 1 }, - "id": 2, + "id": 31, "options": { "legend": { "calcs": [ @@ -137,20 +164,24 @@ "sort": "none" } }, + "pluginVersion": "11.4.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_EXTERNAL_METRICS}" }, - "exemplar": true, - "expr": "sum(rate(temporal_cloud_v0_state_transition_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval])) by (temporal_namespace)", + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(temporal_cloud_v0_total_action_count{temporal_namespace=~\"$temporal_namespace\", namespace_mode=\"active\"}[$__rate_interval])) by (temporal_namespace)", + "instant": false, "interval": "", "legendFormat": "{{temporal_namespace}}", + "range": true, "refId": "A" } ], - "title": "State Transitions", + "title": "Actions", "type": "timeseries" }, { @@ -188,12 +219,13 @@ "x": 12, "y": 1 }, - "id": 10, + "id": 32, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -201,23 +233,27 @@ "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.1.4", + "pluginVersion": "11.4.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_EXTERNAL_METRICS}" }, + "editorMode": "code", "exemplar": true, - "expr": "sum(increase(temporal_cloud_v0_state_transition_count{temporal_namespace=~\"$temporal_namespace\"}[30d])) by (temporal_namespace)", + "expr": "sum(increase(temporal_cloud_v0_total_action_count{temporal_namespace=~\"$temporal_namespace\", namespace_mode=\"active\"}[30d])) by (temporal_namespace)", "interval": "", "legendFormat": "{{temporal_namespace}}", + "range": true, "refId": "A" } ], - "title": "30 Day State Transitions", + "title": "30 Days Actions", "type": "stat" }, { @@ -255,12 +291,13 @@ "x": 16, "y": 1 }, - "id": 14, + "id": 33, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -268,23 +305,28 @@ "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.1.4", + "pluginVersion": "11.4.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_EXTERNAL_METRICS}" }, - "exemplar": true, - "expr": "sum(increase(temporal_cloud_v0_state_transition_count{temporal_namespace=~\"$temporal_namespace\"}[7d])) by (temporal_namespace)", + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(temporal_cloud_v0_total_action_count{temporal_namespace=~\"$temporal_namespace\", namespace_mode=\"active\"}[7d])) by (temporal_namespace)", + "instant": false, "interval": "", "legendFormat": "{{temporal_namespace}}", + "range": true, "refId": "A" } ], - "title": "7 Day State Transitions", + "title": "7 Days Actions", "type": "stat" }, { @@ -322,12 +364,13 @@ "x": 20, "y": 1 }, - "id": 15, + "id": 34, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" @@ -335,23 +378,27 @@ "fields": "", "values": false }, - "textMode": "auto" + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "9.1.4", + "pluginVersion": "11.4.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_EXTERNAL_METRICS}" }, + "editorMode": "code", "exemplar": true, - "expr": "sum(increase(temporal_cloud_v0_state_transition_count{temporal_namespace=~\"$temporal_namespace\"}[1d])) by (temporal_namespace)", + "expr": "sum(increase(temporal_cloud_v0_total_action_count{temporal_namespace=~\"$temporal_namespace\", namespace_mode=\"active\"}[1d])) by (temporal_namespace)", "interval": "", "legendFormat": "{{temporal_namespace}}", + "range": true, "refId": "A" } ], - "title": "1 Day State Transitions", + "title": "1 Day Actions", "type": "stat" }, { @@ -362,9 +409,9 @@ "x": 0, "y": 10 }, - "id": 29, + "id": 21, "panels": [], - "title": "Actions", + "title": "Workflows Overview", "type": "row" }, { @@ -372,17 +419,20 @@ "type": "prometheus", "uid": "${datasource}" }, + "description": "Workflows that successfully completed.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -391,6 +441,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -430,7 +481,7 @@ "x": 0, "y": 11 }, - "id": 31, + "id": 4, "options": { "legend": { "calcs": [ @@ -449,23 +500,21 @@ "sort": "none" } }, + "pluginVersion": "11.4.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_EXTERNAL_METRICS}" }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(temporal_cloud_v0_total_action_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval])) by (temporal_namespace)", - "instant": false, + "exemplar": true, + "expr": "sum(rate(temporal_cloud_v0_workflow_success_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval])) by (temporal_namespace)", "interval": "", "legendFormat": "{{temporal_namespace}}", - "range": true, "refId": "A" } ], - "title": "Actions", + "title": "Workflow Success", "type": "timeseries" }, { @@ -473,11 +522,44 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "", + "description": "Workflows that timed out before completing execution.", "fieldConfig": { "defaults": { "color": { - "mode": "continuous-GrYlRd" + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "thresholds": { @@ -492,59 +574,97 @@ "value": 80 } ] - }, - "unit": "short" + } }, "overrides": [] }, "gridPos": { "h": 9, - "w": 4, + "w": 12, "x": 12, "y": 11 }, - "id": 32, + "id": 41, "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { + "legend": { "calcs": [ - "lastNotNull" + "lastNotNull", + "min", + "max" ], - "fields": "", - "values": false + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true }, - "textMode": "auto" + "tooltip": { + "mode": "single", + "sort": "none" + } }, - "pluginVersion": "9.1.4", + "pluginVersion": "11.4.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_EXTERNAL_METRICS}" }, + "editorMode": "code", "exemplar": true, - "expr": "sum(increase(temporal_cloud_v0_total_action_count{temporal_namespace=~\"$temporal_namespace\"}[30d])) by (temporal_namespace)", + "expr": "sum(rate(temporal_cloud_v0_workflow_timeout_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval])) by (temporal_namespace)", "interval": "", "legendFormat": "{{temporal_namespace}}", + "range": true, "refId": "A" } ], - "title": "30 Days Actions", - "type": "stat" + "title": "Workflow Timeouts", + "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "description": "", + "description": "Workflows that failed before completion.", "fieldConfig": { "defaults": { "color": { - "mode": "continuous-GrYlRd" + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "thresholds": { @@ -559,159 +679,71 @@ "value": 80 } ] - }, - "unit": "short" + } }, "overrides": [] }, "gridPos": { "h": 9, - "w": 4, - "x": 16, - "y": 11 + "w": 12, + "x": 0, + "y": 20 }, - "id": 33, + "id": 12, "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { + "legend": { "calcs": [ - "lastNotNull" + "lastNotNull", + "min", + "max" ], - "fields": "", - "values": false + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true }, - "textMode": "auto" + "tooltip": { + "mode": "single", + "sort": "none" + } }, - "pluginVersion": "9.1.4", + "pluginVersion": "11.4.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_EXTERNAL_METRICS}" }, - "exemplar": true, - "expr": "sum(increase(temporal_cloud_v0_total_action_count{temporal_namespace=~\"$temporal_namespace\"}[7d])) by (temporal_namespace)", + "exemplar": false, + "expr": "sum(rate(temporal_cloud_v0_workflow_failed_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval])) by (temporal_namespace)", "interval": "", "legendFormat": "{{temporal_namespace}}", "refId": "A" } ], - "title": "7 Days Actions", - "type": "stat" + "title": "Workflow Failed", + "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "description": "", + "description": "Workflow Executions that were Continued-As-New from a past execution.", "fieldConfig": { "defaults": { "color": { - "mode": "continuous-GrYlRd" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 4, - "x": 20, - "y": 11 - }, - "id": 34, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.1.4", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "exemplar": true, - "expr": "sum(increase(temporal_cloud_v0_total_action_count{temporal_namespace=~\"$temporal_namespace\"}[1d])) by (temporal_namespace)", - "interval": "", - "legendFormat": "{{temporal_namespace}}", - "refId": "A" - } - ], - "title": "1 Day Actions", - "type": "stat" - }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 20 - }, - "id": 21, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Workflows Overview", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -720,6 +752,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -756,10 +789,10 @@ "gridPos": { "h": 9, "w": 12, - "x": 0, - "y": 21 + "x": 12, + "y": 20 }, - "id": 4, + "id": 13, "options": { "legend": { "calcs": [ @@ -778,20 +811,21 @@ "sort": "none" } }, + "pluginVersion": "11.4.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_EXTERNAL_METRICS}" }, - "exemplar": true, - "expr": "sum(rate(temporal_cloud_v0_workflow_success_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval])) by (temporal_namespace)", + "exemplar": false, + "expr": "sum(rate(temporal_cloud_v0_workflow_continued_as_new_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval])) by (temporal_namespace)", "interval": "", "legendFormat": "{{temporal_namespace}}", "refId": "A" } ], - "title": "Workflow Success", + "title": "Workflow Continued as New", "type": "timeseries" }, { @@ -799,18 +833,20 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "", + "description": "Workflows terminated before completing execution.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -819,6 +855,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -855,10 +892,10 @@ "gridPos": { "h": 9, "w": 12, - "x": 12, - "y": 21 + "x": 0, + "y": 29 }, - "id": 41, + "id": 48, "options": { "legend": { "calcs": [ @@ -877,22 +914,23 @@ "sort": "none" } }, + "pluginVersion": "11.4.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_EXTERNAL_METRICS}" }, "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(temporal_cloud_v0_workflow_timeout_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval])) by (temporal_namespace)", + "exemplar": false, + "expr": "sum(rate(temporal_cloud_v0_workflow_terminate_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval])) by (temporal_namespace)", "interval": "", "legendFormat": "{{temporal_namespace}}", "range": true, "refId": "A" } ], - "title": "Workflow Timeouts", + "title": "Workflow Terminations", "type": "timeseries" }, { @@ -900,18 +938,20 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "", + "description": "Workflows canceled before completing execution.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -920,6 +960,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -956,10 +997,10 @@ "gridPos": { "h": 9, "w": 12, - "x": 0, - "y": 30 + "x": 12, + "y": 29 }, - "id": 12, + "id": 49, "options": { "legend": { "calcs": [ @@ -978,39 +1019,57 @@ "sort": "none" } }, + "pluginVersion": "11.4.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_EXTERNAL_METRICS}" }, + "editorMode": "code", "exemplar": false, - "expr": "sum(rate(temporal_cloud_v0_workflow_failed_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval])) by (temporal_namespace)", + "expr": "sum(rate(temporal_cloud_v0_workflow_cancel_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval])) by (temporal_namespace)", "interval": "", "legendFormat": "{{temporal_namespace}}", + "range": true, "refId": "A" } ], - "title": "Workflow Failed", + "title": "Workflow Cancellations", "type": "timeseries" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 38 + }, + "id": 40, + "panels": [], + "title": "Schedules", + "type": "row" + }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "description": "", + "description": "Successful execution of a Scheduled Workflow.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1019,6 +1078,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1055,10 +1115,10 @@ "gridPos": { "h": 9, "w": 12, - "x": 12, - "y": 30 + "x": 0, + "y": 39 }, - "id": 13, + "id": 11, "options": { "legend": { "calcs": [ @@ -1077,51 +1137,44 @@ "sort": "none" } }, + "pluginVersion": "11.4.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_EXTERNAL_METRICS}" }, + "editorMode": "code", "exemplar": false, - "expr": "sum(rate(temporal_cloud_v0_workflow_continued_as_new_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval])) by (temporal_namespace)", + "expr": "sum(rate(temporal_cloud_v0_schedule_action_success_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval])) by (temporal_namespace)", "interval": "", "legendFormat": "{{temporal_namespace}}", + "range": true, "refId": "A" } ], - "title": "Workflow Continued as New", + "title": "Successful Schedule Actions", "type": "timeseries" }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 39 - }, - "id": 40, - "panels": [], - "title": "Schedules", - "type": "row" - }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "When average schedule run length is greater than average schedule interval while a buffer_all overlap policy is configured.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1130,6 +1183,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1165,11 +1219,11 @@ }, "gridPos": { "h": 9, - "w": 12, - "x": 0, - "y": 40 + "w": 4, + "x": 12, + "y": 39 }, - "id": 11, + "id": 42, "options": { "legend": { "calcs": [ @@ -1188,22 +1242,23 @@ "sort": "none" } }, + "pluginVersion": "11.4.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_EXTERNAL_METRICS}" }, "editorMode": "code", "exemplar": false, - "expr": "sum(rate(temporal_cloud_v0_schedule_action_success_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval])) by (temporal_namespace)", + "expr": "sum(rate(temporal_cloud_v0_schedule_buffer_overruns_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval])) by (temporal_namespace)", "interval": "", "legendFormat": "{{temporal_namespace}}", "range": true, "refId": "A" } ], - "title": "Successful Schedule Actions", + "title": "Schedule Buffer Overruns", "type": "timeseries" }, { @@ -1211,17 +1266,20 @@ "type": "prometheus", "uid": "${datasource}" }, + "description": "Skipped Scheduled executions when Workflows were delayed longer than the catchup window.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1230,6 +1288,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1266,10 +1325,10 @@ "gridPos": { "h": 9, "w": 4, - "x": 12, - "y": 40 + "x": 16, + "y": 39 }, - "id": 42, + "id": 43, "options": { "legend": { "calcs": [ @@ -1288,22 +1347,23 @@ "sort": "none" } }, + "pluginVersion": "11.4.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_EXTERNAL_METRICS}" }, "editorMode": "code", "exemplar": false, - "expr": "sum(rate(temporal_cloud_v0_schedule_buffer_overruns_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval])) by (temporal_namespace)", + "expr": "sum(rate(temporal_cloud_v0_schedule_missed_catchup_window_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval])) by (temporal_namespace)", "interval": "", "legendFormat": "{{temporal_namespace}}", "range": true, "refId": "A" } ], - "title": "Schedule Buffer Overruns", + "title": "Schedule Missed Catchup Windows", "type": "timeseries" }, { @@ -1311,17 +1371,20 @@ "type": "prometheus", "uid": "${datasource}" }, + "description": "Workflows that were delayed due to exceeding a rate limit.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1330,6 +1393,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1366,10 +1430,10 @@ "gridPos": { "h": 9, "w": 4, - "x": 16, - "y": 40 + "x": 20, + "y": 39 }, - "id": 43, + "id": 44, "options": { "legend": { "calcs": [ @@ -1388,40 +1452,57 @@ "sort": "none" } }, + "pluginVersion": "11.4.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_EXTERNAL_METRICS}" }, "editorMode": "code", "exemplar": false, - "expr": "sum(rate(temporal_cloud_v0_schedule_missed_catchup_window_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval])) by (temporal_namespace)", + "expr": "sum(rate(temporal_cloud_v0_schedule_rate_limited_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval])) by (temporal_namespace)", "interval": "", "legendFormat": "{{temporal_namespace}}", "range": true, "refId": "A" } ], - "title": "Schedule Missed Catchup Windows", + "title": "Schedules Rate Limited", "type": "timeseries" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 23, + "panels": [], + "title": "Workers and Tasks", + "type": "row" + }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "The Sync Match Rate measures the rate of Tasks that can be delivered to Workers without having to be persisted (Workers are up and available to pick them up) to the rate of all delivered Tasks. [more info](https://docs.temporal.io/production-deployment/cloud/worker-health#sync-match-rate)\n\nShould always be near 100%.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1430,6 +1511,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1459,17 +1541,18 @@ "value": 80 } ] - } + }, + "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 9, - "w": 4, - "x": 20, - "y": 40 + "w": 12, + "x": 0, + "y": 49 }, - "id": 44, + "id": 37, "options": { "legend": { "calcs": [ @@ -1488,66 +1571,44 @@ "sort": "none" } }, + "pluginVersion": "11.4.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_EXTERNAL_METRICS}" }, "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(temporal_cloud_v0_schedule_rate_limited_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval])) by (temporal_namespace)", + "exemplar": true, + "expr": "sum by(temporal_namespace) (\n rate(\n temporal_cloud_v0_poll_success_sync_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval]\n )\n )\n/\nsum by(temporal_namespace) (\n rate(\n temporal_cloud_v0_poll_success_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval]\n )\n )", "interval": "", "legendFormat": "{{temporal_namespace}}", "range": true, "refId": "A" } ], - "title": "Schedules Rate Limited", + "title": "Sync Match Rate", "type": "timeseries" }, - { - "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 49 - }, - "id": 19, - "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Temporal Service Requests", - "type": "row" - }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "Tasks that are successfully matched to a poller.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1556,6 +1617,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1592,10 +1654,10 @@ "gridPos": { "h": 9, "w": 12, - "x": 0, - "y": 50 + "x": 12, + "y": 49 }, - "id": 6, + "id": 24, "options": { "legend": { "calcs": [ @@ -1614,22 +1676,21 @@ "sort": "none" } }, + "pluginVersion": "11.4.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_EXTERNAL_METRICS}" }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(rate(temporal_cloud_v0_frontend_service_request_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval])) by (temporal_namespace,operation)", + "exemplar": false, + "expr": "sum(rate(temporal_cloud_v0_poll_success_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval])) by (temporal_namespace)", "interval": "", - "legendFormat": "{{temporal_namespace}} - {{operation}}", - "range": true, + "legendFormat": "{{temporal_namespace}}", "refId": "A" } ], - "title": "Frontend Service Requests", + "title": "Worker Poll Success", "type": "timeseries" }, { @@ -1637,17 +1698,20 @@ "type": "prometheus", "uid": "${datasource}" }, + "description": "Tasks that are successfully sync matched to a poller.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1656,6 +1720,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1692,10 +1757,10 @@ "gridPos": { "h": 9, "w": 12, - "x": 12, - "y": 50 + "x": 0, + "y": 58 }, - "id": 8, + "id": 25, "options": { "legend": { "calcs": [ @@ -1714,51 +1779,42 @@ "sort": "none" } }, + "pluginVersion": "11.4.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_EXTERNAL_METRICS}" }, "exemplar": true, - "expr": "sum(rate(temporal_cloud_v0_frontend_service_error_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval])) by (temporal_namespace,operation)", + "expr": "sum(rate(temporal_cloud_v0_poll_success_sync_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval])) by (temporal_namespace)", "interval": "", - "legendFormat": "{{temporal_namespace}} - {{operation}}", + "legendFormat": "{{temporal_namespace}}", "refId": "A" } ], - "title": "Frontend Service Errors", + "title": "Worker Poll Success Sync", "type": "timeseries" }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 59 - }, - "id": 36, - "panels": [], - "title": "Temporal Service Latency", - "type": "row" - }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "description": "Tasks that are successfully async matched to a poller.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1767,6 +1823,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1796,60 +1853,52 @@ "value": 80 } ] - }, - "unit": "s" + } }, "overrides": [] }, "gridPos": { - "h": 10, - "w": 8, - "x": 0, - "y": 60 + "h": 9, + "w": 12, + "x": 12, + "y": 58 }, - "id": 30, + "id": 27, "options": { "legend": { "calcs": [ "lastNotNull", - "mean", + "min", "max" ], "displayMode": "table", "placement": "bottom", - "showLegend": true + "showLegend": true, + "sortBy": "Max", + "sortDesc": true }, "tooltip": { "mode": "single", "sort": "none" } }, + "pluginVersion": "11.4.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_EXTERNAL_METRICS}" }, "editorMode": "code", - "expr": "histogram_quantile(0.5, sum(rate(temporal_cloud_v0_service_latency_bucket{temporal_namespace=~\"$temporal_namespace\", operation=\"StartWorkflowExecution\"}[$__rate_interval])) by (temporal_namespace, operation, le))", - "legendFormat": "{{ temporal_namespace }} - 50th Percentile", + "exemplar": false, + "expr": "sum(rate(temporal_cloud_v0_poll_success_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval])) by (temporal_namespace) - sum(rate(temporal_cloud_v0_poll_success_sync_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval])) by (temporal_namespace)", + "interval": "", + "legendFormat": "{{temporal_namespace}}", "range": true, "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(temporal_cloud_v0_service_latency_bucket{temporal_namespace=~\"$temporal_namespace\", operation=\"StartWorkflowExecution\"}[$__rate_interval])) by (temporal_namespace, operation, le))", - "hide": false, - "legendFormat": "{{ temporal_namespace }} - 99th Percentile", - "range": true, - "refId": "B" } ], - "title": "StartWorkflowExecution Latency", + "title": "Worker Poll Success Async", "type": "timeseries" }, { @@ -1857,17 +1906,20 @@ "type": "prometheus", "uid": "${datasource}" }, + "description": "Successful task matching to poller\n/\nAll polling requests (including timeouts)", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1876,6 +1928,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1906,59 +1959,52 @@ } ] }, - "unit": "s" + "unit": "percentunit" }, "overrides": [] }, "gridPos": { - "h": 10, - "w": 8, - "x": 8, - "y": 60 + "h": 9, + "w": 12, + "x": 0, + "y": 67 }, - "id": 1, + "id": 38, "options": { "legend": { "calcs": [ "lastNotNull", - "mean", + "min", "max" ], "displayMode": "table", "placement": "bottom", - "showLegend": true + "showLegend": true, + "sortBy": "Max", + "sortDesc": true }, "tooltip": { "mode": "single", "sort": "none" } }, + "pluginVersion": "11.4.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_EXTERNAL_METRICS}" }, "editorMode": "code", - "expr": "histogram_quantile(0.5, sum(rate(temporal_cloud_v0_service_latency_bucket{temporal_namespace=~\"$temporal_namespace\", operation=\"SignalWorkflowExecution\"}[$__rate_interval])) by (temporal_namespace, operation, le))", - "legendFormat": "{{ temporal_namespace }} - 50th Percentile", + "exemplar": true, + "expr": "(\n (\n sum by(temporal_namespace) (\n rate(\n temporal_cloud_v0_poll_success_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval]\n )\n )\n +\n sum by(temporal_namespace) (\n rate(\n temporal_cloud_v0_poll_success_sync_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval]\n )\n )\n )\n /\n (\n (\n sum by(temporal_namespace) (\n rate(\n temporal_cloud_v0_poll_success_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval]\n )\n )\n +\n sum by(temporal_namespace) (\n rate(\n temporal_cloud_v0_poll_success_sync_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval]\n )\n )\n )\n +\n sum by(temporal_namespace) (\n rate(\n temporal_cloud_v0_poll_timeout_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval]\n )\n )\n )\n)", + "interval": "", + "legendFormat": "{{temporal_namespace}}", "range": true, "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(temporal_cloud_v0_service_latency_bucket{temporal_namespace=~\"$temporal_namespace\", operation=\"SignalWorkflowExecution\"}[$__rate_interval])) by (temporal_namespace, operation, le))", - "hide": false, - "legendFormat": "{{ temporal_namespace }} - 99th Percentile", - "range": true, - "refId": "B" } ], - "title": "SignalWorkflowExecution Latency", + "title": "Poll Success Rate", "type": "timeseries" }, { @@ -1966,17 +2012,20 @@ "type": "prometheus", "uid": "${datasource}" }, + "description": "When no tasks are available for a poller before timing out. If this is high, you may have too many pollers.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1985,6 +2034,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2014,86 +2064,63 @@ "value": 80 } ] - }, - "unit": "s" + } }, "overrides": [] }, "gridPos": { - "h": 10, - "w": 8, - "x": 16, - "y": 60 + "h": 9, + "w": 12, + "x": 12, + "y": 67 }, - "id": 35, + "id": 26, "options": { "legend": { "calcs": [ "lastNotNull", - "mean", + "min", "max" ], "displayMode": "table", "placement": "bottom", - "showLegend": true + "showLegend": true, + "sortBy": "Max", + "sortDesc": true }, "tooltip": { "mode": "single", "sort": "none" } }, + "pluginVersion": "11.4.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_EXTERNAL_METRICS}" }, - "editorMode": "code", - "expr": "histogram_quantile(0.5, sum(rate(temporal_cloud_v0_service_latency_bucket{temporal_namespace=~\"$temporal_namespace\", operation=\"SignalWithStartWorkflowExecution\"}[$__rate_interval])) by (temporal_namespace, operation, le))", - "legendFormat": "{{ temporal_namespace }} - 50th Percentile", - "range": true, + "exemplar": false, + "expr": "sum(rate(temporal_cloud_v0_poll_timeout_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval])) by (temporal_namespace)", + "interval": "", + "legendFormat": "{{temporal_namespace}}", "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(temporal_cloud_v0_service_latency_bucket{temporal_namespace=~\"$temporal_namespace\", operation=\"SignalWithStartWorkflowExecution\"}[$__rate_interval])) by (temporal_namespace, operation, le))", - "hide": false, - "legendFormat": "{{ temporal_namespace }} - 99th Percentile", - "range": true, - "refId": "B" } ], - "title": "SignalWithStartWorkflowExecution Latency", + "title": "Worker Poll Timeout", "type": "timeseries" }, { "collapsed": false, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 70 + "y": 76 }, - "id": 23, + "id": 19, "panels": [], - "targets": [ - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "refId": "A" - } - ], - "title": "Workers and Tasks", + "title": "Temporal Cloud gRPC Requests", "type": "row" }, { @@ -2101,18 +2128,20 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "", + "description": "A count of gRPC requests received aggregated by operation.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2121,6 +2150,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2158,9 +2188,9 @@ "h": 9, "w": 12, "x": 0, - "y": 71 + "y": 77 }, - "id": 24, + "id": 6, "options": { "legend": { "calcs": [ @@ -2179,20 +2209,23 @@ "sort": "none" } }, + "pluginVersion": "11.4.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_EXTERNAL_METRICS}" }, - "exemplar": false, - "expr": "sum(rate(temporal_cloud_v0_poll_success_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval])) by (temporal_namespace)", + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(temporal_cloud_v0_frontend_service_request_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval])) by (temporal_namespace,operation)", "interval": "", - "legendFormat": "{{temporal_namespace}}", + "legendFormat": "{{temporal_namespace}} - {{operation}}", + "range": true, "refId": "A" } ], - "title": "Worker Poll Success", + "title": "Frontend Service Requests", "type": "timeseries" }, { @@ -2200,18 +2233,20 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "", + "description": "Count of gRPC errors returned aggregated by operation, including resource exhausted errors.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2220,6 +2255,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2257,9 +2293,9 @@ "h": 9, "w": 12, "x": 12, - "y": 71 + "y": 77 }, - "id": 26, + "id": 8, "options": { "legend": { "calcs": [ @@ -2278,39 +2314,71 @@ "sort": "none" } }, + "pluginVersion": "11.4.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_EXTERNAL_METRICS}" }, - "exemplar": false, - "expr": "sum(rate(temporal_cloud_v0_poll_timeout_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval])) by (temporal_namespace)", + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(temporal_cloud_v0_frontend_service_error_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval])) by (temporal_namespace,operation)", + "hide": false, "interval": "", - "legendFormat": "{{temporal_namespace}}", + "legendFormat": "{{temporal_namespace}} - {{operation}}", + "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(temporal_cloud_v0_resource_exhausted_error_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval])) by (temporal_namespace,operation,resource_exhausted_cause)", + "hide": false, + "instant": false, + "legendFormat": "Rate Limited - {{ resource_exhausted_cause }} / {{temporal_namespace}}", + "range": true, + "refId": "B" } ], - "title": "Worker Poll Timeout", + "title": "Frontend Service Errors", "type": "timeseries" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 86 + }, + "id": 36, + "panels": [], + "title": "Temporal Service Latency", + "type": "row" + }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_EXTERNAL_METRICS}" }, - "description": "", + "description": "Should generally be under 200ms", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2319,6 +2387,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2348,28 +2417,29 @@ "value": 80 } ] - } + }, + "unit": "s" }, "overrides": [] }, "gridPos": { - "h": 9, - "w": 12, + "h": 10, + "w": 8, "x": 0, - "y": 80 + "y": 87 }, - "id": 25, + "id": 30, "options": { "legend": { "calcs": [ "lastNotNull", - "min", + "mean", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, - "sortBy": "Max", + "sortBy": "Mean", "sortDesc": true }, "tooltip": { @@ -2377,20 +2447,33 @@ "sort": "none" } }, + "pluginVersion": "11.4.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "exemplar": true, - "expr": "sum(rate(temporal_cloud_v0_poll_success_sync_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval])) by (temporal_namespace)", - "interval": "", - "legendFormat": "{{temporal_namespace}}", + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum(rate(temporal_cloud_v0_service_latency_bucket{temporal_namespace=~\"$temporal_namespace\", operation=\"StartWorkflowExecution\"}[$__rate_interval])) by (temporal_namespace, operation, le))", + "legendFormat": "{{ temporal_namespace }} - 50th Percentile", + "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_EXTERNAL_METRICS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(temporal_cloud_v0_service_latency_bucket{temporal_namespace=~\"$temporal_namespace\", operation=\"StartWorkflowExecution\"}[$__rate_interval])) by (temporal_namespace, operation, le))", + "hide": false, + "legendFormat": "{{ temporal_namespace }} - 99th Percentile", + "range": true, + "refId": "B" } ], - "title": "Worker Poll Success Sync", + "title": "StartWorkflowExecution Latency", "type": "timeseries" }, { @@ -2398,18 +2481,20 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "", + "description": "Should generally be under 200ms", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2418,6 +2503,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2447,68 +2533,82 @@ "value": 80 } ] - } + }, + "unit": "s" }, "overrides": [] }, "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 80 + "h": 10, + "w": 8, + "x": 8, + "y": 87 }, - "id": 27, + "id": 1, "options": { "legend": { "calcs": [ "lastNotNull", - "min", + "mean", "max" ], "displayMode": "table", "placement": "bottom", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true + "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, + "pluginVersion": "11.4.0", "targets": [ { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_EXTERNAL_METRICS}" }, - "exemplar": false, - "expr": "sum(rate(temporal_cloud_v0_poll_success_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval])) by (temporal_namespace) - sum(rate(temporal_cloud_v0_poll_success_sync_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval])) by (temporal_namespace)", - "interval": "", - "legendFormat": "{{temporal_namespace}}", + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum(rate(temporal_cloud_v0_service_latency_bucket{temporal_namespace=~\"$temporal_namespace\", operation=\"SignalWorkflowExecution\"}[$__rate_interval])) by (temporal_namespace, operation, le))", + "legendFormat": "{{ temporal_namespace }} - 50th Percentile", + "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(temporal_cloud_v0_service_latency_bucket{temporal_namespace=~\"$temporal_namespace\", operation=\"SignalWorkflowExecution\"}[$__rate_interval])) by (temporal_namespace, operation, le))", + "hide": false, + "legendFormat": "{{ temporal_namespace }} - 99th Percentile", + "range": true, + "refId": "B" } ], - "title": "Worker Poll Success Async", + "title": "SignalWorkflowExecution Latency", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_EXTERNAL_METRICS}" }, - "description": "", + "description": "Should generally be under 200ms", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2517,6 +2617,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2547,17 +2648,257 @@ } ] }, - "unit": "percentunit" + "unit": "s" }, "overrides": [] }, "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 89 + "h": 10, + "w": 8, + "x": 16, + "y": 87 }, - "id": 37, + "id": 35, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum(rate(temporal_cloud_v0_service_latency_bucket{temporal_namespace=~\"$temporal_namespace\", operation=\"SignalWithStartWorkflowExecution\"}[$__rate_interval])) by (temporal_namespace, operation, le))", + "legendFormat": "{{ temporal_namespace }} - 50th Percentile", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_EXTERNAL_METRICS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(temporal_cloud_v0_service_latency_bucket{temporal_namespace=~\"$temporal_namespace\", operation=\"SignalWithStartWorkflowExecution\"}[$__rate_interval])) by (temporal_namespace, operation, le))", + "hide": false, + "legendFormat": "{{ temporal_namespace }} - 99th Percentile", + "range": true, + "refId": "B" + } + ], + "title": "SignalWithStartWorkflowExecution Latency", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 97 + }, + "id": 45, + "panels": [], + "title": "Multi-region Namespaces", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "The transmission delay of Workflow updates and history events from the primary to the replica. [more info](https://docs.temporal.io/cloud/high-availability/monitor#metrics)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 98 + }, + "id": 47, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_EXTERNAL_METRICS}" + }, + "editorMode": "code", + "expr": "sum(rate(temporal_cloud_v0_replication_lag_sum{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval])) by (temporal_namespace)\n/\nsum(rate(temporal_cloud_v0_replication_lag_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval])) by (temporal_namespace)", + "legendFormat": "{{ temporal_namespace }} - p50", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(temporal_cloud_v0_replication_lag_bucket{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval])) by (temporal_namespace, le))", + "hide": false, + "legendFormat": "{{ temporal_namespace }} - p99", + "range": true, + "refId": "B" + } + ], + "title": "Replication Lag", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_EXTERNAL_METRICS}" + }, + "description": "[Billable actions](https://docs.temporal.io/cloud/pricing#action) split across active/standby namespaces for multi region namespaces.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 98 + }, + "id": 46, "options": { "legend": { "calcs": [ @@ -2576,6 +2917,7 @@ "sort": "none" } }, + "pluginVersion": "11.4.0", "targets": [ { "datasource": { @@ -2583,34 +2925,50 @@ "uid": "${datasource}" }, "editorMode": "code", - "exemplar": true, - "expr": "sum by(temporal_namespace) (\n rate(\n temporal_cloud_v0_poll_success_sync_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval]\n )\n )\n/\nsum by(temporal_namespace) (\n rate(\n temporal_cloud_v0_poll_success_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval]\n )\n )", + "exemplar": false, + "expr": "sum(rate(temporal_cloud_v0_total_action_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval])) by (temporal_namespace, namespace_mode)", + "instant": false, "interval": "", - "legendFormat": "{{temporal_namespace}}", + "legendFormat": "{{temporal_namespace}} - {{ namespace_mode }}", "range": true, "refId": "A" } ], - "title": "Sync Match Rate", + "title": "Actions", "type": "timeseries" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 107 + }, + "id": 17, + "panels": [], + "title": "State Transitions", + "type": "row" + }, { "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_EXTERNAL_METRICS}" }, - "description": "", + "description": "Count of state transitions for each Namespace.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2619,6 +2977,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -2655,10 +3014,10 @@ "gridPos": { "h": 9, "w": 12, - "x": 12, - "y": 89 + "x": 0, + "y": 108 }, - "id": 38, + "id": 2, "options": { "legend": { "calcs": [ @@ -2677,58 +3036,257 @@ "sort": "none" } }, + "pluginVersion": "11.4.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "editorMode": "code", "exemplar": true, - "expr": "(\n (\n sum by(temporal_namespace) (\n rate(\n temporal_cloud_v0_poll_success_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval]\n )\n )\n +\n sum by(temporal_namespace) (\n rate(\n temporal_cloud_v0_poll_success_sync_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval]\n )\n )\n )\n /\n (\n (\n sum by(temporal_namespace) (\n rate(\n temporal_cloud_v0_poll_success_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval]\n )\n )\n +\n sum by(temporal_namespace) (\n rate(\n temporal_cloud_v0_poll_success_sync_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval]\n )\n )\n )\n +\n sum by(temporal_namespace) (\n rate(\n temporal_cloud_v0_poll_timeout_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval]\n )\n )\n )\n)", + "expr": "sum(rate(temporal_cloud_v0_state_transition_count{temporal_namespace=~\"$temporal_namespace\"}[$__rate_interval])) by (temporal_namespace)", "interval": "", "legendFormat": "{{temporal_namespace}}", - "range": true, "refId": "A" } ], - "title": "Poll Success Rate", + "title": "State Transitions", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_EXTERNAL_METRICS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 4, + "x": 12, + "y": 108 + }, + "id": 10, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(increase(temporal_cloud_v0_state_transition_count{temporal_namespace=~\"$temporal_namespace\"}[30d])) by (temporal_namespace)", + "interval": "", + "legendFormat": "{{temporal_namespace}}", + "refId": "A" + } + ], + "title": "30 Day State Transitions", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_EXTERNAL_METRICS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 4, + "x": 16, + "y": 108 + }, + "id": 14, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(increase(temporal_cloud_v0_state_transition_count{temporal_namespace=~\"$temporal_namespace\"}[7d])) by (temporal_namespace)", + "interval": "", + "legendFormat": "{{temporal_namespace}}", + "refId": "A" + } + ], + "title": "7 Day State Transitions", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_EXTERNAL_METRICS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 4, + "x": 20, + "y": 108 + }, + "id": 15, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": true, + "expr": "sum(increase(temporal_cloud_v0_state_transition_count{temporal_namespace=~\"$temporal_namespace\"}[1d])) by (temporal_namespace)", + "interval": "", + "legendFormat": "{{temporal_namespace}}", + "refId": "A" + } + ], + "title": "1 Day State Transitions", + "type": "stat" } ], "refresh": false, - "schemaVersion": 37, - "style": "dark", + "schemaVersion": 40, "tags": [], "templating": { "list": [ { - "current": { - "selected": true, - "text": "default", - "value": "default" - }, - "hide": 0, + "current": {}, "includeAll": false, "label": "datasource", - "multi": false, "name": "datasource", "options": [], "query": "prometheus", - "queryValue": "", "refresh": 1, "regex": "", - "skipUrlSync": false, "type": "datasource" }, { "current": {}, "datasource": { "type": "prometheus", - "uid": "${datasource}" + "uid": "${DS_EXTERNAL_METRICS}" }, "definition": "label_values(temporal_namespace)", - "hide": 0, "includeAll": false, "multi": true, "name": "temporal_namespace", @@ -2739,8 +3297,6 @@ }, "refresh": 1, "regex": "", - "skipUrlSync": false, - "sort": 0, "type": "query" } ] @@ -2752,6 +3308,7 @@ "timepicker": {}, "timezone": "", "title": "Temporal Cloud External Metrics", + "uid": "fem3anbu421hcc", "version": 1, "weekStart": "" -} +} \ No newline at end of file