Skip to content

Commit 350b94d

Browse files
authored
Merge pull request #753 from openedx/bmtcril/fail_on_performance_metrics_error
fix: Fail Tutor command on performance_metrics failure
2 parents 7aca092 + b77ea4d commit 350b94d

5 files changed

Lines changed: 122 additions & 41 deletions

File tree

.github/workflows/integration-test.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -139,7 +139,7 @@ jobs:
139139
make extract_translations
140140
tutor dev do import-assets
141141
- name: Performance metrics
142-
run: tutor dev do performance-metrics
142+
run: tutor dev do performance-metrics --fail_on_error
143143
- name: Tutor stop
144144
run: tutor dev stop
145145

tutoraspects/commands_v1.py

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -137,16 +137,21 @@ def import_assets() -> list[tuple[str, str]]:
137137
@click.option(
138138
"--print_sql", is_flag=True, default=False, help="Print the SQL that was run."
139139
)
140-
def performance_metrics(course_key, print_sql) -> list[tuple[str, str]]:
140+
@click.option(
141+
"--fail_on_error", is_flag=True, default=False, help="Allow errors to fail the run."
142+
)
143+
def performance_metrics(course_key, print_sql, fail_on_error) -> list[tuple[str, str]]:
141144
"""
142145
Job to measure performance metrics of charts and its queries in Superset and ClickHouse.
143146
"""
144147
options = f"--course_key {course_key}" if course_key else ""
145148
options += " --print_sql" if print_sql else ""
149+
options += " --fail_on_error" if fail_on_error else ""
146150

147151
return [
148152
(
149153
"superset",
154+
"set -e && "
150155
"echo 'Performance...' && "
151156
f"python /app/pythonpath/performance_metrics.py {options} &&"
152157
"echo 'Done!';",

tutoraspects/templates/aspects/apps/superset/pythonpath/performance_metrics.py

Lines changed: 34 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -46,22 +46,29 @@
4646
)
4747

4848
@click.command()
49-
@click.option("--course_key", default="", help="A course_key to apply as a filter.")
49+
@click.option(
50+
"--course_key",
51+
default="",
52+
help="A course_key to apply as a filter, you must include the 'course-v1:'.")
5053
@click.option(
5154
"--print_sql",
5255
is_flag=True,
5356
default=False,
5457
help="Whether to print the SQL run."
5558
)
56-
def performance_metrics(course_key, print_sql):
59+
@click.option(
60+
"--fail_on_error", is_flag=True, default=False, help="Allow errors to fail the run."
61+
)
62+
def performance_metrics(course_key, print_sql, fail_on_error):
5763
"""
5864
Measure the performance of the dashboard.
5965
"""
6066
# Mock the client name to identify the queries in the clickhouse system.query_log
6167
# table by by the http_user_agent field.
6268
extra_filters = []
6369
if course_key:
64-
extra_filters+=[{"col":"course_key","op":"==","val":course_key}]
70+
extra_filters += [{"col": "course_key", "op": "==", "val": course_key}]
71+
6572
with patch("clickhouse_connect.common.build_client_name") as mock_build_client_name:
6673
mock_build_client_name.return_value = RUN_ID
6774
embedable_dashboards = {{SUPERSET_EMBEDDABLE_DASHBOARDS}}
@@ -75,8 +82,12 @@ def performance_metrics(course_key, print_sql):
7582
for dashboard in dashboards:
7683
logger.info(f"Dashboard: {dashboard.slug}")
7784
for slice in dashboard.slices:
78-
query_context = get_slice_query_context(slice, query_contexts)
79-
result = measure_chart(slice, query_context)
85+
query_context = get_slice_query_context(
86+
slice,
87+
query_contexts,
88+
extra_filters
89+
)
90+
result = measure_chart(slice, query_context, fail_on_error)
8091
if not result:
8192
continue
8293
for query in result["queries"]:
@@ -87,7 +98,7 @@ def performance_metrics(course_key, print_sql):
8798

8899
logger.info("Waiting for clickhouse log...")
89100
time.sleep(20)
90-
get_query_log_from_clickhouse(report, query_contexts, print_sql)
101+
get_query_log_from_clickhouse(report, query_contexts, print_sql, fail_on_error)
91102
return report
92103

93104

@@ -108,7 +119,11 @@ def get_query_contexts_from_assets():
108119
logger.info(f"Found {len(query_contexts)} query contexts")
109120
return query_contexts
110121

111-
def get_slice_query_context(slice, query_contexts, extra_filters=[]):
122+
123+
def get_slice_query_context(slice, query_contexts, extra_filters=None):
124+
if not extra_filters:
125+
extra_filters = []
126+
112127
query_context = query_contexts.get(str(slice.uuid), {})
113128
if not query_context:
114129
logger.info(f"SLICE {slice} has no query context! {slice.uuid}")
@@ -128,12 +143,12 @@ def get_slice_query_context(slice, query_contexts, extra_filters=[]):
128143

129144
if extra_filters:
130145
for query in query_context["queries"]:
131-
query["filters"]+=extra_filters
146+
query["filters"] += extra_filters
132147

133148
return query_context
134149

135150

136-
def measure_chart(slice, query_context):
151+
def measure_chart(slice, query_context, fail_on_error):
137152
"""
138153
Measure the performance of a chart and return the results.
139154
"""
@@ -146,17 +161,25 @@ def measure_chart(slice, query_context):
146161
start_time = datetime.now()
147162
try:
148163
result = command.run()
164+
165+
for query in result["queries"]:
166+
if "error" in query and query["error"]:
167+
raise query["error"]
149168
except Exception as e:
150169
logger.error(f"Error fetching slice data: {slice}. Error: {e}")
170+
if fail_on_error:
171+
raise e
151172
return
173+
152174
end_time = datetime.now()
153175

154176
result["time_elapsed"] = (end_time - start_time).total_seconds()
155177
result["slice"] = slice
178+
156179
return result
157180

158181

159-
def get_query_log_from_clickhouse(report, query_contexts, print_sql):
182+
def get_query_log_from_clickhouse(report, query_contexts, print_sql, fail_on_error):
160183
"""
161184
Get the query log from clickhouse and print the results.
162185
"""
@@ -170,7 +193,7 @@ def get_query_log_from_clickhouse(report, query_contexts, print_sql):
170193
{"col": "http_user_agent", "op": "==", "val": RUN_ID}
171194
)
172195

173-
ch_chart_result = measure_chart(slice, query_context)
196+
ch_chart_result = measure_chart(slice, query_context, fail_on_error)
174197

175198
clickhouse_queries = {}
176199
for query in ch_chart_result["queries"]:

tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_of_Current_Course_Grade.yaml

Lines changed: 50 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -6,24 +6,62 @@ dataset_uuid: 633a1d4e-cd40-482f-a5dc-d5901c2181c2
66
description: null
77
params:
88
adhoc_filters: []
9-
all_columns_x:
10-
- Course Grade
119
annotation_layers: []
1210
color_scheme: supersetColors
13-
cumulative: false
11+
comparison_type: values
1412
extra_form_data: {}
13+
forecastInterval: 0.8
14+
forecastPeriods: 10
1515
groupby: []
16-
link_length: '10'
17-
normalized: true
16+
legendOrientation: top
17+
legendType: scroll
18+
metrics:
19+
- aggregate: null
20+
column: null
21+
datasourceWarning: false
22+
expressionType: SQL
23+
hasCustomLabel: true
24+
label: Number of Learners
25+
optionName: metric_kwmmqye349q_e16flj0itp
26+
sqlExpression: count()
27+
only_total: true
28+
order_desc: true
29+
orientation: vertical
30+
rich_tooltip: true
1831
row_limit: 10000
32+
show_empty_columns: true
1933
show_legend: true
20-
viz_type: histogram
21-
x_axis_label: Grade
22-
y_axis_label: Number of Learners
23-
query_context: '{"datasource":{"id":70,"type":"table"},"force":false,"queries":[{"filters":[],"extras":{"having":"","where":""},"applied_time_extras":{},"columns":[],"metrics":[],"annotation_layers":[],"row_limit":10000,"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{}}],"form_data":{"datasource":"70__table","viz_type":"histogram","slice_id":197,"all_columns_x":["Course
24-
Grade"],"adhoc_filters":[],"row_limit":10000,"groupby":[],"color_scheme":"supersetColors","link_length":"10","x_axis_label":"Grade","y_axis_label":"Number
25-
of Learners","show_legend":true,"normalized":true,"cumulative":false,"extra_form_data":{},"dashboards":[16],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}'
34+
sort_series_type: sum
35+
tooltipTimeFormat: smart_date
36+
truncateXAxis: true
37+
truncate_metric: true
38+
viz_type: echarts_timeseries_bar
39+
x_axis:
40+
expressionType: SQL
41+
label: Course Grade Percent
42+
sqlExpression: grade_bucket
43+
x_axis_sort_asc: true
44+
x_axis_sort_series: name
45+
x_axis_sort_series_ascending: true
46+
x_axis_time_format: smart_date
47+
x_axis_title: Course Grade Percent
48+
x_axis_title_margin: 30
49+
y_axis_bounds:
50+
- null
51+
- null
52+
y_axis_format: SMART_NUMBER
53+
y_axis_title: Number of Learners
54+
y_axis_title_margin: 50
55+
y_axis_title_position: Left
56+
query_context: '{"datasource":{"id":64,"type":"table"},"force":false,"queries":[{"filters":[],"extras":{"having":"","where":""},"applied_time_extras":{},"columns":[{"columnType":"BASE_AXIS","label":"Course
57+
Grade Percent","sqlExpression":"grade_bucket","expressionType":"SQL"}],"metrics":[{"expressionType":"SQL","sqlExpression":"count()","column":null,"aggregate":null,"datasourceWarning":false,"hasCustomLabel":true,"label":"Number
58+
of Learners","optionName":"metric_kwmmqye349q_e16flj0itp"}],"orderby":[[{"expressionType":"SQL","sqlExpression":"count()","column":null,"aggregate":null,"datasourceWarning":false,"hasCustomLabel":true,"label":"Number
59+
of Learners","optionName":"metric_kwmmqye349q_e16flj0itp"},false]],"annotation_layers":[],"row_limit":10000,"series_columns":[],"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{},"time_offsets":[],"post_processing":[{"operation":"pivot","options":{"index":["Course
60+
Grade Percent"],"columns":[],"aggregates":{"Number of Learners":{"operator":"mean"}},"drop_missing_columns":false}},{"operation":"flatten"}]}],"form_data":{"datasource":"64__table","viz_type":"echarts_timeseries_bar","slice_id":2221,"x_axis":{"label":"Course
61+
Grade Percent","sqlExpression":"grade_bucket","expressionType":"SQL"},"x_axis_sort_asc":true,"x_axis_sort_series":"name","x_axis_sort_series_ascending":true,"metrics":[{"expressionType":"SQL","sqlExpression":"count()","column":null,"aggregate":null,"datasourceWarning":false,"hasCustomLabel":true,"label":"Number
62+
of Learners","optionName":"metric_kwmmqye349q_e16flj0itp"}],"groupby":[],"adhoc_filters":[],"order_desc":true,"row_limit":10000,"truncate_metric":true,"show_empty_columns":true,"comparison_type":"values","annotation_layers":[],"forecastPeriods":10,"forecastInterval":0.8,"orientation":"vertical","x_axis_title":"Course
63+
Grade Percent","x_axis_title_margin":30,"y_axis_title":"Number of Learners","y_axis_title_margin":50,"y_axis_title_position":"Left","sort_series_type":"sum","color_scheme":"supersetColors","only_total":true,"show_legend":true,"legendType":"scroll","legendOrientation":"top","x_axis_time_format":"smart_date","y_axis_format":"SMART_NUMBER","truncateXAxis":true,"y_axis_bounds":[null,null],"rich_tooltip":true,"tooltipTimeFormat":"smart_date","extra_form_data":{},"dashboards":[2522],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}'
2664
slice_name: Distribution of Current Course Grade
2765
uuid: ea565658-6796-40e8-9d1e-01ffd0778bc9
2866
version: 1.0.0
29-
viz_type: histogram
67+
viz_type: echarts_timeseries_bar

tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Learners_with_Passing_Grade.yaml

Lines changed: 31 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,18 @@ certified_by: null
55
dataset_uuid: 633a1d4e-cd40-482f-a5dc-d5901c2181c2
66
description: null
77
params:
8-
adhoc_filters: []
9-
annotation_layers: []
8+
adhoc_filters:
9+
- clause: WHERE
10+
comparator: passed
11+
datasourceWarning: false
12+
expressionType: SIMPLE
13+
filterOptionName: filter_uld7zeqwh9a_qn2vi29nkud
14+
isExtra: false
15+
isNew: false
16+
operator: ==
17+
operatorId: EQUALS
18+
sqlExpression: null
19+
subject: approving_state
1020
conditional_formatting:
1121
- colorScheme: '#ACE1C4'
1222
column: Number of Learners
@@ -43,31 +53,36 @@ params:
4353
time_format: smart_date
4454
viz_type: big_number_total
4555
y_axis_format: SMART_NUMBER
46-
query_context: "{\"datasource\":{\"id\":70,\"type\":\"table\"},\"force\":false,\"\
47-
queries\":[{\"filters\":[],\"extras\":{\"having\":\"\",\"where\":\"\"},\"applied_time_extras\"\
48-
:{},\"columns\":[],\"metrics\":[{\"aggregate\":\"COUNT_DISTINCT\",\"column\":{\"\
49-
advanced_data_type\":null,\"certification_details\":null,\"certified_by\":null,\"\
50-
column_name\":\"actor_id\",\"description\":null,\"expression\":null,\"filterable\"\
51-
:true,\"groupby\":true,\"id\":457,\"is_certified\":false,\"is_dttm\":false,\"python_date_format\"\
52-
:null,\"type\":\"STRING\",\"type_generic\":1,\"verbose_name\":null,\"warning_markdown\"\
56+
query_context: "{\"datasource\":{\"id\":64,\"type\":\"table\"},\"force\":false,\"\
57+
queries\":[{\"filters\":[{\"col\":\"approving_state\",\"op\":\"==\",\"val\":\"passed\"\
58+
}],\"extras\":{\"having\":\"\",\"where\":\"\"},\"applied_time_extras\":{},\"columns\"\
59+
:[],\"metrics\":[{\"aggregate\":\"COUNT_DISTINCT\",\"column\":{\"advanced_data_type\"\
60+
:null,\"certification_details\":null,\"certified_by\":null,\"column_name\":\"actor_id\"\
61+
,\"description\":null,\"expression\":null,\"filterable\":true,\"groupby\":true,\"\
62+
id\":457,\"is_certified\":false,\"is_dttm\":false,\"python_date_format\":null,\"\
63+
type\":\"STRING\",\"type_generic\":1,\"verbose_name\":null,\"warning_markdown\"\
5364
:null},\"datasourceWarning\":false,\"expressionType\":\"SIMPLE\",\"hasCustomLabel\"\
5465
:true,\"label\":\"Number of Learners\",\"optionName\":\"metric_edlf1m0plr_j1zup9r1ixl\"\
5566
,\"sqlExpression\":null}],\"annotation_layers\":[],\"series_limit\":0,\"order_desc\"\
5667
:true,\"url_params\":{},\"custom_params\":{},\"custom_form_data\":{}}],\"form_data\"\
57-
:{\"datasource\":\"70__table\",\"viz_type\":\"big_number_total\",\"slice_id\":196,\"\
68+
:{\"datasource\":\"64__table\",\"viz_type\":\"big_number_total\",\"slice_id\":2581,\"\
5869
metric\":{\"aggregate\":\"COUNT_DISTINCT\",\"column\":{\"advanced_data_type\":null,\"\
5970
certification_details\":null,\"certified_by\":null,\"column_name\":\"actor_id\"\
6071
,\"description\":null,\"expression\":null,\"filterable\":true,\"groupby\":true,\"\
6172
id\":457,\"is_certified\":false,\"is_dttm\":false,\"python_date_format\":null,\"\
6273
type\":\"STRING\",\"type_generic\":1,\"verbose_name\":null,\"warning_markdown\"\
6374
:null},\"datasourceWarning\":false,\"expressionType\":\"SIMPLE\",\"hasCustomLabel\"\
6475
:true,\"label\":\"Number of Learners\",\"optionName\":\"metric_edlf1m0plr_j1zup9r1ixl\"\
65-
,\"sqlExpression\":null},\"adhoc_filters\":[],\"header_font_size\":0.6,\"subheader_font_size\"\
66-
:0.4,\"y_axis_format\":\"SMART_NUMBER\",\"time_format\":\"smart_date\",\"conditional_formatting\"\
67-
:[{\"colorScheme\":\"#ACE1C4\",\"column\":\"Number of Learners\",\"operator\":\"\
68-
\u2265\",\"targetValue\":0}],\"extra_form_data\":{},\"dashboards\":[16],\"force\"\
69-
:false,\"result_format\":\"json\",\"result_type\":\"full\"},\"result_format\":\"\
70-
json\",\"result_type\":\"full\"}"
76+
,\"sqlExpression\":null},\"adhoc_filters\":[{\"expressionType\":\"SIMPLE\",\"subject\"\
77+
:\"approving_state\",\"operator\":\"==\",\"operatorId\":\"EQUALS\",\"comparator\"\
78+
:\"passed\",\"clause\":\"WHERE\",\"sqlExpression\":null,\"isExtra\":false,\"isNew\"\
79+
:false,\"datasourceWarning\":false,\"filterOptionName\":\"filter_uld7zeqwh9a_qn2vi29nkud\"\
80+
}],\"header_font_size\":0.6,\"subheader_font_size\":0.4,\"y_axis_format\":\"SMART_NUMBER\"\
81+
,\"time_format\":\"smart_date\",\"conditional_formatting\":[{\"colorScheme\":\"\
82+
#ACE1C4\",\"column\":\"Number of Learners\",\"operator\":\"\u2265\",\"targetValue\"\
83+
:0}],\"extra_form_data\":{},\"dashboards\":[2522],\"force\":false,\"result_format\"\
84+
:\"json\",\"result_type\":\"full\"},\"result_format\":\"json\",\"result_type\":\"\
85+
full\"}"
7186
slice_name: Learners with Passing Grade
7287
uuid: b40cdabc-b265-48d2-913d-a9dfee0b6ab1
7388
version: 1.0.0

0 commit comments

Comments
 (0)