tc

Author: Catherine Lee

tools/torchci/update_test_times.py

@@ -8,10 +8,11 @@
 TEST_TIMES_URL = "https://raw.githubusercontent.com/pytorch/test-infra/generated-stats/stats/test-times.json"
 TEST_CLASS_TIMES_URL = "https://raw.githubusercontent.com/pytorch/test-infra/generated-stats/stats/test-class-times.json"
 
-TEST_TIME_PER_FILE_QUERY_NAME = "test_time_per_file"
-TEST_TIME_PER_FILE_PERIODIC_JOBS_QUERY_NAME = "test_time_per_file_periodic_jobs"
-TEST_TIME_PER_CLASS_QUERY_NAME = "test_time_per_class"
-TEST_TIME_PER_CLASS_PERIODIC_JOBS_QUERY_NAME = "test_time_per_class_periodic_jobs"
+TEST_TIME_PER_FILE_QUERY_NAME = "test_times/per_file"
+TEST_TIME_PER_FILE_PERIODIC_JOBS_QUERY_NAME = "test_times/per_file_periodic_jobs"
+TEST_TIME_PER_CLASS_QUERY_NAME = "test_times/per_class"
+TEST_TIME_PER_CLASS_PERIODIC_JOBS_QUERY_NAME = "test_times/per_class_periodic_jobs"
+JOB_NAMES_PAST_MONTH_QUERY_NAME = "test_times/job_names_past_month"
 
 
 def get_file_data_from_db():
@@ -77,6 +78,18 @@ def convert_test_class_times_to_default_dict(d):
     return new_d
 
 
+def clean_up_test_times(test_times) -> None:
+    # Remove old environments that no longer run jobs
+    job_names_past_month = query_clickhouse_saved(
+        JOB_NAMES_PAST_MONTH_QUERY_NAME, {}
+    )
+    job_names = {row["base_name"] for row in job_names_past_month}
+    job_names.add("default")  # Add default to job names
+    for env in list(test_times.keys()):
+        if env not in job_names:
+            del test_times[env]
+
+
 def gen_test_file_times(db_results, old_test_times):
     # Use old test times because sometimes we want to manually edit the test
     # times json and want those changes to persist.  Unfortunately this means
@@ -108,6 +121,8 @@ def gen_test_file_times(db_results, old_test_times):
         # usually in the default test config like distributed
         test_times["default"]["default"] = test_times_no_test_config
 
+    clean_up_test_times(test_times)
+
     return test_times
 
 
@@ -153,6 +168,8 @@ def gen_test_class_times(db_results, old_test_times):
         # usually in the default test config like distributed
         test_times["default"]["default"] = test_times_no_test_config
 
+    clean_up_test_times(test_times)
+
     return test_times
 
 

…_queries/test_time_per_class/params.json …t_times/job_names_past_month/params.jsontorchci/clickhouse_queries/test_time_per_class/params.json renamed to torchci/clickhouse_queries/test_times/job_names_past_month/params.json torchci/clickhouse_queries/test_time_per_class/params.json renamed to torchci/clickhouse_queries/test_times/job_names_past_month/params.json

@@ -0,0 +1,5 @@
+SELECT DISTINCT
+    REGEXP_EXTRACT(j.name, '^(.*) /', 1) AS base_name
+FROM
+    default.workflow_job j
+WHERE j.created_at > now() - INTERVAL 1 MONTH

torchci/clickhouse_queries/test_times/job_names_past_month/query.sql

@@ -0,0 +1,4 @@
+{
+  "params": {},
+  "tests": [{}]
+}