Fix flaky CAgg isolation tests (timescale#10076)

cagg_concurrent_register and cagg_refresh_cleanup_register tests were
flaky due to different completion order of steps on some environments.
Fix them by adding annotations.

Author: melihmutlu

tsl/test/isolation/expected/cagg_concurrent_register.out

@@ -35,7 +35,7 @@ step s5_show_running_jobs:
          to_timestamp(r.end_range / 1000000) AT TIME ZONE 'UTC' AS end_ts_utc
   FROM _timescaledb_catalog.continuous_aggs_jobs_refresh_ranges r
   JOIN _timescaledb_catalog.continuous_agg ca ON r.materialization_id = ca.mat_hypertable_id
-  ORDER BY ca.user_view_name;
+  ORDER BY ca.user_view_name, start_range;
 
 cagg_name|     start_range|       end_range|start_ts_utc            |end_ts_utc              
 ---------+----------------+----------------+------------------------+------------------------
@@ -44,13 +44,14 @@ cagg_2   |1578153600000000|1578182400000000|Sat Jan 04 16:00:00 2020|Sun Jan 05
 
 step s4_release_before_process_cagg_invalidations: 
     SELECT debug_waitpoint_release('before_process_cagg_invalidations_for_refresh_lock');
-
+ <waiting ...>
+step s1_run_cagg1_refresh: <... completed>
+step s2_run_cagg2_overlap_refresh: <... completed>
+step s4_release_before_process_cagg_invalidations: <... completed>
 debug_waitpoint_release
 -----------------------
 
 
-step s1_run_cagg1_refresh: <... completed>
-step s2_run_cagg2_overlap_refresh: <... completed>
 
 starting permutation: s2_insert_new_data_2020 s3_lock_before_register s1_run_cagg2_overlap_refresh s2_run_cagg2_overlap_refresh s4_enable_before_process_cagg_invalidations s3_release_after_register s5_show_running_jobs s4_release_before_process_cagg_invalidations
 step s2_insert_new_data_2020: 
@@ -87,23 +88,24 @@ step s5_show_running_jobs:
          to_timestamp(r.end_range / 1000000) AT TIME ZONE 'UTC' AS end_ts_utc
   FROM _timescaledb_catalog.continuous_aggs_jobs_refresh_ranges r
   JOIN _timescaledb_catalog.continuous_agg ca ON r.materialization_id = ca.mat_hypertable_id
-  ORDER BY ca.user_view_name;
+  ORDER BY ca.user_view_name, start_range;
 
 cagg_name|     start_range|       end_range|start_ts_utc            |end_ts_utc              
 ---------+----------------+----------------+------------------------+------------------------
-cagg_2   |1578268800000000|1578355200000000|Mon Jan 06 00:00:00 2020|Tue Jan 07 00:00:00 2020
 cagg_2   |1578153600000000|1578182400000000|Sat Jan 04 16:00:00 2020|Sun Jan 05 00:00:00 2020
+cagg_2   |1578268800000000|1578355200000000|Mon Jan 06 00:00:00 2020|Tue Jan 07 00:00:00 2020
 
 step s4_release_before_process_cagg_invalidations: 
     SELECT debug_waitpoint_release('before_process_cagg_invalidations_for_refresh_lock');
-
+ <waiting ...>
+step s1_run_cagg2_overlap_refresh: <... completed>
+ERROR:  could not refresh continuous aggregate "cagg_2" due to a concurrent refresh
+step s2_run_cagg2_overlap_refresh: <... completed>
+step s4_release_before_process_cagg_invalidations: <... completed>
 debug_waitpoint_release
 -----------------------
 
 
-step s1_run_cagg2_overlap_refresh: <... completed>
-ERROR:  could not refresh continuous aggregate "cagg_2" due to a concurrent refresh
-step s2_run_cagg2_overlap_refresh: <... completed>
 
 starting permutation: s2_insert_new_data_2020 s3_lock_before_register s1_run_cagg2_nonoverlap_refresh s2_run_cagg2_overlap_refresh s4_enable_before_process_cagg_invalidations s3_release_after_register s5_show_running_jobs s4_release_before_process_cagg_invalidations
 step s2_insert_new_data_2020: 
@@ -140,7 +142,7 @@ step s5_show_running_jobs:
          to_timestamp(r.end_range / 1000000) AT TIME ZONE 'UTC' AS end_ts_utc
   FROM _timescaledb_catalog.continuous_aggs_jobs_refresh_ranges r
   JOIN _timescaledb_catalog.continuous_agg ca ON r.materialization_id = ca.mat_hypertable_id
-  ORDER BY ca.user_view_name;
+  ORDER BY ca.user_view_name, start_range;
 
 cagg_name|     start_range|       end_range|start_ts_utc            |end_ts_utc              
 ---------+----------------+----------------+------------------------+------------------------
@@ -149,10 +151,11 @@ cagg_2   |1578153600000000|1578182400000000|Sat Jan 04 16:00:00 2020|Sun Jan 05
 
 step s4_release_before_process_cagg_invalidations: 
     SELECT debug_waitpoint_release('before_process_cagg_invalidations_for_refresh_lock');
-
+ <waiting ...>
+step s1_run_cagg2_nonoverlap_refresh: <... completed>
+step s2_run_cagg2_overlap_refresh: <... completed>
+step s4_release_before_process_cagg_invalidations: <... completed>
 debug_waitpoint_release
 -----------------------
 
 
-step s1_run_cagg2_nonoverlap_refresh: <... completed>
-step s2_run_cagg2_overlap_refresh: <... completed>

tsl/test/isolation/expected/cagg_refresh_cleanup_register.out

@@ -1,6 +1,6 @@
 Parsed test spec with 9 sessions
 
-starting permutation: WP_mat_enable R2_refresh L1_lock WP_mat_disable WP_enable_after_refresh R3_refresh R4_refresh check_locks check_jobs L1_unlock WP_disable_after_refresh check_locks check_jobs
+starting permutation: WP_mat_enable R2_refresh L1_lock WP_mat_disable R3_refresh R4_refresh check_locks check_jobs L1_unlock check_locks check_jobs
 step WP_mat_enable: SELECT debug_waitpoint_enable('after_process_cagg_materializations');
 debug_waitpoint_enable
 ----------------------
@@ -19,11 +19,6 @@ debug_waitpoint_release
 -----------------------
 
 
-step WP_enable_after_refresh: SELECT debug_waitpoint_enable('after_cagg_refresh_window');
-debug_waitpoint_enable
-----------------------
-                      
-
 step R3_refresh: 
     CALL refresh_continuous_aggregate('cond_daily', '2026-02-15', '2026-03-15');
  <waiting ...>
@@ -59,15 +54,11 @@ cond_daily    |Sun Feb 08 16:00:00 2026 PST|Sat Feb 14 16:00:00 2026 PST
 
 step L1_unlock: 
     COMMIT;
-
-step WP_disable_after_refresh: SELECT debug_waitpoint_release('after_cagg_refresh_window');
-debug_waitpoint_release
------------------------
-                       
-
+ <waiting ...>
 step R2_refresh: <... completed>
 step R3_refresh: <... completed>
 step R4_refresh: <... completed>
+step L1_unlock: <... completed>
 step check_locks: 
     SELECT l.mode, l.granted
     FROM pg_locks l
@@ -91,7 +82,7 @@ user_view_name|start_time|end_time
 --------------+----------+--------
 
 
-starting permutation: WP_mat_enable R2_refresh L1_lock WP_mat_disable WP_after_register_enable R3_refresh R4_overlapping_refresh check_locks check_jobs L1_unlock WP_after_register_disable check_locks check_jobs
+starting permutation: WP_mat_enable R2_refresh L1_lock WP_mat_disable R3_refresh R4_overlapping_refresh check_locks check_jobs L1_unlock check_locks check_jobs
 step WP_mat_enable: SELECT debug_waitpoint_enable('after_process_cagg_materializations');
 debug_waitpoint_enable
 ----------------------
@@ -110,11 +101,6 @@ debug_waitpoint_release
 -----------------------
 
 
-step WP_after_register_enable: SELECT debug_waitpoint_enable('cagg_refresh_after_register');
-debug_waitpoint_enable
-----------------------
-                      
-
 step R3_refresh: 
     CALL refresh_continuous_aggregate('cond_daily', '2026-02-15', '2026-03-15');
  <waiting ...>
@@ -150,16 +136,12 @@ cond_daily    |Sun Feb 08 16:00:00 2026 PST|Sat Feb 14 16:00:00 2026 PST
 
 step L1_unlock: 
     COMMIT;
-
-step WP_after_register_disable: SELECT debug_waitpoint_release('cagg_refresh_after_register');
-debug_waitpoint_release
------------------------
-                       
-
+ <waiting ...>
 step R2_refresh: <... completed>
 step R3_refresh: <... completed>
 step R4_overlapping_refresh: <... completed>
 ERROR:  could not refresh continuous aggregate "cond_daily" due to a concurrent refresh
+step L1_unlock: <... completed>
 step check_locks: 
     SELECT l.mode, l.granted
     FROM pg_locks l

tsl/test/isolation/specs/cagg_concurrent_register.spec

@@ -114,18 +114,18 @@ step "s5_show_running_jobs" {
          to_timestamp(r.end_range / 1000000) AT TIME ZONE 'UTC' AS end_ts_utc
   FROM _timescaledb_catalog.continuous_aggs_jobs_refresh_ranges r
   JOIN _timescaledb_catalog.continuous_agg ca ON r.materialization_id = ca.mat_hypertable_id
-  ORDER BY ca.user_view_name;
+  ORDER BY ca.user_view_name, start_range;
 }
 
 # TEST: when 2 concurrent refresh processes on cagg1 and cagg2 start, 1 will wait for the other one to finish registration
 ## block the processes before they register, then block again before they finish cagg invalidation processing
 ## so that we can see the active ranges before the refresh processes exit.
-permutation "s2_insert_new_data_2020" "s3_lock_before_register" "s1_run_cagg1_refresh" "s2_run_cagg2_overlap_refresh" "s4_enable_before_process_cagg_invalidations" "s3_release_after_register" "s5_show_running_jobs" "s4_release_before_process_cagg_invalidations"
+permutation "s2_insert_new_data_2020" "s3_lock_before_register" "s1_run_cagg1_refresh" "s2_run_cagg2_overlap_refresh"("s1_run_cagg1_refresh") "s4_enable_before_process_cagg_invalidations" "s3_release_after_register" "s5_show_running_jobs" "s4_release_before_process_cagg_invalidations"("s2_run_cagg2_overlap_refresh")
 
 # TEST: Check that two overlapping refresh on cagg2 will not run concurrently. overlap refresh job will fail.
 ## so we will see only 1 running job.
-permutation "s2_insert_new_data_2020" "s3_lock_before_register" "s1_run_cagg2_overlap_refresh" "s2_run_cagg2_overlap_refresh" "s4_enable_before_process_cagg_invalidations" "s3_release_after_register" "s5_show_running_jobs" "s4_release_before_process_cagg_invalidations"
+permutation "s2_insert_new_data_2020" "s3_lock_before_register" "s1_run_cagg2_overlap_refresh" "s2_run_cagg2_overlap_refresh"("s1_run_cagg2_overlap_refresh") "s4_enable_before_process_cagg_invalidations" "s3_release_after_register" "s5_show_running_jobs" "s4_release_before_process_cagg_invalidations"("s2_run_cagg2_overlap_refresh")
 
 # TEST: Check that two non-overlapping refresh on cagg2 will run concurrently
 ## we should see both jobs
-permutation "s2_insert_new_data_2020" "s3_lock_before_register" "s1_run_cagg2_nonoverlap_refresh" "s2_run_cagg2_overlap_refresh"(s1_run_cagg2_nonoverlap_refresh) "s4_enable_before_process_cagg_invalidations" "s3_release_after_register" "s5_show_running_jobs" "s4_release_before_process_cagg_invalidations"
+permutation "s2_insert_new_data_2020" "s3_lock_before_register" "s1_run_cagg2_nonoverlap_refresh" "s2_run_cagg2_overlap_refresh"("s1_run_cagg2_nonoverlap_refresh") "s4_enable_before_process_cagg_invalidations" "s3_release_after_register" "s5_show_running_jobs" "s4_release_before_process_cagg_invalidations"("s2_run_cagg2_overlap_refresh")

tsl/test/isolation/specs/cagg_refresh_cleanup_register.spec

@@ -74,8 +74,6 @@ step "WP_before_txn2_start_enable"  { SELECT debug_waitpoint_enable('cagg_refres
 step "WP_before_txn2_start_disable"  { SELECT debug_waitpoint_release('cagg_refresh_after_register'); }
 step "WP_after_register_enable"  { SELECT debug_waitpoint_enable('cagg_refresh_after_register'); }
 step "WP_after_register_disable"  { SELECT debug_waitpoint_release('cagg_refresh_after_register'); }
-step "WP_enable_after_refresh"  { SELECT debug_waitpoint_enable('after_cagg_refresh_window'); }
-step "WP_disable_after_refresh"  { SELECT debug_waitpoint_release('after_cagg_refresh_window'); }
 
 # Session K1: terminate R1's backend so its PID becomes dead in the
 # registration table, then wait until the process is gone.
@@ -222,11 +220,11 @@ step "P1_run_policy" {
 
 # Two refreshes wait for registration, one waits for cleanup before exiting. All blocked on an AccessExclusiveLock on continuous_aggs_jobs_refresh_ranges.
 # None of those refreshes overlaps, so all should succeed.
-permutation "WP_mat_enable" "R2_refresh" "L1_lock" "WP_mat_disable" "WP_enable_after_refresh" "R3_refresh"("R2_refresh") "R4_refresh"("R3_refresh") "check_locks" "check_jobs" "L1_unlock" "WP_disable_after_refresh" "check_locks" "check_jobs"
+permutation "WP_mat_enable" "R2_refresh" "L1_lock" "WP_mat_disable" "R3_refresh"("R2_refresh") "R4_refresh"("R3_refresh") "check_locks" "check_jobs" "L1_unlock"("R4_refresh") "check_locks" "check_jobs"
 
 # Two refreshes wait for registration, one waits for cleanup before exiting. All blocked on an AccessExclusiveLock on continuous_aggs_jobs_refresh_ranges.
 # Refreshes waiting for registration overlap with each other, so one should fail.
-permutation "WP_mat_enable" "R2_refresh" "L1_lock" "WP_mat_disable" "WP_after_register_enable" "R3_refresh"("R2_refresh") "R4_overlapping_refresh"("R3_refresh") "check_locks" "check_jobs" "L1_unlock" "WP_after_register_disable" "check_locks" "check_jobs"
+permutation "WP_mat_enable" "R2_refresh" "L1_lock" "WP_mat_disable" "R3_refresh"("R2_refresh") "R4_overlapping_refresh"("R3_refresh") "check_locks" "check_jobs" "L1_unlock"("R4_overlapping_refresh") "check_locks" "check_jobs"
 
 ## Refresh registers . But fails in txn2. Gets into catch block. Cleanup should succeed
 permutation "WP_before_txn2_start_enable" "R3_refresh" "check_jobs" "A1_revoke_perm" "WP_before_txn2_start_disable"("A1_revoke_perm") "check_jobs"