Skip to content

Commit 7547859

Browse files
melihmutlumelihmutlu
committed
Fix flaky cagg_hierarchical_concurrent_refresh
Synchronize with a lock on cagg_6h's materialization hypertable instead. L1 cagg refresh is blocked on the lock mid-txn 3 after deleting its processed log entries but before the commit. While L2 moves and process its log entries concurrently, blocking only when it starts materializing from cagg_6h.
1 parent 56f1656 commit 7547859

2 files changed

Lines changed: 59 additions & 57 deletions

File tree

‎tsl/test/isolation/expected/cagg_hierarchical_concurrent_refresh.out‎

Lines changed: 37 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
Parsed test spec with 8 sessions
1+
Parsed test spec with 7 sessions
22

33
starting permutation: chk_hyper_invals WP_before_enable L1_refresh_jan1 WP_before_release chk_hyper_invals WP_before_enable L2_refresh_jan1 WP_before_release chk_cagg_6h chk_cagg_1d chk_6h_consistency chk_1d_consistency
44
step chk_hyper_invals:
@@ -463,7 +463,7 @@ Fri Jan 02 00:00:00 2026 UTC|32.1666666666667|32.1666666666667| 288|
463463
Sat Jan 03 00:00:00 2026 UTC| 31.5| 31.5| 96| 96|t |t
464464

465465

466-
starting permutation: L1_refresh_full chk_1d_consistency WP_before_enable chk_hyper_invals lock_L2_source L2_refresh_jan1 insert_ht L1_refresh_full L2b_refresh_jan1_2 WP_before_release unlock chk_cagg_1d chk_1d_consistency
466+
starting permutation: L1_refresh_full chk_1d_consistency WP_before_enable chk_hyper_invals lock_L2_source L2_refresh_jan1 insert_ht L1_refresh_full L2b_refresh_jan1_2 WP_before_release unlock_L2_source chk_cagg_1d chk_1d_consistency
467467
step L1_refresh_full:
468468
CALL refresh_continuous_aggregate('cagg_6h', '2026-01-01', '2026-01-04');
469469

@@ -546,7 +546,7 @@ debug_waitpoint_release
546546

547547

548548
step L1_refresh_full: <... completed>
549-
step unlock:
549+
step unlock_L2_source:
550550
ROLLBACK;
551551

552552
step L2_refresh_jan1: <... completed>
@@ -586,7 +586,7 @@ Fri Jan 02 00:00:00 2026 UTC| 31.5|32.1666666666667| 96|
586586
Sat Jan 03 00:00:00 2026 UTC| 31.5| 40.5| 96| 288|f |f
587587

588588

589-
starting permutation: L1_refresh_full chk_1d_consistency WP_before_enable chk_hyper_invals lock_L2_source L2b_refresh_jan1_2 insert_ht L1_refresh_full L2_refresh_jan1 WP_before_release unlock chk_cagg_1d chk_1d_consistency
589+
starting permutation: L1_refresh_full chk_1d_consistency WP_before_enable chk_hyper_invals lock_L2_source L2b_refresh_jan1_2 insert_ht L1_refresh_full L2_refresh_jan1 WP_before_release unlock_L2_source chk_cagg_1d chk_1d_consistency
590590
step L1_refresh_full:
591591
CALL refresh_continuous_aggregate('cagg_6h', '2026-01-01', '2026-01-04');
592592

@@ -669,7 +669,7 @@ debug_waitpoint_release
669669

670670

671671
step L1_refresh_full: <... completed>
672-
step unlock:
672+
step unlock_L2_source:
673673
ROLLBACK;
674674

675675
step L2b_refresh_jan1_2: <... completed>
@@ -864,7 +864,7 @@ Fri Jan 02 00:00:00 2026 UTC| 31.5| 31.5| 96|
864864
Sat Jan 03 00:00:00 2026 UTC| 40.5| 40.5| 288| 288|t |t
865865

866866

867-
starting permutation: L1_refresh_full chk_1d_consistency WP_before_enable chk_hyper_invals lock_L2_source L2_refresh_jan1 insert_ht L1_refresh_full L2b_refresh_jan3 WP_before_release unlock chk_cagg_1d chk_1d_consistency
867+
starting permutation: L1_refresh_full chk_1d_consistency WP_before_enable chk_hyper_invals lock_L2_source L2_refresh_jan1 insert_ht L1_refresh_full L2b_refresh_jan3 WP_before_release unlock_L2_source chk_cagg_1d chk_1d_consistency
868868
step L1_refresh_full:
869869
CALL refresh_continuous_aggregate('cagg_6h', '2026-01-01', '2026-01-04');
870870

@@ -946,7 +946,7 @@ debug_waitpoint_release
946946

947947

948948
step L1_refresh_full: <... completed>
949-
step unlock:
949+
step unlock_L2_source:
950950
ROLLBACK;
951951

952952
step L2_refresh_jan1: <... completed>
@@ -987,19 +987,15 @@ Fri Jan 02 00:00:00 2026 UTC| 31.5|32.1666666666667| 96|
987987
Sat Jan 03 00:00:00 2026 UTC| 40.5| 40.5| 288| 288|t |t
988988

989989

990-
starting permutation: insert_ht WP_after_enable chk_6h_consistency L1_refresh_jan1 lock_mat_invals WP_after_release L2_refresh_full unlock chk_mat_invals chk_6h_consistency chk_1d_consistency
990+
starting permutation: insert_ht L1b_refresh_jan3 chk_6h_consistency lock_6h_mat_table L1_refresh_jan1 L2_refresh_full unlock_6h_mat_table chk_mat_invals chk_6h_consistency chk_1d_consistency
991991
step insert_ht:
992992
INSERT INTO conditions (time, temp)
993993
SELECT ts, 50.0
994994
FROM generate_series('2026-01-01 00:00:00+00'::timestamptz,
995995
'2026-01-04 23:45:00+00', '15 minutes') ts;
996996

997-
step WP_after_enable:
998-
SELECT debug_waitpoint_enable('after_process_cagg_invalidations_for_refresh_lock');
999-
1000-
debug_waitpoint_enable
1001-
----------------------
1002-
997+
step L1b_refresh_jan3:
998+
CALL refresh_continuous_aggregate('cagg_6h', '2026-01-03 00:00', '2026-01-04 00:00');
1003999

10041000
step chk_6h_consistency:
10051001
-- Verify L1 (6-hour) matches re-aggregation from raw hypertable
@@ -1029,34 +1025,35 @@ Fri Jan 02 00:00:00 2026 UTC| 22.5| 27.5| 24| 72|f
10291025
Fri Jan 02 06:00:00 2026 UTC| 28.5| 29.5| 24| 72|f |f
10301026
Fri Jan 02 12:00:00 2026 UTC| 34.5|34.8333333333333| 24| 72|f |f
10311027
Fri Jan 02 18:00:00 2026 UTC| 40.5|36.8333333333333| 24| 72|f |f
1032-
Sat Jan 03 00:00:00 2026 UTC| 22.5| 37.5| 24| 72|f |f
1033-
Sat Jan 03 06:00:00 2026 UTC| 28.5| 39.5| 24| 72|f |f
1034-
Sat Jan 03 12:00:00 2026 UTC| 34.5| 41.5| 24| 72|f |f
1035-
Sat Jan 03 18:00:00 2026 UTC| 40.5| 43.5| 24| 72|f |f
1028+
Sat Jan 03 00:00:00 2026 UTC| 37.5| 37.5| 72| 72|t |t
1029+
Sat Jan 03 06:00:00 2026 UTC| 39.5| 39.5| 72| 72|t |t
1030+
Sat Jan 03 12:00:00 2026 UTC| 41.5| 41.5| 72| 72|t |t
1031+
Sat Jan 03 18:00:00 2026 UTC| 43.5| 43.5| 72| 72|t |t
1032+
1033+
step lock_6h_mat_table:
1034+
BEGIN;
1035+
DO $$
1036+
DECLARE
1037+
mat_table text;
1038+
BEGIN
1039+
SELECT format('%I.%I', h.schema_name, h.table_name) INTO mat_table
1040+
FROM _timescaledb_catalog.continuous_agg ca
1041+
JOIN _timescaledb_catalog.hypertable h ON h.id = ca.mat_hypertable_id
1042+
WHERE ca.user_view_name = 'cagg_6h';
1043+
EXECUTE format('LOCK TABLE %s IN ACCESS EXCLUSIVE MODE', mat_table);
1044+
END;
1045+
$$;
10361046

10371047
step L1_refresh_jan1:
10381048
CALL refresh_continuous_aggregate('cagg_6h', '2026-01-01 00:00', '2026-01-02 00:00');
10391049
<waiting ...>
1040-
step lock_mat_invals:
1041-
BEGIN;
1042-
LOCK _timescaledb_catalog.continuous_aggs_materialization_invalidation_log
1043-
IN ACCESS EXCLUSIVE MODE;
1044-
1045-
step WP_after_release:
1046-
SELECT debug_waitpoint_release('after_process_cagg_invalidations_for_refresh_lock');
1047-
1048-
debug_waitpoint_release
1049-
-----------------------
1050-
1051-
10521050
step L2_refresh_full:
10531051
CALL refresh_continuous_aggregate('cagg_1d', '2026-01-01', '2026-01-05');
10541052
<waiting ...>
1055-
step unlock:
1053+
step unlock_6h_mat_table:
10561054
ROLLBACK;
1057-
1055+
<waiting ...>
10581056
step L1_refresh_jan1: <... completed>
1059-
L2: NOTICE: continuous aggregate "cagg_1d" is already up-to-date
10601057
step L2_refresh_full: <... completed>
10611058
step chk_mat_invals:
10621059
SELECT ca.user_view_name AS cagg,
@@ -1071,9 +1068,10 @@ cagg |lowest |greatest
10711068
cagg_1d|-infinity |Wed Dec 31 23:59:59.999999 2025 UTC
10721069
cagg_1d|Mon Jan 05 00:00:00 2026 UTC|infinity
10731070
cagg_6h|-infinity |Wed Dec 31 23:59:59.999999 2025 UTC
1074-
cagg_6h|Fri Jan 02 00:00:00 2026 UTC|Sat Jan 03 23:59:59.999999 2026 UTC
1071+
cagg_6h|Fri Jan 02 00:00:00 2026 UTC|Fri Jan 02 23:59:59.999999 2026 UTC
10751072
cagg_6h|Sun Jan 04 00:00:00 2026 UTC|infinity
10761073

1074+
step unlock_6h_mat_table: <... completed>
10771075
step chk_6h_consistency:
10781076
-- Verify L1 (6-hour) matches re-aggregation from raw hypertable
10791077
SELECT c.bucket,
@@ -1102,10 +1100,10 @@ Fri Jan 02 00:00:00 2026 UTC| 22.5| 27.5| 24|
11021100
Fri Jan 02 06:00:00 2026 UTC| 28.5| 29.5| 24| 72|f |f
11031101
Fri Jan 02 12:00:00 2026 UTC| 34.5|34.8333333333333| 24| 72|f |f
11041102
Fri Jan 02 18:00:00 2026 UTC| 40.5|36.8333333333333| 24| 72|f |f
1105-
Sat Jan 03 00:00:00 2026 UTC| 22.5| 37.5| 24| 72|f |f
1106-
Sat Jan 03 06:00:00 2026 UTC| 28.5| 39.5| 24| 72|f |f
1107-
Sat Jan 03 12:00:00 2026 UTC| 34.5| 41.5| 24| 72|f |f
1108-
Sat Jan 03 18:00:00 2026 UTC| 40.5| 43.5| 24| 72|f |f
1103+
Sat Jan 03 00:00:00 2026 UTC| 37.5| 37.5| 72| 72|t |t
1104+
Sat Jan 03 06:00:00 2026 UTC| 39.5| 39.5| 72| 72|t |t
1105+
Sat Jan 03 12:00:00 2026 UTC| 41.5| 41.5| 72| 72|t |t
1106+
Sat Jan 03 18:00:00 2026 UTC| 43.5| 43.5| 72| 72|t |t
11091107

11101108
step chk_1d_consistency:
11111109
-- Verify L2 (daily) matches re-aggregation from L1 (6-hour)
@@ -1129,5 +1127,5 @@ bucket |daily_avg| from_6h_avg|daily_count|from_6h_coun
11291127
----------------------------+---------+----------------+-----------+-------------+---------+-----------
11301128
Thu Jan 01 00:00:00 2026 UTC| 31.5|37.1666666666667| 96| 288|f |f
11311129
Fri Jan 02 00:00:00 2026 UTC| 31.5| 31.5| 96| 96|t |t
1132-
Sat Jan 03 00:00:00 2026 UTC| 31.5| 31.5| 96| 96|t |t
1130+
Sat Jan 03 00:00:00 2026 UTC| 40.5| 40.5| 288| 288|t |t
11331131

‎tsl/test/isolation/specs/cagg_hierarchical_concurrent_refresh.spec‎

Lines changed: 22 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -116,16 +116,6 @@ step "WP_before_release"
116116
SELECT debug_waitpoint_release('before_process_cagg_invalidations_for_refresh_lock');
117117
}
118118

119-
session "WP_after"
120-
step "WP_after_enable"
121-
{
122-
SELECT debug_waitpoint_enable('after_process_cagg_invalidations_for_refresh_lock');
123-
}
124-
step "WP_after_release"
125-
{
126-
SELECT debug_waitpoint_release('after_process_cagg_invalidations_for_refresh_lock');
127-
}
128-
129119
# Session to refresh L1 (6-hour)
130120
session "L1"
131121
setup
@@ -210,13 +200,27 @@ step "lock_L2_source"
210200
END;
211201
$$;
212202
}
213-
step "lock_mat_invals"
203+
step "unlock_L2_source"
204+
{
205+
ROLLBACK;
206+
}
207+
208+
step "lock_6h_mat_table"
214209
{
215210
BEGIN;
216-
LOCK _timescaledb_catalog.continuous_aggs_materialization_invalidation_log
217-
IN ACCESS EXCLUSIVE MODE;
211+
DO $$
212+
DECLARE
213+
mat_table text;
214+
BEGIN
215+
SELECT format('%I.%I', h.schema_name, h.table_name) INTO mat_table
216+
FROM _timescaledb_catalog.continuous_agg ca
217+
JOIN _timescaledb_catalog.hypertable h ON h.id = ca.mat_hypertable_id
218+
WHERE ca.user_view_name = 'cagg_6h';
219+
EXECUTE format('LOCK TABLE %s IN ACCESS EXCLUSIVE MODE', mat_table);
220+
END;
221+
$$;
218222
}
219-
step "unlock"
223+
step "unlock_6h_mat_table"
220224
{
221225
ROLLBACK;
222226
}
@@ -317,10 +321,10 @@ permutation "WP_before_enable" "chk_hyper_invals" "L1b_refresh_jan1_2" "insert_h
317321

318322
# Two concurrent refreshes on L2 CAgg with overlapping ranges. One should fail due to overlap.
319323
# L1 is refreshed between the two L2 refreshes so that the second L2 refresh encounters an overlapping materialization range and fails.
320-
permutation "L1_refresh_full" "chk_1d_consistency" "WP_before_enable" "chk_hyper_invals" "lock_L2_source" "L2_refresh_jan1" "insert_ht" "L1_refresh_full" "L2b_refresh_jan1_2" "WP_before_release" "unlock" "chk_cagg_1d" "chk_1d_consistency"
324+
permutation "L1_refresh_full" "chk_1d_consistency" "WP_before_enable" "chk_hyper_invals" "lock_L2_source" "L2_refresh_jan1" "insert_ht" "L1_refresh_full" "L2b_refresh_jan1_2" "WP_before_release" "unlock_L2_source" "chk_cagg_1d" "chk_1d_consistency"
321325

322326
# Same as above, reverse refresh order.
323-
permutation "L1_refresh_full" "chk_1d_consistency" "WP_before_enable" "chk_hyper_invals" "lock_L2_source" "L2b_refresh_jan1_2" "insert_ht" "L1_refresh_full" "L2_refresh_jan1" "WP_before_release" "unlock" "chk_cagg_1d" "chk_1d_consistency"
327+
permutation "L1_refresh_full" "chk_1d_consistency" "WP_before_enable" "chk_hyper_invals" "lock_L2_source" "L2b_refresh_jan1_2" "insert_ht" "L1_refresh_full" "L2_refresh_jan1" "WP_before_release" "unlock_L2_source" "chk_cagg_1d" "chk_1d_consistency"
324328

325329
# Non-overlapping concurrent refreshes on L1. Both refreshes should succeed. Both are blocked before Txn 3.
326330
# Refreshing Jan 3 adds cagg invalidations that are partially overlapping with the one created by refreshing Jan 1.
@@ -330,8 +334,8 @@ permutation "WP_before_enable" "chk_hyper_invals" "L1_refresh_jan1" "insert_ht"
330334
# Non-overlapping concurrent refresh on L2. Both refreshes should succeed. Both are blocked before Txn 3.
331335
# Refreshing Jan 3 adds cagg invalidations that are partially overlapping with the one created by refreshing Jan 1.
332336
# Those invalidations are left behind, but both Jan 1 and Jan 3 are refreshed successfully.
333-
permutation "L1_refresh_full" "chk_1d_consistency" "WP_before_enable" "chk_hyper_invals" "lock_L2_source" "L2_refresh_jan1" "insert_ht" "L1_refresh_full" "L2b_refresh_jan3" "WP_before_release" "unlock" "chk_cagg_1d" "chk_1d_consistency"
337+
permutation "L1_refresh_full" "chk_1d_consistency" "WP_before_enable" "chk_hyper_invals" "lock_L2_source" "L2_refresh_jan1" "insert_ht" "L1_refresh_full" "L2b_refresh_jan3" "WP_before_release" "unlock_L2_source" "chk_cagg_1d" "chk_1d_consistency"
334338

335339
# L1 (txn3, deleting/inserting mat_inval entries) and L2 (txn2, processing mat_inval entries) should not block each other.
336340
# Lock materialization invalidation table to make both refreshes wait before processing entries, then release simultaneously.
337-
permutation "insert_ht" "WP_after_enable" "chk_6h_consistency" "L1_refresh_jan1" "lock_mat_invals" "WP_after_release" "L2_refresh_full" "unlock" "chk_mat_invals" "chk_6h_consistency" "chk_1d_consistency"
341+
permutation "insert_ht" "L1b_refresh_jan3" "chk_6h_consistency" "lock_6h_mat_table" "L1_refresh_jan1" "L2_refresh_full"("L1_refresh_jan1") "unlock_6h_mat_table"("L2_refresh_full") "chk_mat_invals" "chk_6h_consistency" "chk_1d_consistency"

0 commit comments

Comments
 (0)