Skip to content

Commit edc83df

Browse files
melihmutlumelihmutlu
committed
Verify cagg invalidation logs survive updates
Update scripts can rebuild the invalidation log catalogs, but the update tests never checked their contents. Add a fixture with two continuous aggregates on one hypertable that leaves pending rows in both invalidation logs at upgrade time, snapshot the rows into a plain table at the end of setup, and verify after the update that the live logs still match the snapshot exactly, raising an error on any missing or unexpected row.
1 parent 56f1656 commit edc83df

4 files changed

Lines changed: 146 additions & 0 deletions

File tree

‎test/sql/updates/cleanup.continuous_aggs.v2.sql‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,17 @@ DROP VIEW mat_inval;
5454

5555
DROP TABLE inval_test;
5656

57+
\if :has_create_mat_view
58+
DROP MATERIALIZED VIEW mat_invallog_1;
59+
DROP MATERIALIZED VIEW mat_invallog_2;
60+
\else
61+
DROP VIEW mat_invallog_1;
62+
DROP VIEW mat_invallog_2;
63+
\endif
64+
65+
DROP TABLE inval_log_test;
66+
DROP TABLE inval_log_snapshot;
67+
5768
\if :has_create_mat_view
5869
DROP MATERIALIZED VIEW mat_ignoreinval;
5970
\else

‎test/sql/updates/post.continuous_aggs.sql‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,3 +103,66 @@ BEGIN
103103
END LOOP;
104104
END
105105
$$ LANGUAGE PLPGSQL;
106+
107+
-- Dump the invalidation log rows of the inval_log_test fixture so they
108+
-- are part of the baseline/updated/restored comparison.
109+
SELECT h.table_name AS hypertable,
110+
l.lowest_modified_value, l.greatest_modified_value
111+
FROM _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log l
112+
JOIN _timescaledb_catalog.hypertable h ON h.id = l.hypertable_id
113+
WHERE h.table_name = 'inval_log_test'
114+
ORDER BY 1, 2, 3;
115+
116+
SELECT ca.user_view_name AS cagg,
117+
l.lowest_modified_value, l.greatest_modified_value
118+
FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log l
119+
JOIN _timescaledb_catalog.continuous_agg ca ON ca.mat_hypertable_id = l.materialization_id
120+
WHERE ca.user_view_name IN ('mat_invallog_1', 'mat_invallog_2')
121+
ORDER BY 1, 2, 3;
122+
123+
-- Verify the live invalidation logs still hold exactly the rows
124+
-- snapshotted at the end of setup, i.e. an update script that rebuilds
125+
-- the log catalogs neither lost nor invented rows.
126+
DO $$
127+
DECLARE
128+
difference TEXT;
129+
BEGIN
130+
WITH live (log, name, lowest_modified_value, greatest_modified_value) AS (
131+
SELECT 'hypertable'::text, h.table_name,
132+
l.lowest_modified_value, l.greatest_modified_value
133+
FROM _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log l
134+
JOIN _timescaledb_catalog.hypertable h ON h.id = l.hypertable_id
135+
WHERE h.table_name = 'inval_log_test'
136+
UNION ALL
137+
SELECT 'materialization', ca.user_view_name,
138+
l.lowest_modified_value, l.greatest_modified_value
139+
FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log l
140+
JOIN _timescaledb_catalog.continuous_agg ca ON ca.mat_hypertable_id = l.materialization_id
141+
WHERE ca.user_view_name IN ('mat_invallog_1', 'mat_invallog_2')
142+
)
143+
SELECT string_agg(format('%s [%s]: (%s, %s, %s)', src, diff.log, diff.name,
144+
diff.lowest_modified_value, diff.greatest_modified_value), E'\n')
145+
INTO difference
146+
FROM (
147+
SELECT 'missing after update' AS src, *
148+
FROM (SELECT * FROM inval_log_snapshot
149+
EXCEPT ALL
150+
SELECT * FROM live) missing
151+
UNION ALL
152+
SELECT 'unexpected after update', *
153+
FROM (SELECT * FROM live
154+
EXCEPT ALL
155+
SELECT * FROM inval_log_snapshot) unexpected
156+
) diff (src, log, name, lowest_modified_value, greatest_modified_value);
157+
158+
IF difference IS NOT NULL THEN
159+
RAISE EXCEPTION 'invalidation log content changed across the update'
160+
USING DETAIL = difference;
161+
END IF;
162+
163+
IF NOT EXISTS (SELECT FROM inval_log_snapshot WHERE log = 'hypertable') OR
164+
NOT EXISTS (SELECT FROM inval_log_snapshot WHERE log = 'materialization') THEN
165+
RAISE EXCEPTION 'invalidation log snapshot is missing the expected pending rows';
166+
END IF;
167+
END
168+
$$ LANGUAGE PLPGSQL;

‎test/sql/updates/pre.cleanup.sql‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99
SET client_min_messages TO WARNING;
1010

1111
DROP MATERIALIZED VIEW IF EXISTS mat_inval CASCADE;
12+
DROP MATERIALIZED VIEW IF EXISTS mat_invallog_1 CASCADE;
13+
DROP MATERIALIZED VIEW IF EXISTS mat_invallog_2 CASCADE;
1214
DROP MATERIALIZED VIEW IF EXISTS mat_drop CASCADE;
1315
DROP MATERIALIZED VIEW IF EXISTS mat_before CASCADE;
1416
DROP MATERIALIZED VIEW IF EXISTS mat_conflict CASCADE;
@@ -21,6 +23,8 @@ DROP TABLE IF EXISTS public.hyper_timestamp;
2123
DROP TABLE IF EXISTS public."two_Partitions";
2224
DROP TABLE IF EXISTS conditions_before;
2325
DROP TABLE IF EXISTS inval_test;
26+
DROP TABLE IF EXISTS inval_log_test;
27+
DROP TABLE IF EXISTS inval_log_snapshot;
2428
DROP TABLE IF EXISTS int_time_test;
2529
DROP TABLE IF EXISTS conflict_test;
2630
DROP TABLE IF EXISTS drop_test;

‎test/sql/updates/setup.continuous_aggs.sql‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -299,5 +299,73 @@ CALL refresh_continuous_aggregate('mat_drop',NULL,NULL);
299299

300300
SELECT drop_chunks('drop_test', NOW() - INTERVAL '7 days');
301301

302+
-- Test that pending rows in the invalidation log catalogs survive the
303+
-- update/downgrade. Two continuous aggregates on one hypertable:
304+
-- refreshing only the first one moves hypertable-log entries into BOTH
305+
-- caggs' materialization logs but drains only the refreshed one,
306+
-- leaving a finite pending row for the second cagg; a final insert
307+
-- with no refresh afterwards leaves a pending row in the hypertable
308+
-- invalidation log.
309+
CREATE TABLE inval_log_test (time TIMESTAMPTZ NOT NULL, device TEXT NOT NULL, value INTEGER);
310+
SELECT create_hypertable('inval_log_test', 'time', chunk_time_interval => INTERVAL '1 week');
311+
312+
INSERT INTO inval_log_test
313+
SELECT ts, 'dev1', 1
314+
FROM generate_series('2020-01-01 00:00:00+00'::timestamptz,
315+
'2020-01-10 00:00:00+00'::timestamptz, '30 minutes') ts;
316+
317+
CREATE MATERIALIZED VIEW mat_invallog_1
318+
WITH (timescaledb.continuous, timescaledb.materialized_only=true)
319+
AS
320+
SELECT time_bucket('1 hour', time) AS bucket, device,
321+
count(*) AS cnt
322+
FROM inval_log_test
323+
GROUP BY bucket, device WITH NO DATA;
324+
325+
CREATE MATERIALIZED VIEW mat_invallog_2
326+
WITH (timescaledb.continuous, timescaledb.materialized_only=true)
327+
AS
328+
SELECT time_bucket('1 day', time) AS bucket, device,
329+
count(*) AS cnt
330+
FROM inval_log_test
331+
GROUP BY bucket, device WITH NO DATA;
332+
333+
CALL refresh_continuous_aggregate('mat_invallog_1', NULL, NULL);
334+
CALL refresh_continuous_aggregate('mat_invallog_2', NULL, NULL);
335+
336+
-- Below-threshold insert
337+
INSERT INTO inval_log_test
338+
SELECT ts, 'dev2', 42
339+
FROM generate_series('2020-01-05 10:00:00+00'::timestamptz,
340+
'2020-01-05 12:00:00+00'::timestamptz, '30 minutes') ts;
341+
342+
-- Refresh only the first cagg: moves ht invalidation log entries int
343+
-- materialization logs for both caggs, but drains only the refreshed one,
344+
-- leaving pending row for the second cagg.
345+
CALL refresh_continuous_aggregate('mat_invallog_1', NULL, NULL);
346+
347+
-- Below-threshold insert with no refresh afterwards, leaves ht invalidation
348+
-- log entry pending for both caggs.
349+
INSERT INTO inval_log_test
350+
SELECT ts, 'dev3', 5
351+
FROM generate_series('2020-01-07 07:00:00+00'::timestamptz,
352+
'2020-01-07 09:00:00+00'::timestamptz, '30 minutes') ts;
353+
354+
-- Snapshot invalidation log rows into a plain table so the post script can
355+
-- verify the live logs still hold the same content after the update rebuilt
356+
-- the catalogs.
357+
CREATE TABLE inval_log_snapshot AS
358+
SELECT 'hypertable'::text AS log, h.table_name AS name,
359+
l.lowest_modified_value, l.greatest_modified_value
360+
FROM _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log l
361+
JOIN _timescaledb_catalog.hypertable h ON h.id = l.hypertable_id
362+
WHERE h.table_name = 'inval_log_test'
363+
UNION ALL
364+
SELECT 'materialization', ca.user_view_name,
365+
l.lowest_modified_value, l.greatest_modified_value
366+
FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log l
367+
JOIN _timescaledb_catalog.continuous_agg ca ON ca.mat_hypertable_id = l.materialization_id
368+
WHERE ca.user_view_name IN ('mat_invallog_1', 'mat_invallog_2');
369+
302370
RESET timescaledb.enable_chunkwise_aggregation;
303371
RESET enable_hashagg;

0 commit comments

Comments
 (0)