Skip to content

Commit e2b8cec

Browse files
melihmutlumelihmutlu
authored
Verify cagg invalidation logs survive updates (#10351)
1 parent 0078f49 commit e2b8cec

4 files changed

Lines changed: 146 additions & 0 deletions

File tree

‎test/sql/updates/cleanup.continuous_aggs.v2.sql‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,17 @@ DROP VIEW mat_inval;
5454

5555
DROP TABLE inval_test;
5656

57+
\if :has_create_mat_view
58+
DROP MATERIALIZED VIEW mat_invallog_1;
59+
DROP MATERIALIZED VIEW mat_invallog_2;
60+
\else
61+
DROP VIEW mat_invallog_1;
62+
DROP VIEW mat_invallog_2;
63+
\endif
64+
65+
DROP TABLE inval_log_test;
66+
DROP TABLE inval_log_snapshot;
67+
5768
\if :has_create_mat_view
5869
DROP MATERIALIZED VIEW mat_ignoreinval;
5970
\else

‎test/sql/updates/post.continuous_aggs.sql‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,3 +103,66 @@ BEGIN
103103
END LOOP;
104104
END
105105
$$ LANGUAGE PLPGSQL;
106+
107+
-- Dump the invalidation log rows of the inval_log_test fixture so they
108+
-- are part of the baseline/updated/restored comparison.
109+
SELECT h.table_name AS hypertable,
110+
l.lowest_modified_value, l.greatest_modified_value
111+
FROM _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log l
112+
JOIN _timescaledb_catalog.hypertable h ON h.id = l.hypertable_id
113+
WHERE h.table_name = 'inval_log_test'
114+
ORDER BY 1, 2, 3;
115+
116+
SELECT ca.user_view_name AS cagg,
117+
l.lowest_modified_value, l.greatest_modified_value
118+
FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log l
119+
JOIN _timescaledb_catalog.continuous_agg ca ON ca.mat_hypertable_id = l.materialization_id
120+
WHERE ca.user_view_name IN ('mat_invallog_1', 'mat_invallog_2')
121+
ORDER BY 1, 2, 3;
122+
123+
-- Verify the live invalidation logs still hold exactly the rows
124+
-- snapshotted at the end of setup, i.e. an update script that rebuilds
125+
-- the log catalogs neither lost nor invented rows.
126+
DO $$
127+
DECLARE
128+
difference TEXT;
129+
BEGIN
130+
WITH live (log, name, lowest_modified_value, greatest_modified_value) AS (
131+
SELECT 'hypertable'::text, h.table_name,
132+
l.lowest_modified_value, l.greatest_modified_value
133+
FROM _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log l
134+
JOIN _timescaledb_catalog.hypertable h ON h.id = l.hypertable_id
135+
WHERE h.table_name = 'inval_log_test'
136+
UNION ALL
137+
SELECT 'materialization', ca.user_view_name,
138+
l.lowest_modified_value, l.greatest_modified_value
139+
FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log l
140+
JOIN _timescaledb_catalog.continuous_agg ca ON ca.mat_hypertable_id = l.materialization_id
141+
WHERE ca.user_view_name IN ('mat_invallog_1', 'mat_invallog_2')
142+
)
143+
SELECT string_agg(format('%s [%s]: (%s, %s, %s)', src, diff.log, diff.name,
144+
diff.lowest_modified_value, diff.greatest_modified_value), E'\n')
145+
INTO difference
146+
FROM (
147+
SELECT 'missing after update' AS src, *
148+
FROM (SELECT * FROM inval_log_snapshot
149+
EXCEPT ALL
150+
SELECT * FROM live) missing
151+
UNION ALL
152+
SELECT 'unexpected after update', *
153+
FROM (SELECT * FROM live
154+
EXCEPT ALL
155+
SELECT * FROM inval_log_snapshot) unexpected
156+
) diff (src, log, name, lowest_modified_value, greatest_modified_value);
157+
158+
IF difference IS NOT NULL THEN
159+
RAISE EXCEPTION 'invalidation log content changed across the update'
160+
USING DETAIL = difference;
161+
END IF;
162+
163+
IF NOT EXISTS (SELECT FROM inval_log_snapshot WHERE log = 'hypertable') OR
164+
NOT EXISTS (SELECT FROM inval_log_snapshot WHERE log = 'materialization') THEN
165+
RAISE EXCEPTION 'invalidation log snapshot is missing the expected pending rows';
166+
END IF;
167+
END
168+
$$ LANGUAGE PLPGSQL;

‎test/sql/updates/pre.cleanup.sql‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99
SET client_min_messages TO WARNING;
1010

1111
DROP MATERIALIZED VIEW IF EXISTS mat_inval CASCADE;
12+
DROP MATERIALIZED VIEW IF EXISTS mat_invallog_1 CASCADE;
13+
DROP MATERIALIZED VIEW IF EXISTS mat_invallog_2 CASCADE;
1214
DROP MATERIALIZED VIEW IF EXISTS mat_drop CASCADE;
1315
DROP MATERIALIZED VIEW IF EXISTS mat_before CASCADE;
1416
DROP MATERIALIZED VIEW IF EXISTS mat_conflict CASCADE;
@@ -21,6 +23,8 @@ DROP TABLE IF EXISTS public.hyper_timestamp;
2123
DROP TABLE IF EXISTS public."two_Partitions";
2224
DROP TABLE IF EXISTS conditions_before;
2325
DROP TABLE IF EXISTS inval_test;
26+
DROP TABLE IF EXISTS inval_log_test;
27+
DROP TABLE IF EXISTS inval_log_snapshot;
2428
DROP TABLE IF EXISTS int_time_test;
2529
DROP TABLE IF EXISTS conflict_test;
2630
DROP TABLE IF EXISTS drop_test;

‎test/sql/updates/setup.continuous_aggs.sql‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -299,5 +299,73 @@ CALL refresh_continuous_aggregate('mat_drop',NULL,NULL);
299299

300300
SELECT drop_chunks('drop_test', NOW() - INTERVAL '7 days');
301301

302+
-- Test that pending rows in the invalidation log catalogs survive the
303+
-- update/downgrade. Two continuous aggregates on one hypertable:
304+
-- refreshing only the first one moves hypertable-log entries into BOTH
305+
-- caggs' materialization logs but drains only the refreshed one,
306+
-- leaving a finite pending row for the second cagg; a final insert
307+
-- with no refresh afterwards leaves a pending row in the hypertable
308+
-- invalidation log.
309+
CREATE TABLE inval_log_test (time TIMESTAMPTZ NOT NULL, device TEXT NOT NULL, value INTEGER);
310+
SELECT create_hypertable('inval_log_test', 'time', chunk_time_interval => INTERVAL '1 week');
311+
312+
INSERT INTO inval_log_test
313+
SELECT ts, 'dev1', 1
314+
FROM generate_series('2020-01-01 00:00:00+00'::timestamptz,
315+
'2020-01-10 00:00:00+00'::timestamptz, '30 minutes') ts;
316+
317+
CREATE MATERIALIZED VIEW mat_invallog_1
318+
WITH (timescaledb.continuous, timescaledb.materialized_only=true)
319+
AS
320+
SELECT time_bucket('1 hour', time) AS bucket, device,
321+
count(*) AS cnt
322+
FROM inval_log_test
323+
GROUP BY bucket, device WITH NO DATA;
324+
325+
CREATE MATERIALIZED VIEW mat_invallog_2
326+
WITH (timescaledb.continuous, timescaledb.materialized_only=true)
327+
AS
328+
SELECT time_bucket('1 day', time) AS bucket, device,
329+
count(*) AS cnt
330+
FROM inval_log_test
331+
GROUP BY bucket, device WITH NO DATA;
332+
333+
CALL refresh_continuous_aggregate('mat_invallog_1', NULL, NULL);
334+
CALL refresh_continuous_aggregate('mat_invallog_2', NULL, NULL);
335+
336+
-- Below-threshold insert
337+
INSERT INTO inval_log_test
338+
SELECT ts, 'dev2', 42
339+
FROM generate_series('2020-01-05 10:00:00+00'::timestamptz,
340+
'2020-01-05 12:00:00+00'::timestamptz, '30 minutes') ts;
341+
342+
-- Refresh only the first cagg: moves ht invalidation log entries int
343+
-- materialization logs for both caggs, but drains only the refreshed one,
344+
-- leaving pending row for the second cagg.
345+
CALL refresh_continuous_aggregate('mat_invallog_1', NULL, NULL);
346+
347+
-- Below-threshold insert with no refresh afterwards, leaves ht invalidation
348+
-- log entry pending for both caggs.
349+
INSERT INTO inval_log_test
350+
SELECT ts, 'dev3', 5
351+
FROM generate_series('2020-01-07 07:00:00+00'::timestamptz,
352+
'2020-01-07 09:00:00+00'::timestamptz, '30 minutes') ts;
353+
354+
-- Snapshot invalidation log rows into a plain table so the post script can
355+
-- verify the live logs still hold the same content after the update rebuilt
356+
-- the catalogs.
357+
CREATE TABLE inval_log_snapshot AS
358+
SELECT 'hypertable'::text AS log, h.table_name AS name,
359+
l.lowest_modified_value, l.greatest_modified_value
360+
FROM _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log l
361+
JOIN _timescaledb_catalog.hypertable h ON h.id = l.hypertable_id
362+
WHERE h.table_name = 'inval_log_test'
363+
UNION ALL
364+
SELECT 'materialization', ca.user_view_name,
365+
l.lowest_modified_value, l.greatest_modified_value
366+
FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log l
367+
JOIN _timescaledb_catalog.continuous_agg ca ON ca.mat_hypertable_id = l.materialization_id
368+
WHERE ca.user_view_name IN ('mat_invallog_1', 'mat_invallog_2');
369+
302370
RESET timescaledb.enable_chunkwise_aggregation;
303371
RESET enable_hashagg;

0 commit comments

Comments
 (0)