Skip to content

Commit 94a1864

Browse files
melihmutlumelihmutlu
committed
Add granular_refresh_enabled flag to CAgg catalog
CAggs should be able to opt-in to granular refresh. Add a flag for CAggs to represent whether granular refresh is enabled on a CAgg.
1 parent a867c07 commit 94a1864

6 files changed

Lines changed: 200 additions & 2 deletions

File tree

‎sql/pre_install/tables.sql‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -327,6 +327,7 @@ CREATE TABLE _timescaledb_catalog.continuous_agg (
327327
direct_view_name name NOT NULL,
328328
materialized_only bool NOT NULL DEFAULT FALSE,
329329
schema_change_timestamp bigint,
330+
granular_refresh_enabled bool NOT NULL DEFAULT FALSE,
330331
-- table constraints
331332
CONSTRAINT continuous_agg_pkey PRIMARY KEY (mat_hypertable_id),
332333
CONSTRAINT continuous_agg_partial_view_schema_partial_view_name_key UNIQUE (partial_view_schema, partial_view_name),

‎sql/updates/latest-dev.sql‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -215,3 +215,99 @@ GRANT SELECT ON _timescaledb_catalog.chunk TO PUBLIC;
215215
-- END add chunk.relid
216216
--
217217

218+
SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.hypertable_cagg_settings', '');
219+
220+
-- Rebuild _timescaledb_catalog.continuous_agg to add granular_refresh_enabled.
221+
DROP VIEW IF EXISTS timescaledb_experimental.policies;
222+
223+
ALTER TABLE _timescaledb_catalog.continuous_aggs_jobs_refresh_ranges
224+
DROP CONSTRAINT continuous_aggs_jobs_refresh_ranges_materialization_id_fkey;
225+
ALTER TABLE _timescaledb_catalog.continuous_aggs_materialization_invalidation_log
226+
DROP CONSTRAINT continuous_aggs_materialization_invalid_materialization_id_fkey;
227+
ALTER TABLE _timescaledb_catalog.continuous_aggs_watermark
228+
DROP CONSTRAINT continuous_aggs_watermark_mat_hypertable_id_fkey;
229+
ALTER TABLE _timescaledb_catalog.continuous_aggs_materialization_ranges
230+
DROP CONSTRAINT continuous_aggs_materialization_ranges_materialization_id_fkey;
231+
232+
ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.continuous_agg;
233+
234+
CREATE TABLE _timescaledb_catalog._tmp_continuous_agg AS
235+
SELECT
236+
mat_hypertable_id,
237+
raw_hypertable_id,
238+
parent_mat_hypertable_id,
239+
user_view_schema,
240+
user_view_name,
241+
partial_view_schema,
242+
partial_view_name,
243+
direct_view_schema,
244+
direct_view_name,
245+
materialized_only,
246+
schema_change_timestamp
247+
FROM
248+
_timescaledb_catalog.continuous_agg
249+
ORDER BY
250+
mat_hypertable_id;
251+
252+
DROP TABLE _timescaledb_catalog.continuous_agg;
253+
254+
CREATE TABLE _timescaledb_catalog.continuous_agg (
255+
mat_hypertable_id integer NOT NULL,
256+
raw_hypertable_id integer NOT NULL,
257+
parent_mat_hypertable_id integer,
258+
user_view_schema name NOT NULL,
259+
user_view_name name NOT NULL,
260+
partial_view_schema name NOT NULL,
261+
partial_view_name name NOT NULL,
262+
direct_view_schema name NOT NULL,
263+
direct_view_name name NOT NULL,
264+
materialized_only bool NOT NULL DEFAULT FALSE,
265+
schema_change_timestamp bigint,
266+
granular_refresh_enabled bool NOT NULL DEFAULT FALSE,
267+
CONSTRAINT continuous_agg_pkey PRIMARY KEY (mat_hypertable_id),
268+
CONSTRAINT continuous_agg_partial_view_schema_partial_view_name_key UNIQUE (partial_view_schema, partial_view_name),
269+
CONSTRAINT continuous_agg_user_view_schema_user_view_name_key UNIQUE (user_view_schema, user_view_name),
270+
CONSTRAINT continuous_agg_mat_hypertable_id_fkey
271+
FOREIGN KEY (mat_hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id) ON DELETE CASCADE,
272+
CONSTRAINT continuous_agg_raw_hypertable_id_fkey
273+
FOREIGN KEY (raw_hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id) ON DELETE CASCADE,
274+
CONSTRAINT continuous_agg_parent_mat_hypertable_id_fkey
275+
FOREIGN KEY (parent_mat_hypertable_id)
276+
REFERENCES _timescaledb_catalog.continuous_agg (mat_hypertable_id) ON DELETE CASCADE
277+
);
278+
279+
INSERT INTO _timescaledb_catalog.continuous_agg (
280+
mat_hypertable_id,
281+
raw_hypertable_id,
282+
parent_mat_hypertable_id,
283+
user_view_schema,
284+
user_view_name,
285+
partial_view_schema,
286+
partial_view_name,
287+
direct_view_schema,
288+
direct_view_name,
289+
materialized_only,
290+
schema_change_timestamp
291+
)
292+
SELECT * FROM _timescaledb_catalog._tmp_continuous_agg;
293+
DROP TABLE _timescaledb_catalog._tmp_continuous_agg;
294+
295+
CREATE INDEX continuous_agg_raw_hypertable_id_idx ON _timescaledb_catalog.continuous_agg (raw_hypertable_id);
296+
297+
SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.continuous_agg', '');
298+
299+
GRANT SELECT ON TABLE _timescaledb_catalog.continuous_agg TO PUBLIC;
300+
301+
ALTER TABLE _timescaledb_catalog.continuous_aggs_jobs_refresh_ranges
302+
ADD CONSTRAINT continuous_aggs_jobs_refresh_ranges_materialization_id_fkey
303+
FOREIGN KEY (materialization_id) REFERENCES _timescaledb_catalog.continuous_agg(mat_hypertable_id) ON DELETE CASCADE;
304+
ALTER TABLE _timescaledb_catalog.continuous_aggs_materialization_invalidation_log
305+
ADD CONSTRAINT continuous_aggs_materialization_invalid_materialization_id_fkey
306+
FOREIGN KEY (materialization_id) REFERENCES _timescaledb_catalog.continuous_agg(mat_hypertable_id) ON DELETE CASCADE;
307+
ALTER TABLE _timescaledb_catalog.continuous_aggs_materialization_ranges
308+
ADD CONSTRAINT continuous_aggs_materialization_ranges_materialization_id_fkey
309+
FOREIGN KEY (materialization_id) REFERENCES _timescaledb_catalog.continuous_agg(mat_hypertable_id) ON DELETE CASCADE;
310+
ALTER TABLE _timescaledb_catalog.continuous_aggs_watermark
311+
ADD CONSTRAINT continuous_aggs_watermark_mat_hypertable_id_fkey
312+
FOREIGN KEY (mat_hypertable_id) REFERENCES _timescaledb_catalog.continuous_agg(mat_hypertable_id) ON DELETE CASCADE;
313+
-- end rebuild _timescaledb_catalog.continuous_agg --

‎sql/updates/reverse-dev.sql‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -315,3 +315,96 @@ SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.continuous_aggs
315315
-- drop continuous_aggs_tenant_tracking
316316
ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.continuous_aggs_tenant_tracking;
317317
DROP TABLE _timescaledb_catalog.continuous_aggs_tenant_tracking;
318+
-- Rebuild _timescaledb_catalog.continuous_agg to drop granular_refresh_enabled.
319+
DROP VIEW IF EXISTS timescaledb_experimental.policies;
320+
321+
ALTER TABLE _timescaledb_catalog.continuous_aggs_jobs_refresh_ranges
322+
DROP CONSTRAINT continuous_aggs_jobs_refresh_ranges_materialization_id_fkey;
323+
ALTER TABLE _timescaledb_catalog.continuous_aggs_materialization_invalidation_log
324+
DROP CONSTRAINT continuous_aggs_materialization_invalid_materialization_id_fkey;
325+
ALTER TABLE _timescaledb_catalog.continuous_aggs_watermark
326+
DROP CONSTRAINT continuous_aggs_watermark_mat_hypertable_id_fkey;
327+
ALTER TABLE _timescaledb_catalog.continuous_aggs_materialization_ranges
328+
DROP CONSTRAINT continuous_aggs_materialization_ranges_materialization_id_fkey;
329+
330+
ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.continuous_agg;
331+
332+
CREATE TABLE _timescaledb_catalog._tmp_continuous_agg AS
333+
SELECT
334+
mat_hypertable_id,
335+
raw_hypertable_id,
336+
parent_mat_hypertable_id,
337+
user_view_schema,
338+
user_view_name,
339+
partial_view_schema,
340+
partial_view_name,
341+
direct_view_schema,
342+
direct_view_name,
343+
materialized_only,
344+
schema_change_timestamp
345+
FROM
346+
_timescaledb_catalog.continuous_agg
347+
ORDER BY
348+
mat_hypertable_id;
349+
350+
DROP TABLE _timescaledb_catalog.continuous_agg;
351+
352+
CREATE TABLE _timescaledb_catalog.continuous_agg (
353+
mat_hypertable_id integer NOT NULL,
354+
raw_hypertable_id integer NOT NULL,
355+
parent_mat_hypertable_id integer,
356+
user_view_schema name NOT NULL,
357+
user_view_name name NOT NULL,
358+
partial_view_schema name NOT NULL,
359+
partial_view_name name NOT NULL,
360+
direct_view_schema name NOT NULL,
361+
direct_view_name name NOT NULL,
362+
materialized_only bool NOT NULL DEFAULT FALSE,
363+
schema_change_timestamp bigint,
364+
CONSTRAINT continuous_agg_pkey PRIMARY KEY (mat_hypertable_id),
365+
CONSTRAINT continuous_agg_partial_view_schema_partial_view_name_key UNIQUE (partial_view_schema, partial_view_name),
366+
CONSTRAINT continuous_agg_user_view_schema_user_view_name_key UNIQUE (user_view_schema, user_view_name),
367+
CONSTRAINT continuous_agg_mat_hypertable_id_fkey
368+
FOREIGN KEY (mat_hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id) ON DELETE CASCADE,
369+
CONSTRAINT continuous_agg_raw_hypertable_id_fkey
370+
FOREIGN KEY (raw_hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id) ON DELETE CASCADE,
371+
CONSTRAINT continuous_agg_parent_mat_hypertable_id_fkey
372+
FOREIGN KEY (parent_mat_hypertable_id)
373+
REFERENCES _timescaledb_catalog.continuous_agg (mat_hypertable_id) ON DELETE CASCADE
374+
);
375+
376+
INSERT INTO _timescaledb_catalog.continuous_agg (
377+
mat_hypertable_id,
378+
raw_hypertable_id,
379+
parent_mat_hypertable_id,
380+
user_view_schema,
381+
user_view_name,
382+
partial_view_schema,
383+
partial_view_name,
384+
direct_view_schema,
385+
direct_view_name,
386+
materialized_only,
387+
schema_change_timestamp
388+
)
389+
SELECT * FROM _timescaledb_catalog._tmp_continuous_agg;
390+
DROP TABLE _timescaledb_catalog._tmp_continuous_agg;
391+
392+
CREATE INDEX continuous_agg_raw_hypertable_id_idx ON _timescaledb_catalog.continuous_agg (raw_hypertable_id);
393+
394+
SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.continuous_agg', '');
395+
396+
GRANT SELECT ON TABLE _timescaledb_catalog.continuous_agg TO PUBLIC;
397+
398+
ALTER TABLE _timescaledb_catalog.continuous_aggs_jobs_refresh_ranges
399+
ADD CONSTRAINT continuous_aggs_jobs_refresh_ranges_materialization_id_fkey
400+
FOREIGN KEY (materialization_id) REFERENCES _timescaledb_catalog.continuous_agg(mat_hypertable_id) ON DELETE CASCADE;
401+
ALTER TABLE _timescaledb_catalog.continuous_aggs_materialization_invalidation_log
402+
ADD CONSTRAINT continuous_aggs_materialization_invalid_materialization_id_fkey
403+
FOREIGN KEY (materialization_id) REFERENCES _timescaledb_catalog.continuous_agg(mat_hypertable_id) ON DELETE CASCADE;
404+
ALTER TABLE _timescaledb_catalog.continuous_aggs_materialization_ranges
405+
ADD CONSTRAINT continuous_aggs_materialization_ranges_materialization_id_fkey
406+
FOREIGN KEY (materialization_id) REFERENCES _timescaledb_catalog.continuous_agg(mat_hypertable_id) ON DELETE CASCADE;
407+
ALTER TABLE _timescaledb_catalog.continuous_aggs_watermark
408+
ADD CONSTRAINT continuous_aggs_watermark_mat_hypertable_id_fkey
409+
FOREIGN KEY (mat_hypertable_id) REFERENCES _timescaledb_catalog.continuous_agg(mat_hypertable_id) ON DELETE CASCADE;
410+
-- end rebuild _timescaledb_catalog.continuous_agg --

‎src/ts_catalog/catalog.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -789,6 +789,7 @@ typedef enum Anum_continuous_agg
789789
Anum_continuous_agg_direct_view_name,
790790
Anum_continuous_agg_materialize_only,
791791
Anum_continuous_agg_schema_change_timestamp,
792+
Anum_continuous_agg_granular_refresh_enabled,
792793
_Anum_continuous_agg_max,
793794
} Anum_continuous_agg;
794795

@@ -807,6 +808,7 @@ typedef struct FormData_continuous_agg
807808
NameData direct_view_name;
808809
bool materialized_only;
809810
int64 schema_change_timestamp;
811+
bool granular_refresh_enabled;
810812
} FormData_continuous_agg;
811813

812814
typedef FormData_continuous_agg *Form_continuous_agg;

‎src/ts_catalog/continuous_agg.c‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -323,6 +323,9 @@ continuous_agg_formdata_make_tuple(const FormData_continuous_agg *fd, TupleDesc
323323
Int64GetDatum(fd->schema_change_timestamp);
324324
}
325325

326+
values[AttrNumberGetAttrOffset(Anum_continuous_agg_granular_refresh_enabled)] =
327+
BoolGetDatum(fd->granular_refresh_enabled);
328+
326329
return heap_form_tuple(desc, values, nulls);
327330
}
328331

@@ -386,6 +389,9 @@ continuous_agg_formdata_fill(FormData_continuous_agg *fd, const TupleInfo *ti)
386389
values[AttrNumberGetAttrOffset(Anum_continuous_agg_schema_change_timestamp)]);
387390
}
388391

392+
fd->granular_refresh_enabled =
393+
DatumGetBool(values[AttrNumberGetAttrOffset(Anum_continuous_agg_granular_refresh_enabled)]);
394+
389395
if (should_free)
390396
{
391397
heap_freetuple(tuple);

‎tsl/test/expected/cagg_bgw.out‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,8 +64,8 @@ SELECT * FROM timescaledb_information.job_stats;
6464
-------------------+-----------------+--------+---------------------+------------------------+-----------------+------------+-------------------+------------+------------+-----------------+----------------
6565

6666
SELECT * FROM _timescaledb_catalog.continuous_agg;
67-
mat_hypertable_id | raw_hypertable_id | parent_mat_hypertable_id | user_view_schema | user_view_name | partial_view_schema | partial_view_name | direct_view_schema | direct_view_name | materialized_only | schema_change_timestamp
68-
-------------------+-------------------+--------------------------+------------------+----------------+---------------------+-------------------+--------------------+------------------+-------------------+-------------------------
67+
mat_hypertable_id | raw_hypertable_id | parent_mat_hypertable_id | user_view_schema | user_view_name | partial_view_schema | partial_view_name | direct_view_schema | direct_view_name | materialized_only | schema_change_timestamp | granular_refresh_enabled
68+
-------------------+-------------------+--------------------------+------------------+----------------+---------------------+-------------------+--------------------+------------------+-------------------+-------------------------+--------------------------
6969

7070
-- though user on access node has required GRANTS, this will propagate GRANTS to the connected data nodes
7171
GRANT CREATE ON SCHEMA public TO :ROLE_DEFAULT_PERM_USER;

0 commit comments

Comments
 (0)