Skip to content

Commit cf7e5b6

Browse files
dbecksvenklemm
dbeck
and
svenklemm
committed
Adding migration scripts for composite bloom filters
These scripts provide automated migration path for composite bloom filters created in 2.26. These are no-op on other versions and when there is no composite bloom filters at all. Co-authored-by: Sven Klemm <31455525+svenklemm@users.noreply.github.com> Signed-off-by: David Beck <david.beck.priv@gmail.com>
1 parent 6513377 commit cf7e5b6

3 files changed

Lines changed: 215 additions & 0 deletions

File tree

‎.unreleased/pr_9824‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Fixes: #9824 Adding migration scripts for composite bloom filters

‎sql/updates/latest-dev.sql‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
2+
-- The naming scheme for the composite bloom filter metadata columns has changed in 2.27.
3+
-- See the commit message for more details and the bug report here:
4+
--
5+
-- See bug report here: https://github.com/timescale/timescaledb/issues/9578
6+
--
7+
8+
DO $$
9+
DECLARE
10+
rename_data RECORD;
11+
BEGIN
12+
FOR rename_data IN
13+
--
14+
-- Make sure the old meta name actually exists for the compressed chunk
15+
-- relation, so the renaming that follows only impact real columns not
16+
-- some hallucinated ones.
17+
--
18+
SELECT
19+
att.attrelid::regclass,
20+
e.old_meta_name,
21+
e.new_meta_name
22+
FROM
23+
pg_attribute att,
24+
(
25+
--
26+
-- Calculate the old and new metadata column names of the composite bloom filters.
27+
-- Note that the new scheme always use a hash string to distinguish between the
28+
-- composite columns, but the old one only used the hash if the concatenated column
29+
-- names were too long.
30+
--
31+
SELECT
32+
compress_relid,
33+
CASE
34+
WHEN length(joined_cols_underscores) > 39
35+
THEN '_ts_meta_v2_bloomh_' || hash_underscores || '_' || joined_cols_underscores
36+
ELSE '_ts_meta_v2_bloomh_' || joined_cols_underscores
37+
END as old_meta_name,
38+
'_ts_meta_v2_bloomh_' || hash_zeroes || '_' || joined_cols_underscores as new_meta_name
39+
FROM
40+
(
41+
--
42+
-- Calculate the first 4 characters of the md5 hashes of both the
43+
-- zero and underscore concatenated column names of the composite
44+
-- bloom filters.
45+
--
46+
SELECT
47+
compress_relid,
48+
substr(md5(joined_cols_zeroes),1,4) as hash_zeroes,
49+
substr(md5(joined_cols_underscores),1,4) as hash_underscores,
50+
joined_cols_underscores
51+
FROM (
52+
--
53+
-- Select the compression settings objects that are actually a
54+
-- a 'bloom' filter, out of the already selected 'column' arrays
55+
-- and return the compressed chunk relation along with the column
56+
-- names concatenated with underscores as well as zeroes.
57+
--
58+
SELECT
59+
compress_relid,
60+
(SELECT string_agg(value::bytea, '\x00'::bytea) FROM jsonb_array_elements_text(cols::jsonb)) as joined_cols_zeroes,
61+
array_to_string(array(select jsonb_array_elements_text(cols::jsonb)), '_') as joined_cols_underscores
62+
FROM (
63+
--
64+
-- Select the settings where the column field is an array
65+
-- which is a must for the composite bloom filters
66+
--
67+
SELECT
68+
*,
69+
ae->>'column' cols
70+
FROM (
71+
--
72+
-- Capture the compression settings for the compressed
73+
-- tables, and separate the individual settings along
74+
-- with their types
75+
--
76+
SELECT
77+
compress_relid::text,
78+
jsonb_array_elements(index) ae,
79+
jsonb_array_elements(index)->>'type' ty
80+
FROM _timescaledb_catalog.compression_settings
81+
WHERE compress_relid IS NOT NULL
82+
) a
83+
WHERE jsonb_typeof(ae->'column') = 'array'
84+
) b
85+
WHERE ty = 'bloom'
86+
) c
87+
) d
88+
) e
89+
WHERE att.attrelid = e.compress_relid::regclass AND att.attname = e.old_meta_name
90+
LOOP
91+
RAISE NOTICE 'RENAMING: %s.% to %',
92+
rename_data.attrelid,
93+
rename_data.old_meta_name,
94+
rename_data.new_meta_name;
95+
96+
EXECUTE format(
97+
'ALTER TABLE %s RENAME COLUMN %I TO %I',
98+
rename_data.attrelid,
99+
rename_data.old_meta_name,
100+
rename_data.new_meta_name
101+
);
102+
END LOOP;
103+
END;
104+
$$;

‎sql/updates/reverse-dev.sql‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
-- The naming scheme for the composite bloom filter metadata columns has changed in 2.27.
2+
-- See the commit message for more details and the bug report here:
3+
--
4+
-- See bug report here: https://github.com/timescale/timescaledb/issues/9578
5+
--
6+
7+
DO $$
8+
DECLARE
9+
rename_data RECORD;
10+
BEGIN
11+
FOR rename_data IN
12+
--
13+
-- Make sure the old meta name actually exists for the compressed chunk
14+
-- relation, so the renaming that follows only impact real columns not
15+
-- some hallucinated ones.
16+
--
17+
SELECT
18+
att.attrelid::regclass,
19+
e.old_meta_name,
20+
e.new_meta_name,
21+
count(*) OVER (PARTITION BY att.attrelid, e.old_meta_name) AS n
22+
FROM
23+
pg_attribute att,
24+
(
25+
--
26+
-- Calculate the old and new metadata column names of the composite bloom filters.
27+
-- Note that the new scheme always use a hash string to distinguish between the
28+
-- composite columns, but the old one only used the hash if the concatenated column
29+
-- names were too long.
30+
--
31+
SELECT
32+
compress_relid,
33+
CASE
34+
WHEN length(joined_cols_underscores) > 39
35+
THEN '_ts_meta_v2_bloomh_' || hash_underscores || '_' || joined_cols_underscores
36+
ELSE '_ts_meta_v2_bloomh_' || joined_cols_underscores
37+
END as old_meta_name,
38+
'_ts_meta_v2_bloomh_' || hash_zeroes || '_' || joined_cols_underscores as new_meta_name
39+
FROM
40+
(
41+
--
42+
-- Calculate the first 4 characters of the md5 hashes of both the
43+
-- zero and underscore concatenated column names of the composite
44+
-- bloom filters.
45+
--
46+
SELECT
47+
compress_relid,
48+
substr(md5(joined_cols_zeroes),1,4) as hash_zeroes,
49+
substr(md5(joined_cols_underscores),1,4) as hash_underscores,
50+
joined_cols_underscores
51+
FROM (
52+
--
53+
-- Select the compression settings objects that are actually a
54+
-- a 'bloom' filter, out of the already selected 'column' arrays
55+
-- and return the compressed chunk relation along with the column
56+
-- names concatenated with underscores as well as zeroes.
57+
--
58+
SELECT
59+
compress_relid,
60+
(SELECT string_agg(value::bytea, '\x00'::bytea) FROM jsonb_array_elements_text(cols::jsonb)) as joined_cols_zeroes,
61+
array_to_string(array(select jsonb_array_elements_text(cols::jsonb)), '_') as joined_cols_underscores
62+
FROM (
63+
--
64+
-- Select the settings where the column field is an array
65+
-- which is a must for the composite bloom filters
66+
--
67+
SELECT
68+
*,
69+
ae->>'column' cols
70+
FROM (
71+
--
72+
-- Capture the compression settings for the compressed
73+
-- tables, and separate the individual settings along
74+
-- with their types
75+
--
76+
SELECT
77+
compress_relid::text,
78+
jsonb_array_elements(index) ae,
79+
jsonb_array_elements(index)->>'type' ty
80+
FROM _timescaledb_catalog.compression_settings
81+
WHERE compress_relid IS NOT NULL
82+
) a
83+
WHERE jsonb_typeof(ae->'column') = 'array'
84+
) b
85+
WHERE ty = 'bloom'
86+
) c
87+
) d
88+
) e
89+
WHERE att.attrelid = e.compress_relid::regclass AND att.attname = e.new_meta_name
90+
LOOP
91+
IF rename_data.n > 1 THEN
92+
RAISE EXCEPTION 'Downgrade is not possible because ambiguous composite bloom filters found. The ambiguous composite bloom filter column % for relation % need to be removed with the ALTER command.',
93+
rename_data.new_meta_name,
94+
rename_data.attrelid;
95+
END IF;
96+
97+
RAISE NOTICE 'RENAMING: %.% to %',
98+
rename_data.attrelid,
99+
rename_data.new_meta_name,
100+
rename_data.old_meta_name;
101+
102+
EXECUTE format(
103+
'ALTER TABLE %s RENAME COLUMN %I TO %I',
104+
rename_data.attrelid,
105+
rename_data.new_meta_name,
106+
rename_data.old_meta_name
107+
);
108+
END LOOP;
109+
END;
110+
$$;

0 commit comments

Comments
 (0)