Normalize chunk names in updates/downgrade test

With the removal of compressed chunks as separate chunk entry all
chunks will have different numbering between fresh install/upgrade.
Prepare for this by ignoring chunk numbering in update test.

Author: svenklemm

test/sql/updates/post.catalog.sql

@@ -2,6 +2,16 @@
 -- Please see the included NOTICE for copyright information and
 -- LICENSE-APACHE for a copy of the license.
 
+-- Chunk relation names are not deterministic between a fresh install and a post-upgrade catalog.
+CREATE OR REPLACE FUNCTION pg_temp.normalize_chunk(t text) RETURNS text
+  LANGUAGE sql IMMUTABLE AS $$
+  SELECT regexp_replace(
+           regexp_replace(t,
+             'compress_hyper_[0-9]+_[0-9]+_chunk|_hyper_[0-9]+_[0-9]+_chunk_compressed',
+             'compressed_chunk', 'g'),
+           '(_hyper_[0-9]+)_[0-9]+_chunk', '\1_X_chunk', 'g')
+$$;
+
 SELECT NOT (extversion >= '2.19.0' AND extversion <= '2.20.3') AS has_fixed_compression_algorithms
   FROM pg_extension
  WHERE extname = 'timescaledb' \gset
@@ -161,10 +171,10 @@ ORDER BY conrelid::regclass::text, contype, conname, confrelid::regclass::text;
 
 -- child tables
 SELECT parent.relname AS table_name,
-    i.inhrelid::regclass AS child_table
+    pg_temp.normalize_chunk(i.inhrelid::regclass::text) AS child_table
 FROM pg_catalog.pg_inherits i
 JOIN pg_catalog.pg_class parent ON parent.oid = i.inhparent AND parent.relnamespace = 'public'::regnamespace
-ORDER BY parent.relname, i.inhrelid::regclass::text;
+ORDER BY parent.relname, pg_temp.normalize_chunk(i.inhrelid::regclass::text);
 
 -- Keep the output backward compatible
 \if :PG_UPGRADE_TEST
@@ -199,24 +209,25 @@ ORDER BY parent.relname, i.inhrelid::regclass::text;
 -- The list of tables configured to be dumped.
 SELECT unnest(extconfig)::regclass::text, unnest(extcondition) FROM pg_extension WHERE extname = 'timescaledb' ORDER BY 1;
 
--- Show chunks that include owner in the output
-SELECT c.id, c.hypertable_id, c.schema_name, c.table_name, cl.relowner::regrole
+-- Show chunks that include owner in the output. The chunk id is not
+-- deterministic post-upgrade, so drop it and normalize the chunk relation name.
+SELECT c.hypertable_id, c.schema_name, pg_temp.normalize_chunk(c.table_name) AS table_name, cl.relowner::regrole
 FROM  _timescaledb_catalog.chunk c
 INNER JOIN pg_class cl ON (cl.oid=format('%I.%I', schema_name, table_name)::regclass)
-ORDER BY c.id, c.hypertable_id;
+ORDER BY c.hypertable_id, pg_temp.normalize_chunk(c.table_name);
 
--- Per-chunk dimensional ranges. Slice ids are assigned by SERIAL and differ
--- between a fresh install and a post-upgrade catalog, so dump path-stable
--- columns only.
+-- Per-chunk dimensional ranges. Slice ids are assigned by SERIAL and chunk ids
+-- are renumbered, so both differ between a fresh install and a post-upgrade
+-- catalog. Dump and order by the dimensional range only.
 \if :has_chunk_owned_slices
-SELECT ds.chunk_id, ds.dimension_id, ds.range_start, ds.range_end
+SELECT ds.dimension_id, ds.range_start, ds.range_end
 FROM _timescaledb_catalog.dimension_slice ds
-ORDER BY ds.chunk_id, ds.dimension_id;
+ORDER BY ds.dimension_id, ds.range_start, ds.range_end;
 \else
-SELECT cc.chunk_id, ds.dimension_id, ds.range_start, ds.range_end
+SELECT ds.dimension_id, ds.range_start, ds.range_end
 FROM _timescaledb_catalog.chunk_constraint cc
 JOIN _timescaledb_catalog.dimension_slice ds ON ds.id = cc.dimension_slice_id
-ORDER BY cc.chunk_id, ds.dimension_id;
+ORDER BY ds.dimension_id, ds.range_start, ds.range_end;
 \endif
 
 -- Show attributes of all regclass objects belonging to our extension
@@ -240,7 +251,7 @@ ORDER BY attrelid::regclass::text COLLATE "C", a.attnum;
 -- form (2.28.0) compare equal. The dimensional CHECKs are named
 -- "constraint_<slice_id>" and slice ids are not deterministic between a
 -- fresh install and a post-upgrade catalog, so collapse the suffix too.
-SELECT conrelid::regclass::text,
+SELECT pg_temp.normalize_chunk(conrelid::regclass::text) AS conrelid,
        regexp_replace(
            regexp_replace(conname, '^([0-9]+_){1,2}', ''),
            '^constraint_[0-9]+$', 'constraint_dim') AS conname,
@@ -256,4 +267,8 @@ ORDER BY 1, 2, 3;
 -- orderby sparse index type changed across releases and existing chunks are
 -- not rewritten on upgrade), so it differs between a fresh install and a
 -- post-upgrade catalog. Skip it and compare the remaining columns.
-SELECT relid, compress_relid, segmentby, orderby, orderby_desc, orderby_nullsfirst FROM _timescaledb_catalog.compression_settings ORDER BY relid::regclass::text;
+SELECT pg_temp.normalize_chunk(relid::regclass::text) AS relid,
+       pg_temp.normalize_chunk(compress_relid::regclass::text) AS compress_relid,
+       segmentby, orderby, orderby_desc, orderby_nullsfirst
+FROM _timescaledb_catalog.compression_settings
+ORDER BY pg_temp.normalize_chunk(relid::regclass::text), segmentby, orderby;

test/sql/updates/post.chunk_skipping.sql

@@ -6,6 +6,8 @@
 -- exclude the rows with chunk_id = 0 because we cannot keep those on
 -- downgrade due to FK constraint. Showing them would mean a diff in
 -- the output. We can still test that 0 chunk_ids are converted to
--- NULL values during upgrades, however.
-SELECT * FROM _timescaledb_catalog.chunk_column_stats
+-- NULL values during upgrades, however. The chunk id itself is renumbered
+-- across the upgrade, so report only whether it is set instead of its value.
+SELECT id, hypertable_id, (chunk_id IS NOT NULL) AS chunk_id_set, column_name, range_start, range_end, valid
+FROM _timescaledb_catalog.chunk_column_stats
 WHERE chunk_id IS NULL OR chunk_id > 0 ORDER BY id;

test/sql/updates/post.compression.sql

@@ -43,9 +43,11 @@ SELECT hypertable_name,
        (SELECT relacl FROM pg_class WHERE oid = hypertable_name::regclass) AS hypertable_acl,
        compressed_hypertable_name,
        (SELECT relacl FROM pg_class WHERE oid = compressed_hypertable_name::regclass) AS compressed_hypertable_acl,
-       compressed_chunk_name,
+       -- the compressed relation keeps its pre-upgrade name across a downgrade,
+       -- so normalize it; the acl is still looked up by the real name
+       pg_temp.normalize_chunk(compressed_chunk_name) AS compressed_chunk_name,
        (SELECT relacl FROM pg_class WHERE oid = compressed_chunk_name::regclass) AS compressed_chunk_acl
   FROM table_summary
-  ORDER BY hypertable_name, compressed_hypertable_name, compressed_chunk_name;
+  ORDER BY hypertable_name, compressed_hypertable_name, pg_temp.normalize_chunk(compressed_chunk_name);
 \x off
 

test/sql/updates/post.continuous_aggs.sql

@@ -51,9 +51,10 @@ ON (cl.oid IN (format('%I.%I', h.schema_name, h.table_name)::regclass,
                format('%I.%I', partial_view_schema, partial_view_name)::regclass))
 ORDER BY reloid, relacl;
 
--- Output ACLs for chunks on materialized hypertables
+-- Output ACLs for chunks on materialized hypertables. Chunk relation names are
+-- renumbered across the upgrade, so normalize them.
 SELECT inhparent::regclass::text AS parent,
-       cl.oid::regclass::text AS chunk,
+       pg_temp.normalize_chunk(cl.oid::regclass::text) AS chunk,
        unnest(relacl)::text AS acl
 FROM _timescaledb_catalog.continuous_agg ca
 JOIN _timescaledb_catalog.hypertable h

test/sql/updates/post.integrity_test.sql

@@ -5,8 +5,17 @@
 -- We do not dump the size of the tables here since that might differ
 -- between an updated node and a restored node. For examples, stats
 -- tables can have different sizes, and this is not relevant for an
--- update test.
-\dt _timescaledb_internal.*
+-- update test. This mirrors \dt _timescaledb_internal.* but normalizes the
+-- chunk relation names, which are renumbered and renamed across the upgrade.
+SELECT n.nspname AS "Schema",
+       pg_temp.normalize_chunk(c.relname) AS "Name",
+       'table' AS "Type",
+       pg_catalog.pg_get_userbyid(c.relowner) AS "Owner"
+FROM pg_catalog.pg_class c
+JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
+WHERE n.nspname = '_timescaledb_internal'
+  AND c.relkind = 'r'
+ORDER BY 1, 2, 4;
 
 CREATE OR REPLACE FUNCTION timescaledb_integrity_test()
     RETURNS VOID LANGUAGE PLPGSQL STABLE AS

test/sql/updates/post.sequences.sql

@@ -9,9 +9,12 @@
 -- burned ids for slices of chunks dropped before the upgrade. Those dropped
 -- slices leave no trace to reconstruct, so the next id legitimately differs
 -- from a clean install even though the slice rows match.
+-- chunk_id_seq is excluded too: a fresh install no longer burns chunk ids for
+-- compressed relations, so the next id is lower than in a pre-upgrade catalog.
 SELECT seqrelid::regclass,
   CASE WHEN seqrelid::regclass::text IN ('_timescaledb_catalog.chunk_constraint_name',
-                                         '_timescaledb_catalog.dimension_slice_id_seq')
+                                         '_timescaledb_catalog.dimension_slice_id_seq',
+                                         '_timescaledb_catalog.chunk_id_seq')
        THEN NULL ELSE nextval(seqrelid) END AS nextval,
   seqstart,
   seqincrement,

test/sql/updates/post.sparse_index.sql

@@ -2,35 +2,25 @@
 -- Please see the included NOTICE for copyright information and
 -- LICENSE-APACHE for a copy of the license.
 
--- Show chunk column stats after updating the extension. We need to
--- exclude the rows with chunk_id = 0 because we cannot keep those on
--- downgrade due to FK constraint. Showing them would mean a diff in
--- the output. We can still test that 0 chunk_ids are converted to
--- NULL values during upgrades, however.
-
-SELECT
-    cs.compress_relid::text AS chunk
-FROM _timescaledb_catalog.chunk ch
-JOIN _timescaledb_catalog.compression_settings cs
-    ON cs.relid = format('%I.%I', ch.schema_name, ch.table_name)::regclass
-WHERE ch.hypertable_id = (
-    SELECT id
-    FROM _timescaledb_catalog.hypertable
-    WHERE table_name = 'bloom'
-)
-LIMIT 1
-\gset
-
 -- This test checks that the bloom sparse indexes survive the upgrade, so only
 -- look at the bloom columns of the compressed chunk. Dumping the whole chunk
 -- would also pull in the orderby sparse metadata, whose layout depends on the
 -- version that compressed the chunk (minmax vs firstlast) and is not rewritten
 -- on upgrade, which would cause a spurious diff unrelated to bloom indexes.
+-- look at the bloom columns of one compressed chunk of the bloom hypertable.
 \a
 SELECT a.attname, format_type(a.atttypid, a.atttypmod) AS type
 FROM pg_attribute a
 JOIN pg_type t ON t.oid = a.atttypid
-WHERE a.attrelid = :'chunk'::regclass
+WHERE a.attrelid = (
+    SELECT cs.compress_relid
+    FROM _timescaledb_catalog.chunk ch
+    JOIN _timescaledb_catalog.compression_settings cs
+        ON cs.relid = format('%I.%I', ch.schema_name, ch.table_name)::regclass
+    WHERE ch.hypertable_id = (SELECT id FROM _timescaledb_catalog.hypertable WHERE table_name = 'bloom')
+    ORDER BY ch.id
+    LIMIT 1
+  )
   AND a.attnum > 0
   AND NOT a.attisdropped
   AND t.typname = 'bloom1'