Add chunks to schema-based publications

Chunks live in _timescaledb_internal, so FOR TABLES IN SCHEMA
publications miss them and replication silently skips chunk data.

Backfill chunk membership by reading the schema set straight from the
CREATE/ALTER PUBLICATION parse tree (no before/after snapshots):

- CREATE PUBLICATION is reconciled in the ddl_command_end event trigger
  (the publication Oid exists by then); add CREATE PUBLICATION to the
  timescaledb_ddl_command_end WHEN TAG list.
- ALTER PUBLICATION ADD/DROP/SET TABLES IN SCHEMA is reconciled in
  process_ddl_command_start after prev_ProcessUtility. SET rebuilds the
  full new schema set (PG drops every tracked chunk row on SET); only
  DROP removes them. Options-only ALTER is a no-op.

ALTER TABLE ... SET SCHEMA on a hypertable now reconciles its chunks:
remove from the old schema publications, add to the new ones. Runs
before ts_hypertable_set_schema flips the catalog schema.

Skip adding a redundant explicit pg_publication_rel row when a chunk
already lives in a schema this publication covers natively (chunks
placed via associated_schema_name); the explicit row would survive a
later DROP TABLES IN SCHEMA and orphan. OSM (foreign-table) chunks are
skipped throughout; publication_add_relation would fail on them.

Tests 6b-6i cover CREATE/ADD/SET/DROP/CURRENT_SCHEMA, schema moves,
the redundancy guard (counts of pg_publication_rel rows), options-only
ALTER, and GUC-off on the ALTER/schema-change paths.

Signed-off-by: Arunprasad Rajkumar <ar.arunprasad@gmail.com>

Author: arajkumar

src/chunk.c

@@ -1032,6 +1032,15 @@ chunk_add_to_publication(Oid puboid, const Chunk *chunk)
 	List *columns = NIL;
 	Node *whereClause = NULL;
 
+	/* publication_add_relation errors on duplicates; the chunk may already be
+	 * an explicit member (user-added or backfilled by an earlier reconcile). */
+	if (SearchSysCacheExists2(PUBLICATIONRELMAP,
+							  ObjectIdGetDatum(chunk->fd.relid),
+							  ObjectIdGetDatum(puboid)))
+	{
+		return;
+	}
+
 	get_hypertable_publication_filters(puboid, chunk, &columns, &whereClause);
 
 	chunk_rel = table_open(chunk->fd.relid, AccessShareLock);
@@ -1123,6 +1132,21 @@ ts_chunk_publication_reconcile_schema_chunks(Oid pubid, List *schema_oids, bool
 				ts_hypertable_cache_get_entry(hcache, lfirst_oid(rc), CACHE_FLAG_MISSING_OK);
 			if (ht == NULL)
 			{
+				/* The scanned schema may hold chunks placed there via
+				 * associated_schema_name rather than hypertable roots. On DROP such a
+				 * chunk loses this publication's native schema coverage; if its
+				 * hypertable's schema is still covered, keep the chunk published with
+				 * an explicit row (the mirror image of the add-path guard). */
+				if (!add)
+				{
+					Chunk *chunk = ts_chunk_get_by_relid(lfirst_oid(rc), false);
+					if (chunk != NULL && list_member_oid(GetSchemaPublications(get_rel_namespace(
+															 chunk->hypertable_relid)),
+														 pubid))
+					{
+						chunk_add_to_publication(pubid, chunk);
+					}
+				}
 				continue;
 			}
 			ListCell *cc;
@@ -1138,7 +1162,6 @@ ts_chunk_publication_reconcile_schema_chunks(Oid pubid, List *schema_oids, bool
 				 * Skip adding an explicit row for a chunk that lives in a schema this
 				 * publication already covers natively: the explicit row would be
 				 * redundant and would survive a DROP that removes the schema mapping.
-				 * (Mirrors chunk_add_to_publications' guard.)
 				 */
 				if (add)
 				{

src/chunk.h

@@ -234,7 +234,7 @@ extern TSDLLEXPORT Datum ts_chunk_status_text(PG_FUNCTION_ARGS);
 extern TSDLLEXPORT List *ts_chunk_get_chunk_ids_by_hypertable_id(int32 hypertable_id);
 extern TSDLLEXPORT List *ts_chunk_get_by_hypertable_id(int32 hypertable_id);
 extern TSDLLEXPORT void ts_chunk_publication_reconcile_schema_chunks(Oid pubid, List *schema_oids,
-												bool add);
+																	 bool add);
 extern TSDLLEXPORT void ts_chunk_publication_reconcile_ht_schema_change(int32 hypertable_id,
 																		Oid old_schema,
 																		Oid new_schema);

src/process_utility.c

@@ -6138,15 +6138,10 @@ publication_schema_oids(List *pubobjects)
 }
 
 /*
- * Reconcile chunk membership for the schemas named in a publication command.
- * Chunks live in _timescaledb_internal, so FOR TABLES IN SCHEMA never adds them
- * (PostgreSQL only expands inheritance children for FOR TABLE / FOR ALL
- * TABLES). For SET, PostgreSQL has already dropped every chunk row it
- * tracked, so we rebuild for the full new schema set rather than a difference;
- * only DROP removes chunks, because PostgreSQL leaves our backfilled chunk
- * rows untouched when a schema is dropped from the publication. We read the
- * schema set straight from the parse tree, so only the schemas the command
- * touched are reconciled -- no before/after snapshot is needed.
+ * Add or remove chunk rows for the schemas named in this publication command;
+ * see ts_chunk_publication_reconcile_schema_chunks for the rationale. For SET,
+ * PostgreSQL has already dropped every chunk row it tracked, so we rebuild for
+ * the full new schema set rather than a difference; only DROP removes chunks.
  */
 static void
 reconcile_publication_schemas(const char *pubname, List *schema_oids, bool add)

test/expected/chunk_publication.out

@@ -401,7 +401,7 @@ INSERT INTO test_hypertable VALUES ('2024-01-01 00:00:00+00', 1, 1.0, 'data1');
 -- Create FOR ALL TABLES publication
 CREATE PUBLICATION test_pub_all_tables FOR ALL TABLES;
 -- Verify initial state (1 chunk)
-SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub_all_tables' AND tablename LIKE '%test_hypertable%' OR tablename LIKE '_hyper_%' ORDER BY schemaname, tablename;
+SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub_all_tables' AND (tablename LIKE '%test_hypertable%' OR tablename LIKE '_hyper_%') ORDER BY schemaname, tablename;
       schemaname       |     tablename     |           attnames           | rowfilter 
 -----------------------+-------------------+------------------------------+-----------
  _timescaledb_internal | _hyper_6_41_chunk | {time,device_id,value,extra} | 
@@ -411,7 +411,7 @@ SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHE
 INSERT INTO test_hypertable VALUES ('2024-01-02 00:00:00+00', 2, 2.0, 'data2');
 INSERT INTO test_hypertable VALUES ('2024-01-03 00:00:00+00', 3, 3.0, 'data3');
 -- Verify state (3 chunks)
-SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub_all_tables' AND tablename LIKE '%test_hypertable%' OR tablename LIKE '_hyper_%' ORDER BY schemaname, tablename;
+SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub_all_tables' AND (tablename LIKE '%test_hypertable%' OR tablename LIKE '_hyper_%') ORDER BY schemaname, tablename;
       schemaname       |     tablename     |           attnames           | rowfilter 
 -----------------------+-------------------+------------------------------+-----------
  _timescaledb_internal | _hyper_6_41_chunk | {time,device_id,value,extra} | 
@@ -426,7 +426,7 @@ INSERT INTO test_hypertable VALUES ('2024-01-06 00:00:00+00', 6, 6.0, 'data6');
 INSERT INTO test_hypertable VALUES ('2024-01-07 00:00:00+00', 7, 7.0, 'data7');
 INSERT INTO test_hypertable VALUES ('2024-01-08 00:00:00+00', 8, 8.0, 'data8');
 -- Verify final state (8 chunks)
-SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub_all_tables' AND tablename LIKE '%test_hypertable%' OR tablename LIKE '_hyper_%' ORDER BY schemaname, tablename;
+SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub_all_tables' AND (tablename LIKE '%test_hypertable%' OR tablename LIKE '_hyper_%') ORDER BY schemaname, tablename;
       schemaname       |     tablename     |           attnames           | rowfilter 
 -----------------------+-------------------+------------------------------+-----------
  _timescaledb_internal | _hyper_6_41_chunk | {time,device_id,value,extra} | 
@@ -845,6 +845,101 @@ DROP PUBLICATION test_pub_dst CASCADE;
 DROP TABLE ht_dst_schema.test_hypertable CASCADE;
 DROP SCHEMA ht_src_schema;
 DROP SCHEMA ht_dst_schema;
+-- Test 6j: DROP TABLES IN SCHEMA on the chunk-home schema while the
+-- hypertable's schema stays published. The chunks were covered natively via
+-- their own schema (no explicit rows); dropping that schema from the
+-- publication would silently unpublish them, so the reconcile must add
+-- explicit rows for chunks whose hypertable schema is still covered.
+CREATE SCHEMA ht_root_schema;
+CREATE SCHEMA ht_chunk_home;
+CREATE TABLE ht_root_schema.test_hypertable (time timestamptz NOT NULL, device_id int, value float);
+SELECT create_hypertable('ht_root_schema.test_hypertable', 'time',
+    chunk_time_interval => interval '1 day',
+    associated_schema_name => 'ht_chunk_home');
+           create_hypertable           
+---------------------------------------
+ (15,ht_root_schema,test_hypertable,t)
+
+INSERT INTO ht_root_schema.test_hypertable VALUES ('2024-01-01 00:00:00+00', 1, 1.0);
+INSERT INTO ht_root_schema.test_hypertable VALUES ('2024-01-02 00:00:00+00', 2, 2.0);
+CREATE PUBLICATION test_pub_both FOR TABLES IN SCHEMA ht_root_schema, ht_chunk_home;
+-- Chunks are covered natively via ht_chunk_home: no explicit rows.
+SELECT count(*) AS explicit_rows FROM pg_publication_rel r
+  JOIN pg_publication p ON p.oid = r.prpubid WHERE p.pubname = 'test_pub_both';
+ explicit_rows 
+---------------
+             0
+
+-- Drop the chunk-home schema: the hypertable stays published via
+-- ht_root_schema, so the chunks get explicit rows to stay published too.
+ALTER PUBLICATION test_pub_both DROP TABLES IN SCHEMA ht_chunk_home;
+SELECT count(*) AS explicit_rows FROM pg_publication_rel r
+  JOIN pg_publication p ON p.oid = r.prpubid WHERE p.pubname = 'test_pub_both';
+ explicit_rows 
+---------------
+             2
+
+SELECT schemaname, tablename FROM pg_publication_tables
+  WHERE pubname = 'test_pub_both' ORDER BY schemaname, tablename;
+   schemaname   |     tablename      
+----------------+--------------------
+ ht_chunk_home  | _hyper_15_68_chunk
+ ht_chunk_home  | _hyper_15_69_chunk
+ ht_root_schema | test_hypertable
+
+-- A chunk created afterwards converges to the same state (explicit row).
+INSERT INTO ht_root_schema.test_hypertable VALUES ('2024-01-03 00:00:00+00', 3, 3.0);
+SELECT count(*) AS explicit_rows FROM pg_publication_rel r
+  JOIN pg_publication p ON p.oid = r.prpubid WHERE p.pubname = 'test_pub_both';
+ explicit_rows 
+---------------
+             3
+
+DROP PUBLICATION test_pub_both CASCADE;
+DROP TABLE ht_root_schema.test_hypertable CASCADE;
+DROP SCHEMA ht_root_schema;
+DROP SCHEMA ht_chunk_home;
+-- Test 6k: Documented limitation - with the GUC turned off after chunks were
+-- backfilled, DROP TABLES IN SCHEMA skips the reconcile entirely and leaves
+-- the explicit chunk rows orphaned (they keep the chunks published). This
+-- pins the behavior; see ts_chunk_publication_reconcile_schema_chunks.
+CREATE SCHEMA ht_guc_orphan;
+CREATE TABLE ht_guc_orphan.test_hypertable (time timestamptz NOT NULL, device_id int, value float);
+SELECT create_hypertable('ht_guc_orphan.test_hypertable', 'time', chunk_time_interval => interval '1 day');
+          create_hypertable           
+--------------------------------------
+ (16,ht_guc_orphan,test_hypertable,t)
+
+INSERT INTO ht_guc_orphan.test_hypertable VALUES ('2024-01-01 00:00:00+00', 1, 1.0);
+INSERT INTO ht_guc_orphan.test_hypertable VALUES ('2024-01-02 00:00:00+00', 2, 2.0);
+CREATE PUBLICATION test_pub_orphan FOR TABLES IN SCHEMA ht_guc_orphan;
+SELECT count(*) AS explicit_rows FROM pg_publication_rel r
+  JOIN pg_publication p ON p.oid = r.prpubid WHERE p.pubname = 'test_pub_orphan';
+ explicit_rows 
+---------------
+             2
+
+-- With the GUC off, the DROP reconcile is skipped: the explicit chunk rows
+-- are orphaned and the chunks stay published.
+SET timescaledb.enable_chunk_auto_publication = false;
+ALTER PUBLICATION test_pub_orphan DROP TABLES IN SCHEMA ht_guc_orphan;
+SELECT count(*) AS explicit_rows FROM pg_publication_rel r
+  JOIN pg_publication p ON p.oid = r.prpubid WHERE p.pubname = 'test_pub_orphan';
+ explicit_rows 
+---------------
+             2
+
+SELECT schemaname, tablename FROM pg_publication_tables
+  WHERE pubname = 'test_pub_orphan' ORDER BY schemaname, tablename;
+      schemaname       |     tablename      
+-----------------------+--------------------
+ _timescaledb_internal | _hyper_16_71_chunk
+ _timescaledb_internal | _hyper_16_72_chunk
+
+SET timescaledb.enable_chunk_auto_publication = true;
+DROP PUBLICATION test_pub_orphan CASCADE;
+DROP TABLE ht_guc_orphan.test_hypertable CASCADE;
+DROP SCHEMA ht_guc_orphan;
 -- Test 6i: GUC off suppresses reconciliation on the ALTER PUBLICATION and
 -- schema-change paths (not only on the per-chunk create hook tested in Test 9).
 SET timescaledb.enable_chunk_auto_publication = false;
@@ -854,7 +949,7 @@ CREATE TABLE ht_guc_schema.test_hypertable (time timestamptz NOT NULL, device_id
 SELECT create_hypertable('ht_guc_schema.test_hypertable', 'time', chunk_time_interval => interval '1 day');
           create_hypertable           
 --------------------------------------
- (15,ht_guc_schema,test_hypertable,t)
+ (17,ht_guc_schema,test_hypertable,t)
 
 INSERT INTO ht_guc_schema.test_hypertable VALUES ('2024-01-01 00:00:00+00', 1, 1.0);
 INSERT INTO ht_guc_schema.test_hypertable VALUES ('2024-01-02 00:00:00+00', 2, 2.0);
@@ -870,7 +965,7 @@ CREATE TABLE ht_guc_schema2.test_hypertable2 (time timestamptz NOT NULL, device_
 SELECT create_hypertable('ht_guc_schema2.test_hypertable2', 'time', chunk_time_interval => interval '1 day');
            create_hypertable            
 ----------------------------------------
- (16,ht_guc_schema2,test_hypertable2,t)
+ (18,ht_guc_schema2,test_hypertable2,t)
 
 INSERT INTO ht_guc_schema2.test_hypertable2 VALUES ('2024-02-01 00:00:00+00', 1, 1.0);
 INSERT INTO ht_guc_schema2.test_hypertable2 VALUES ('2024-02-02 00:00:00+00', 2, 2.0);
@@ -897,10 +992,10 @@ ALTER PUBLICATION test_pub_guc SET TABLES IN SCHEMA ht_guc_schema, ht_guc_schema
 SELECT schemaname, tablename FROM pg_publication_tables WHERE pubname = 'test_pub_guc' ORDER BY schemaname, tablename;
       schemaname       |     tablename      
 -----------------------+--------------------
- _timescaledb_internal | _hyper_15_68_chunk
- _timescaledb_internal | _hyper_15_69_chunk
- _timescaledb_internal | _hyper_16_70_chunk
- _timescaledb_internal | _hyper_16_71_chunk
+ _timescaledb_internal | _hyper_17_73_chunk
+ _timescaledb_internal | _hyper_17_74_chunk
+ _timescaledb_internal | _hyper_18_75_chunk
+ _timescaledb_internal | _hyper_18_76_chunk
  ht_guc_schema2        | test_hypertable
  ht_guc_schema2        | test_hypertable2
 
@@ -914,7 +1009,7 @@ CREATE TABLE test_hypertable (time timestamptz NOT NULL, device_id int, value fl
 SELECT create_hypertable('test_hypertable', 'time', chunk_time_interval => interval '1 day');
        create_hypertable       
 -------------------------------
- (17,public,test_hypertable,t)
+ (19,public,test_hypertable,t)
 
 -- Insert to create 8 chunks without any publication
 INSERT INTO test_hypertable VALUES ('2024-01-01 00:00:00+00', 1, 1.0, 'data1');
@@ -939,7 +1034,7 @@ CREATE TABLE test_hypertable (time timestamptz NOT NULL, device_id int, value fl
 SELECT create_hypertable('test_hypertable', 'time', chunk_time_interval => interval '1 day');
        create_hypertable       
 -------------------------------
- (18,public,test_hypertable,t)
+ (20,public,test_hypertable,t)
 
 -- Insert to create first chunk
 INSERT INTO test_hypertable VALUES ('2024-01-01 00:00:00+00', 1, 1.0, 'data1');
@@ -949,7 +1044,7 @@ CREATE PUBLICATION test_pub FOR TABLE test_hypertable;
 SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub' ORDER BY schemaname, tablename;
       schemaname       |     tablename      |           attnames           | rowfilter 
 -----------------------+--------------------+------------------------------+-----------
- _timescaledb_internal | _hyper_18_80_chunk | {time,device_id,value,extra} | 
+ _timescaledb_internal | _hyper_20_85_chunk | {time,device_id,value,extra} | 
  public                | test_hypertable    | {time,device_id,value,extra} | 
 
 -- Drop the publication
@@ -972,7 +1067,7 @@ CREATE TABLE test_hypertable (time timestamptz NOT NULL, device_id int, value fl
 SELECT create_hypertable('test_hypertable', 'time', chunk_time_interval => interval '1 day');
        create_hypertable       
 -------------------------------
- (19,public,test_hypertable,t)
+ (21,public,test_hypertable,t)
 
 -- Insert to create first chunk
 INSERT INTO test_hypertable VALUES ('2024-01-01 00:00:00+00', 1, 1.0, 'data1');
@@ -982,7 +1077,7 @@ CREATE PUBLICATION test_pub_guc FOR TABLE test_hypertable;
 SELECT schemaname, tablename FROM pg_publication_tables WHERE pubname = 'test_pub_guc' ORDER BY schemaname, tablename;
       schemaname       |     tablename      
 -----------------------+--------------------
- _timescaledb_internal | _hyper_19_83_chunk
+ _timescaledb_internal | _hyper_21_88_chunk
  public                | test_hypertable
 
 -- Test Part 1: GUC enabled - chunks should be added to publication automatically
@@ -992,8 +1087,8 @@ INSERT INTO test_hypertable VALUES ('2024-01-02 00:00:00+00', 2, 2.0, 'data2');
 SELECT schemaname, tablename FROM pg_publication_tables WHERE pubname = 'test_pub_guc' ORDER BY schemaname, tablename;
       schemaname       |     tablename      
 -----------------------+--------------------
- _timescaledb_internal | _hyper_19_83_chunk
- _timescaledb_internal | _hyper_19_84_chunk
+ _timescaledb_internal | _hyper_21_88_chunk
+ _timescaledb_internal | _hyper_21_89_chunk
  public                | test_hypertable
 
 -- Test Part 2: Disable the GUC and create another chunk
@@ -1004,17 +1099,17 @@ INSERT INTO test_hypertable VALUES ('2024-01-03 00:00:00+00', 3, 3.0, 'data3');
 SELECT schemaname, tablename FROM pg_publication_tables WHERE pubname = 'test_pub_guc' ORDER BY schemaname, tablename;
       schemaname       |     tablename      
 -----------------------+--------------------
- _timescaledb_internal | _hyper_19_83_chunk
- _timescaledb_internal | _hyper_19_84_chunk
+ _timescaledb_internal | _hyper_21_88_chunk
+ _timescaledb_internal | _hyper_21_89_chunk
  public                | test_hypertable
 
 -- Verify that chunk 3 exists but is not in the publication
-SELECT chunk_schema, chunk_name FROM timescaledb_information.chunks WHERE hypertable_name = 'test_hypertable';
+SELECT chunk_schema, chunk_name FROM timescaledb_information.chunks WHERE hypertable_name = 'test_hypertable' ORDER BY chunk_name;
      chunk_schema      |     chunk_name     
 -----------------------+--------------------
- _timescaledb_internal | _hyper_19_83_chunk
- _timescaledb_internal | _hyper_19_84_chunk
- _timescaledb_internal | _hyper_19_85_chunk
+ _timescaledb_internal | _hyper_21_88_chunk
+ _timescaledb_internal | _hyper_21_89_chunk
+ _timescaledb_internal | _hyper_21_90_chunk
 
 -- Test Part 3: Re-enable the GUC and create another chunk
 SET timescaledb.enable_chunk_auto_publication = true;
@@ -1024,18 +1119,18 @@ INSERT INTO test_hypertable VALUES ('2024-01-04 00:00:00+00', 4, 4.0, 'data4');
 SELECT schemaname, tablename FROM pg_publication_tables WHERE pubname = 'test_pub_guc' ORDER BY schemaname, tablename;
       schemaname       |     tablename      
 -----------------------+--------------------
- _timescaledb_internal | _hyper_19_83_chunk
- _timescaledb_internal | _hyper_19_84_chunk
- _timescaledb_internal | _hyper_19_86_chunk
+ _timescaledb_internal | _hyper_21_88_chunk
+ _timescaledb_internal | _hyper_21_89_chunk
+ _timescaledb_internal | _hyper_21_91_chunk
  public                | test_hypertable
 
-SELECT chunk_schema, chunk_name FROM timescaledb_information.chunks WHERE hypertable_name = 'test_hypertable';
+SELECT chunk_schema, chunk_name FROM timescaledb_information.chunks WHERE hypertable_name = 'test_hypertable' ORDER BY chunk_name;
      chunk_schema      |     chunk_name     
 -----------------------+--------------------
- _timescaledb_internal | _hyper_19_83_chunk
- _timescaledb_internal | _hyper_19_84_chunk
- _timescaledb_internal | _hyper_19_85_chunk
- _timescaledb_internal | _hyper_19_86_chunk
+ _timescaledb_internal | _hyper_21_88_chunk
+ _timescaledb_internal | _hyper_21_89_chunk
+ _timescaledb_internal | _hyper_21_90_chunk
+ _timescaledb_internal | _hyper_21_91_chunk
 
 -- Cleanup
 DROP PUBLICATION test_pub_guc CASCADE;