Fix compression policy error message.

gayyappan · fabriziomello · fabriziomello · commit 6053417ee879 · 2025-04-24T18:08:22.000-03:00
Co-authored-by: Fabrízio de Royes Mello &lt;fabriziomello@gmail.com&gt;
diff --git a/.unreleased/pr_8008 b/.unreleased/pr_8008
@@ -0,0 +1 @@
+Fixes: #8008 Fix compression policy error message that shows number of successes
diff --git a/sql/policy_internal.sql b/sql/policy_internal.sql
@@ -48,7 +48,7 @@ AS $$
 DECLARE
   htoid       REGCLASS;
   chunk_rec   RECORD;
-  numchunks   INTEGER := 1;
+  numchunks_compressed   INTEGER := 0;
   _message     text;
   _detail      text;
   _sqlstate    text;
@@ -106,7 +106,7 @@ BEGIN
     BEGIN
       IF chunk_rec.status = bit_compressed OR recompress_enabled IS TRUE THEN
         PERFORM @extschema@.compress_chunk(chunk_rec.oid, hypercore_use_access_method => useam);
-        numchunks := numchunks + 1;
+        numchunks_compressed := numchunks_compressed + 1;
       END IF;
     EXCEPTION WHEN OTHERS THEN
       GET STACKED DIAGNOSTICS
@@ -127,14 +127,14 @@ BEGIN
     IF verbose_log THEN
        RAISE LOG 'job % completed processing chunk %.%', job_id, chunk_rec.schema_name, chunk_rec.table_name;
     END IF;
-    IF maxchunks > 0 AND numchunks >= maxchunks THEN
+    IF maxchunks > 0 AND numchunks_compressed >= maxchunks THEN
          EXIT;
     END IF;
   END LOOP;
 
   IF chunks_failure > 0 THEN
     RAISE EXCEPTION 'compression policy failure'
-      USING DETAIL = format('Failed to compress %L chunks. Successfully compressed %L chunks.', chunks_failure, numchunks - chunks_failure);
+      USING DETAIL = format('Failed to compress %L chunks. Successfully compressed %L chunks.', chunks_failure, numchunks_compressed);
   END IF;
 END;
 $$ LANGUAGE PLPGSQL;
diff --git a/sql/updates/reverse-dev.sql b/sql/updates/reverse-dev.sql
@@ -31,3 +31,108 @@ CREATE FUNCTION @extschema@.add_continuous_aggregate_policy(
 RETURNS INTEGER
 AS '@MODULE_PATHNAME@', 'ts_update_placeholder'
 LANGUAGE C VOLATILE;
+
+CREATE OR REPLACE PROCEDURE
+_timescaledb_functions.policy_compression_execute(
+  job_id              INTEGER,
+  htid                INTEGER,
+  lag                 ANYELEMENT,
+  maxchunks           INTEGER,
+  verbose_log         BOOLEAN,
+  recompress_enabled  BOOLEAN,
+  use_creation_time   BOOLEAN,
+  useam               BOOLEAN = NULL)
+AS $$
+DECLARE
+  htoid       REGCLASS;
+  chunk_rec   RECORD;
+  numchunks   INTEGER := 1;
+  _message     text;
+  _detail      text;
+  _sqlstate    text;
+  -- fully compressed chunk status
+  status_fully_compressed int := 1;
+  -- chunk status bits:
+  bit_compressed int := 1;
+  bit_compressed_unordered int := 2;
+  bit_frozen int := 4;
+  bit_compressed_partial int := 8;
+  creation_lag INTERVAL := NULL;
+  chunks_failure INTEGER := 0;
+BEGIN
+
+  -- procedures with SET clause cannot execute transaction
+  -- control so we adjust search_path in procedure body
+  SET LOCAL search_path TO pg_catalog, pg_temp;
+
+  SELECT format('%I.%I', schema_name, table_name) INTO htoid
+  FROM _timescaledb_catalog.hypertable
+  WHERE id = htid;
+
+  -- for the integer cases, we have to compute the lag w.r.t
+  -- the integer_now function and then pass on to show_chunks
+  IF pg_typeof(lag) IN ('BIGINT'::regtype, 'INTEGER'::regtype, 'SMALLINT'::regtype) THEN
+    -- cannot have use_creation_time set with this
+    IF use_creation_time IS TRUE THEN
+        RAISE EXCEPTION 'job % cannot use creation time with integer_now function', job_id;
+    END IF;
+    lag := _timescaledb_functions.subtract_integer_from_now(htoid, lag::BIGINT);
+  END IF;
+
+  -- if use_creation_time has been specified then the lag needs to be used with the
+  -- "compress_created_before" argument. Otherwise the usual "older_than" argument
+  -- is good enough
+  IF use_creation_time IS TRUE THEN
+    creation_lag := lag;
+    lag := NULL;
+  END IF;
+
+  FOR chunk_rec IN
+    SELECT
+      show.oid, ch.schema_name, ch.table_name, ch.status
+    FROM
+      @extschema@.show_chunks(htoid, older_than => lag, created_before => creation_lag) AS show(oid)
+      INNER JOIN pg_class pgc ON pgc.oid = show.oid
+      INNER JOIN pg_namespace pgns ON pgc.relnamespace = pgns.oid
+      INNER JOIN _timescaledb_catalog.chunk ch ON ch.table_name = pgc.relname AND ch.schema_name = pgns.nspname AND ch.hypertable_id = htid
+    WHERE NOT ch.dropped
+    AND NOT ch.osm_chunk
+    -- Checking for chunks which are not fully compressed and not frozen
+    AND ch.status != status_fully_compressed
+    AND ch.status & bit_frozen = 0
+  LOOP
+    BEGIN
+      IF chunk_rec.status = bit_compressed OR recompress_enabled IS TRUE THEN
+        PERFORM @extschema@.compress_chunk(chunk_rec.oid, hypercore_use_access_method => useam);
+        numchunks := numchunks + 1;
+      END IF;
+    EXCEPTION WHEN OTHERS THEN
+      GET STACKED DIAGNOSTICS
+          _message = MESSAGE_TEXT,
+          _detail = PG_EXCEPTION_DETAIL,
+          _sqlstate = RETURNED_SQLSTATE;
+      RAISE WARNING 'compressing chunk "%" failed when compression policy is executed', chunk_rec.oid::regclass::text
+          USING DETAIL = format('Message: (%s), Detail: (%s).', _message, _detail),
+                ERRCODE = _sqlstate;
+      chunks_failure := chunks_failure + 1;
+    END;
+    COMMIT;
+    -- SET LOCAL is only active until end of transaction.
+    -- While we could use SET at the start of the function we do not
+    -- want to bleed out search_path to caller, so we do SET LOCAL
+    -- again after COMMIT
+    SET LOCAL search_path TO pg_catalog, pg_temp;
+    IF verbose_log THEN
+       RAISE LOG 'job % completed processing chunk %.%', job_id, chunk_rec.schema_name, chunk_rec.table_name;
+    END IF;
+    IF maxchunks > 0 AND numchunks >= maxchunks THEN
+         EXIT;
+    END IF;
+  END LOOP;
+
+  IF chunks_failure > 0 THEN
+    RAISE EXCEPTION 'compression policy failure'
+      USING DETAIL = format('Failed to compress %L chunks. Successfully compressed %L chunks.', chunks_failure, numchunks - chunks_failure);
+  END IF;
+END;
+$$ LANGUAGE PLPGSQL;
diff --git a/tsl/test/expected/compression_bgw.out b/tsl/test/expected/compression_bgw.out
@@ -771,6 +771,99 @@ LIMIT 1;
  69 |      1
 (1 row)
 
+--TEST 8
+--compression policy errors
+CREATE TABLE test_compression_policy_errors(time TIMESTAMPTZ, val SMALLINT);
+SELECT create_hypertable('test_compression_policy_errors', 'time', chunk_time_interval => '1 day'::interval);
+NOTICE:  adding not-null constraint to column "time"
+              create_hypertable               
+----------------------------------------------
+ (20,public,test_compression_policy_errors,t)
+(1 row)
+
+ALTER TABLE test_compression_policy_errors SET (timescaledb.compress, timescaledb.compress_segmentby = 'val', timescaledb.compress_orderby = 'time');
+INSERT INTO test_compression_policy_errors SELECT time, (random()*10)::smallint
+FROM generate_series('2018-12-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '10 min') AS time;
+SELECT
+  add_compression_policy(
+    'test_compression_policy_errors',
+    compress_after=> '1 day'::interval,
+    initial_start => now() - interval '1 day'
+  ) as compressjob_id \gset
+SELECT config AS compressjob_config FROM _timescaledb_config.bgw_job WHERE id = :compressjob_id \gset
+SELECT FROM alter_job(:compressjob_id, config => jsonb_set(:'compressjob_config'::jsonb, '{recompress}', 'true'));
+--
+(1 row)
+
+-- 31 uncompressed chunks (0 - uncompressed, 1 - compressed)
+SELECT c.status, count(*)
+FROM _timescaledb_catalog.chunk c
+INNER JOIN _timescaledb_catalog.hypertable h on (h.id = c.hypertable_id)
+WHERE h.table_name = 'test_compression_policy_errors'
+GROUP BY c.status
+ORDER BY 2 DESC;
+ status | count 
+--------+-------
+      0 |    31
+(1 row)
+
+\c :TEST_DBNAME :ROLE_SUPERUSER
+-- Let's mess with the chunk status to for an error when executing the job
+WITH chunks AS (
+  SELECT c.id, c.status
+  FROM _timescaledb_catalog.chunk c
+  INNER JOIN _timescaledb_catalog.hypertable h on (h.id = c.hypertable_id)
+  WHERE h.table_name = 'test_compression_policy_errors'
+  ORDER BY c.id LIMIT 20
+)
+UPDATE _timescaledb_catalog.chunk
+SET status = 3
+FROM chunks
+WHERE chunk.id = chunks.id
+  AND chunk.status = 0;
+\c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER
+-- After the mess 20 = status 3 and 11 = status 0
+SELECT c.status, count(*)
+FROM _timescaledb_catalog.chunk c
+INNER JOIN _timescaledb_catalog.hypertable h on (h.id = c.hypertable_id)
+WHERE h.table_name = 'test_compression_policy_errors'
+GROUP BY c.status
+ORDER BY 2 DESC;
+ status | count 
+--------+-------
+      3 |    20
+      0 |    11
+(2 rows)
+
+\set ON_ERROR_STOP 0
+SET client_min_messages TO ERROR;
+\set VERBOSITY default
+-- This should fail with
+-- 20 chunks failed to compress and 11 chunks compressed successfully
+CALL run_job(:compressjob_id);
+ERROR:  compression policy failure
+DETAIL:  Failed to compress '20' chunks. Successfully compressed '11' chunks.
+CONTEXT:  PL/pgSQL function _timescaledb_functions.policy_compression_execute(integer,integer,anyelement,integer,boolean,boolean,boolean,boolean) line 90 at RAISE
+SQL statement "CALL _timescaledb_functions.policy_compression_execute(
+        job_id, htid, lag_value::INTERVAL,
+        maxchunks, verbose_log, recompress_enabled, use_creation_time, hypercore_use_access_method
+      )"
+PL/pgSQL function _timescaledb_functions.policy_compression(integer,jsonb) line 63 at CALL
+\set VERBOSITY terse
+\set ON_ERROR_STOP 1
+-- 31 uncompressed chunks (0 - uncompressed, 1 - compressed)
+SELECT c.status, count(*)
+FROM _timescaledb_catalog.chunk c
+INNER JOIN _timescaledb_catalog.hypertable h on (h.id = c.hypertable_id)
+WHERE h.table_name = 'test_compression_policy_errors'
+GROUP BY c.status
+ORDER BY 2 DESC;
+ status | count 
+--------+-------
+      3 |    20
+      1 |    11
+(2 rows)
+
 -- Teardown test
 \c :TEST_DBNAME :ROLE_SUPERUSER
 REVOKE CREATE ON SCHEMA public FROM NOLOGIN_ROLE;
diff --git a/tsl/test/sql/compression_bgw.sql b/tsl/test/sql/compression_bgw.sql
@@ -279,6 +279,76 @@ WHERE h.table_name = 'test_table_frozen'
 ORDER BY c.id
 LIMIT 1;
 
+--TEST 8
+--compression policy errors
+CREATE TABLE test_compression_policy_errors(time TIMESTAMPTZ, val SMALLINT);
+SELECT create_hypertable('test_compression_policy_errors', 'time', chunk_time_interval => '1 day'::interval);
+ALTER TABLE test_compression_policy_errors SET (timescaledb.compress, timescaledb.compress_segmentby = 'val', timescaledb.compress_orderby = 'time');
+
+INSERT INTO test_compression_policy_errors SELECT time, (random()*10)::smallint
+FROM generate_series('2018-12-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '10 min') AS time;
+
+SELECT
+  add_compression_policy(
+    'test_compression_policy_errors',
+    compress_after=> '1 day'::interval,
+    initial_start => now() - interval '1 day'
+  ) as compressjob_id \gset
+
+SELECT config AS compressjob_config FROM _timescaledb_config.bgw_job WHERE id = :compressjob_id \gset
+SELECT FROM alter_job(:compressjob_id, config => jsonb_set(:'compressjob_config'::jsonb, '{recompress}', 'true'));
+
+-- 31 uncompressed chunks (0 - uncompressed, 1 - compressed)
+SELECT c.status, count(*)
+FROM _timescaledb_catalog.chunk c
+INNER JOIN _timescaledb_catalog.hypertable h on (h.id = c.hypertable_id)
+WHERE h.table_name = 'test_compression_policy_errors'
+GROUP BY c.status
+ORDER BY 2 DESC;
+
+\c :TEST_DBNAME :ROLE_SUPERUSER
+
+-- Let's mess with the chunk status to for an error when executing the job
+WITH chunks AS (
+  SELECT c.id, c.status
+  FROM _timescaledb_catalog.chunk c
+  INNER JOIN _timescaledb_catalog.hypertable h on (h.id = c.hypertable_id)
+  WHERE h.table_name = 'test_compression_policy_errors'
+  ORDER BY c.id LIMIT 20
+)
+UPDATE _timescaledb_catalog.chunk
+SET status = 3
+FROM chunks
+WHERE chunk.id = chunks.id
+  AND chunk.status = 0;
+
+\c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER
+
+-- After the mess 20 = status 3 and 11 = status 0
+SELECT c.status, count(*)
+FROM _timescaledb_catalog.chunk c
+INNER JOIN _timescaledb_catalog.hypertable h on (h.id = c.hypertable_id)
+WHERE h.table_name = 'test_compression_policy_errors'
+GROUP BY c.status
+ORDER BY 2 DESC;
+
+\set ON_ERROR_STOP 0
+SET client_min_messages TO ERROR;
+\set VERBOSITY default
+-- This should fail with
+-- 20 chunks failed to compress and 11 chunks compressed successfully
+CALL run_job(:compressjob_id);
+\set VERBOSITY terse
+\set ON_ERROR_STOP 1
+
+-- 31 uncompressed chunks (0 - uncompressed, 1 - compressed)
+SELECT c.status, count(*)
+FROM _timescaledb_catalog.chunk c
+INNER JOIN _timescaledb_catalog.hypertable h on (h.id = c.hypertable_id)
+WHERE h.table_name = 'test_compression_policy_errors'
+GROUP BY c.status
+ORDER BY 2 DESC;
+
 -- Teardown test
 \c :TEST_DBNAME :ROLE_SUPERUSER
 REVOKE CREATE ON SCHEMA public FROM NOLOGIN_ROLE;

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+Fixes: #8008 Fix compression policy error message that shows number of successes`