add timescaledb.compress_column_codec option

Instead of using types and default opclasses, the EXTERNAL compresison
algorithm now uses regular opclasses and only applies when explicitly
configured.

Users alter or otherwise set the timescaledb.column_codec to a
`column_name:some.compression_opclass`. The opclass, as in the
previous commit, is a virtual namespace with 2 support functions.
(1 and 2, compress and decompress respectively).

Author: kvc0

.unreleased/external-compression

@@ -1 +1 @@
-Implements: #1524 Add external compression, letting a type supply its own compress/decompress functions
+Implements: #1524 Add external compression: configure columns with timescaledb.compress_column_codec

cmake/ScriptFiles.cmake

@@ -17,11 +17,14 @@ set(PRE_INSTALL_SOURCE_FILES
     pre_install/types.post.sql # Must be before tables.sql
     pre_install/tables.sql
     pre_install/cache.sql
+    pre_install/compression_codec.functions.sql
+    pre_install/compression_codec.sql
     pre_install/insert_data.sql)
 
 # Source files that define functions and need to be rerun in update
 set(PRE_INSTALL_FUNCTION_FILES
     pre_install/types.functions.sql
+    pre_install/compression_codec.functions.sql
     )
 
 # The rest of the source files defining mostly functions

sql/pre_install/compression_codec.functions.sql

@@ -0,0 +1,6 @@
+-- This file and its contents are licensed under the Apache License 2.0.
+-- Please see the included NOTICE for copyright information and
+-- LICENSE-APACHE for a copy of the license.
+
+CREATE OR REPLACE FUNCTION _timescaledb_functions.compression_codec_handler(internal) RETURNS index_am_handler
+AS '@MODULE_PATHNAME@', 'ts_compression_codec_handler' LANGUAGE C;

sql/pre_install/compression_codec.sql

@@ -0,0 +1,12 @@
+-- This file and its contents are licensed under the Apache License 2.0.
+-- Please see the included NOTICE for copyright information and
+-- LICENSE-APACHE for a copy of the license.
+
+-- Virtual access method: Register an EXTERNAL compression codec for a
+-- type by creating an operator class for this access method. Columns
+-- use a registered codec through the per-column setting
+-- timescaledb.compress_column_codec = '<column>:<operator class>, ...'
+-- * Support function 1:    bytea compress(<type>[])
+-- * Support function 2: <type>[] decompress(bytea)
+CREATE ACCESS METHOD ts_compression_codec TYPE INDEX HANDLER _timescaledb_functions.compression_codec_handler;
+COMMENT ON ACCESS METHOD ts_compression_codec IS 'Registry for compression codec operator classes';

sql/pre_install/tables.sql

@@ -460,10 +460,16 @@ CREATE TABLE _timescaledb_catalog.compression_settings (
   orderby_desc bool[],
   orderby_nullsfirst bool[],
   index jsonb,
+  -- parallel arrays: codec_column[i] is compressed with the EXTERNAL codec
+  -- registered by the ts_compression_codec operator class codec_opclass[i]
+  codec_column text[],
+  codec_opclass text[],
   CONSTRAINT compression_settings_pkey PRIMARY KEY (relid),
   CONSTRAINT compression_settings_check_segmentby CHECK (array_ndims(segmentby) = 1),
   CONSTRAINT compression_settings_check_orderby_null CHECK ((orderby IS NULL AND orderby_desc IS NULL AND orderby_nullsfirst IS NULL) OR (orderby IS NOT NULL AND orderby_desc IS NOT NULL AND orderby_nullsfirst IS NOT NULL)),
-  CONSTRAINT compression_settings_check_orderby_cardinality CHECK (array_ndims(orderby) = 1 AND array_ndims(orderby_desc) = 1 AND array_ndims(orderby_nullsfirst) = 1 AND cardinality(orderby) = cardinality(orderby_desc) AND cardinality(orderby) = cardinality(orderby_nullsfirst))
+  CONSTRAINT compression_settings_check_orderby_cardinality CHECK (array_ndims(orderby) = 1 AND array_ndims(orderby_desc) = 1 AND array_ndims(orderby_nullsfirst) = 1 AND cardinality(orderby) = cardinality(orderby_desc) AND cardinality(orderby) = cardinality(orderby_nullsfirst)),
+  CONSTRAINT compression_settings_check_codec_null CHECK ((codec_column IS NULL AND codec_opclass IS NULL) OR (codec_column IS NOT NULL AND codec_opclass IS NOT NULL)),
+  CONSTRAINT compression_settings_check_codec_cardinality CHECK (array_ndims(codec_column) = 1 AND array_ndims(codec_opclass) = 1 AND cardinality(codec_column) = cardinality(codec_opclass))
 );
 
 SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.compression_settings', '');

sql/updates/latest-dev.sql

@@ -150,3 +150,21 @@ GRANT SELECT ON _timescaledb_catalog.chunk TO PUBLIC;
 
 INSERT INTO _timescaledb_catalog.compression_algorithm( id, version, name, description) values
 ( 8, 1, 'COMPRESSION_ALGORITHM_EXTERNAL', 'external');
+
+-- Registry access method for EXTERNAL compression codec operator classes.
+-- Placeholder is replaced when the function definitions run later in the update.
+CREATE FUNCTION _timescaledb_functions.compression_codec_handler(internal) RETURNS index_am_handler
+AS '@MODULE_PATHNAME@', 'ts_update_placeholder' LANGUAGE C;
+CREATE ACCESS METHOD ts_compression_codec TYPE INDEX HANDLER _timescaledb_functions.compression_codec_handler;
+COMMENT ON ACCESS METHOD ts_compression_codec IS 'Registry for compression codec operator classes';
+
+-- Per-column setting for EXTERNAL compression algorithm. Set via the
+-- timescaledb.compress_column_codec option. codec_column[i] is compressed
+-- with the codec operator class named by codec_opclass[i].
+ALTER TABLE _timescaledb_catalog.compression_settings
+  ADD COLUMN codec_column text[],
+  ADD COLUMN codec_opclass text[],
+  ADD CONSTRAINT compression_settings_check_codec_null
+    CHECK ((codec_column IS NULL AND codec_opclass IS NULL) OR (codec_column IS NOT NULL AND codec_opclass IS NOT NULL)),
+  ADD CONSTRAINT compression_settings_check_codec_cardinality
+    CHECK (array_ndims(codec_column) = 1 AND array_ndims(codec_opclass) = 1 AND cardinality(codec_column) = cardinality(codec_opclass));

sql/updates/reverse-dev.sql

@@ -252,3 +252,81 @@ DROP FUNCTION IF EXISTS @extschema@.alter_job(job_id INTEGER, schedule_interval
 
 
 DELETE FROM _timescaledb_catalog.compression_algorithm WHERE id = 8 AND version = 1 AND name = 'COMPRESSION_ALGORITHM_EXTERNAL';
+
+-- Can't downgrade while external compression codecs are registered. Chunks may
+-- still hold batches only their codec can read, and the older version has no
+-- EXTERNAL algorithm support.
+-- First, clear the timescaledb.compress_column_codec settings that reference
+-- the operator class so new compression uses the built-in algorithms. Then
+-- rewrite every chunk using the codec with decompress_chunk() followed
+-- by compress_chunk(). Then drop the operator class.
+DO $$
+DECLARE
+  reg record;
+  clear_cmds text;
+  rewrite_cmds text;
+  decommission_cmds text := NULL;
+  codec_opclasses text := NULL;
+BEGIN
+  FOR reg IN
+    SELECT format('%I.%I', n.nspname, c.opcname) AS opclass,
+           c.opcintype::regtype AS type
+    FROM pg_catalog.pg_opclass c
+    JOIN pg_catalog.pg_am a ON a.oid = c.opcmethod
+    JOIN pg_catalog.pg_namespace n ON n.oid = c.opcnamespace
+    WHERE a.amname = 'ts_compression_codec'
+  LOOP
+    codec_opclasses := concat_ws(', ', codec_opclasses, reg.opclass);
+
+    -- Clear the codec settings of every hypertable that references this
+    -- operator class. The settings store the name the same way it is quoted
+    -- here, so string comparison finds the references.
+    SELECT string_agg(format('ALTER TABLE %s SET (timescaledb.compress_column_codec = '''');',
+                             format('%I.%I', h.schema_name, h.table_name)), E'\n')
+      INTO clear_cmds
+    FROM _timescaledb_catalog.hypertable h
+    JOIN _timescaledb_catalog.compression_settings cs
+      ON cs.relid = format('%I.%I', h.schema_name, h.table_name)::regclass
+    WHERE reg.opclass = ANY (cs.codec_opclass);
+
+    -- For (comparative) simplicity, just show instructions for rewriting every
+    -- possibly-affected chunk having a compressed column with this registered
+    -- codec.
+    SELECT string_agg(format(E'SELECT decompress_chunk(ch, true) FROM show_chunks(%L) ch;\n'
+                             'SELECT compress_chunk(ch) FROM show_chunks(%L) ch;', ht, ht), E'\n')
+      INTO rewrite_cmds
+    FROM (
+      SELECT DISTINCT format('%I.%I', h.schema_name, h.table_name)::regclass AS ht
+      FROM _timescaledb_catalog.hypertable h
+      JOIN pg_catalog.pg_attribute att
+        ON att.attrelid = format('%I.%I', h.schema_name, h.table_name)::regclass
+      WHERE att.atttypid = reg.type AND att.attnum > 0 AND NOT att.attisdropped
+    ) affected;
+
+    decommission_cmds := concat_ws(E'\n', decommission_cmds, format(
+      E'-- decommission codec %1$s for type %2$s\n'
+      '%3$s'
+      '%4$s'
+      'DROP OPERATOR CLASS %1$s USING ts_compression_codec;',
+      reg.opclass,
+      reg.type,
+      coalesce(clear_cmds || E'\n', ''),
+      coalesce(rewrite_cmds || E'\n', '')));
+  END LOOP;
+
+  IF codec_opclasses IS NOT NULL THEN
+    RAISE EXCEPTION 'cannot downgrade while compression codec operator classes exist: %', codec_opclasses
+      USING DETAIL = 'Chunks compressed while a codec is registered may hold EXTERNAL batches that the older version cannot read.',
+            HINT = format(E'Run the following, then retry the downgrade:\n%s', decommission_cmds);
+  END IF;
+END
+$$;
+DROP ACCESS METHOD IF EXISTS ts_compression_codec;
+DROP FUNCTION IF EXISTS _timescaledb_functions.compression_codec_handler(internal);
+
+-- The older version has no per-column external codec setting.
+ALTER TABLE _timescaledb_catalog.compression_settings
+  DROP CONSTRAINT compression_settings_check_codec_cardinality,
+  DROP CONSTRAINT compression_settings_check_codec_null,
+  DROP COLUMN codec_column,
+  DROP COLUMN codec_opclass;

src/CMakeLists.txt

@@ -11,6 +11,7 @@ set(SOURCES
     chunk_insert_state.c
     chunk_scan.c
     chunk_tuple_routing.c
+    compression_codec_am.c
     constraint.c
     cross_module_fn.c
     copy.c