From 75cbaac39bdefcb439a26f9ce1299d6a9ae73d8c Mon Sep 17 00:00:00 2001 From: Natalya Aksman Date: Thu, 4 Jun 2026 15:39:08 -0400 Subject: [PATCH] Enable segmentwise recompression for nullable order by columns with firstlast metadata index --- tsl/src/compression/api.c | 24 +-- tsl/src/compression/api.h | 2 +- tsl/src/compression/compression_dml.h | 2 +- tsl/src/compression/compression_scankey.c | 22 ++- tsl/src/compression/recompress.c | 14 +- .../recompression_nullable_orderby.out | 184 +++++++++++++++++- .../sql/recompression_nullable_orderby.sql | 137 ++++++++++++- 7 files changed, 355 insertions(+), 30 deletions(-) diff --git a/tsl/src/compression/api.c b/tsl/src/compression/api.c index 070524dc3e5..ce642b8dedc 100644 --- a/tsl/src/compression/api.c +++ b/tsl/src/compression/api.c @@ -715,18 +715,21 @@ decompress_chunk_impl(Chunk *uncompressed_chunk, bool if_compressed) } bool -is_chunk_orderby_nonnullable(CompressionSettings *settings) +is_chunk_orderby_nullhandling(CompressionSettings *settings) { int num_orderby = ts_array_length(settings->fd.orderby); const char *attname; int attnum; for (int i = 1; i <= num_orderby; i++) { - attname = ts_array_get_element_text(settings->fd.orderby, i); - attnum = get_attnum(settings->fd.relid, attname); - if (!AttributeNumberIsValid(attnum) || !ts_get_attnotnull(settings->fd.relid, attnum)) + if (orderby_sparse_kind(settings, i) != ORDERBY_SPARSE_FIRSTLAST) { - return false; + attname = ts_array_get_element_text(settings->fd.orderby, i); + attnum = get_attnum(settings->fd.relid, attname); + if (!AttributeNumberIsValid(attnum) || !ts_get_attnotnull(settings->fd.relid, attnum)) + { + return false; + } } } return true; @@ -777,19 +780,18 @@ recompress_chunk_impl(Chunk *chunk, bool recompress) /* #9444: do not recompress when order by columns are nullable, do segmentwise * decompress/compress instead. It is due to compression min/max metadata not handling - * NULLs. When we implement chunks with min/max NULL-handling metadata, this restriction can - * be lifted. + * NULLs. This restriction is lifted with first/last metadata index. */ - bool nullable_orderby = !is_chunk_orderby_nonnullable(chunk_settings); - if (nullable_orderby) + bool orderby_not_handling_nulls = !is_chunk_orderby_nullhandling(chunk_settings); + if (orderby_not_handling_nulls) { elog(ts_guc_debug_compression_path_info ? INFO : DEBUG1, - "in-memory recompression is disabled due to nullable order by, " + "in-memory recompression is disabled due to nullable order by with no firstlast, " "performing segmentwise decompress/compress on chunk \"%s.%s\"", NameStr(chunk->fd.schema_name), NameStr(chunk->fd.table_name)); } - recompress_chunk_segmentwise_impl(chunk, nullable_orderby); + recompress_chunk_segmentwise_impl(chunk, orderby_not_handling_nulls); recompressed = true; } else diff --git a/tsl/src/compression/api.h b/tsl/src/compression/api.h index 82d0b49ddf6..82b4474650f 100644 --- a/tsl/src/compression/api.h +++ b/tsl/src/compression/api.h @@ -31,4 +31,4 @@ extern void compression_chunk_size_catalog_insert(int32 src_chunk_id, const Rela int64 rowcnt_frozen); extern Datum tsl_estimate_compressed_batch_size(PG_FUNCTION_ARGS); -extern bool is_chunk_orderby_nonnullable(CompressionSettings *settings); +extern bool is_chunk_orderby_nullhandling(CompressionSettings *settings); diff --git a/tsl/src/compression/compression_dml.h b/tsl/src/compression/compression_dml.h index 4cf5b31a0e0..8a97170498f 100644 --- a/tsl/src/compression/compression_dml.h +++ b/tsl/src/compression/compression_dml.h @@ -31,7 +31,7 @@ typedef struct tuple_filtering_constraints bool vectorized_filtering; } tuple_filtering_constraints; -bool slot_key_test(TupleTableSlot *slot, ScanKey skey); +bool slot_key_test(TupleTableSlot *slot, ScanKey skey, bool nulls_first); ScanKeyData *build_mem_scankeys_from_slot(Oid ht_relid, CompressionSettings *settings, Relation out_rel, diff --git a/tsl/src/compression/compression_scankey.c b/tsl/src/compression/compression_scankey.c index ccea6abf138..d636eb78b3a 100644 --- a/tsl/src/compression/compression_scankey.c +++ b/tsl/src/compression/compression_scankey.c @@ -27,11 +27,15 @@ static bool create_segment_filter_scankey(Relation in_rel, char *segment_filter_ * * Unlike HeapKeyTest, this function takes into account SK_ISNULL * and works correctly when looking for null values. + * + * If slot attribute is NULL and key is NOT NULL, + * (key >= NULL) returns True for nulls_first + * and (key <= NULL) returns True for !nulls_first (i.e. for NULLS LAST). */ bool -slot_key_test(TupleTableSlot *compressed_slot, ScanKey key) +slot_key_test(TupleTableSlot *compressed_slot, ScanKey key, bool nulls_first) { - /* No need to get the datum if we are only checking for NULLs */ + /* No need to get the datum if we are only checking for NULL key */ if (key->sk_flags & SK_ISNULL) { return slot_attisnull(compressed_slot, key->sk_attno); @@ -43,6 +47,20 @@ slot_key_test(TupleTableSlot *compressed_slot, ScanKey key) if (is_null) { + /* NULL < key i.e. NULL sorts before key argument */ + if (nulls_first && (key->sk_strategy == BTLessStrategyNumber || + key->sk_strategy == BTLessEqualStrategyNumber)) + { + return true; + } + + /* NULL > key i.e. NULL sorts after key argument */ + if (!nulls_first && (key->sk_strategy == BTGreaterStrategyNumber || + key->sk_strategy == BTGreaterEqualStrategyNumber)) + { + return true; + } + return false; } diff --git a/tsl/src/compression/recompress.c b/tsl/src/compression/recompress.c index 18edbca0b65..5d158937c5f 100644 --- a/tsl/src/compression/recompress.c +++ b/tsl/src/compression/recompress.c @@ -122,16 +122,16 @@ tsl_recompress_chunk_segmentwise(PG_FUNCTION_ARGS) "compression with no " "order by"))); } - bool nullable_orderby = !is_chunk_orderby_nonnullable(settings); - if (nullable_orderby) + bool orderby_not_handling_nulls = !is_chunk_orderby_nullhandling(settings); + if (orderby_not_handling_nulls) { elog(ts_guc_debug_compression_path_info ? INFO : DEBUG1, - "in-memory recompression is disabled due to nullable order by, " + "in-memory recompression is disabled due to nullable order by with no firstlast, " "performing segmentwise decompress/compress on chunk \"%s.%s\"", NameStr(chunk->fd.schema_name), NameStr(chunk->fd.table_name)); } - recompress_chunk_segmentwise_impl(chunk, nullable_orderby); + recompress_chunk_segmentwise_impl(chunk, orderby_not_handling_nulls); } PG_RETURN_OID(uncompressed_relid); @@ -1063,6 +1063,7 @@ update_orderby_scankeys(Datum *values, bool *isnulls, int num_segmentby, int num static enum Batch_match_result handle_null_scan(int key_flags, bool nulls_first, enum Batch_match_result result) { + /* uncompressed tuple key is NULL */ if (key_flags & SK_ISNULL) { return nulls_first ? Tuple_before : Tuple_after; @@ -1086,18 +1087,17 @@ match_tuple_batch(TupleTableSlot *compressed_slot, int num_orderby, ScanKey orde if (num_orderby >= 1) { ScanKey key = &orderby_scankeys[0]; - if (!slot_key_test(compressed_slot, key)) + if (!slot_key_test(compressed_slot, key, nulls_first[0])) { return handle_null_scan(key->sk_flags, nulls_first[0], Tuple_before); } key = &orderby_scankeys[1]; - if (!slot_key_test(compressed_slot, key)) + if (!slot_key_test(compressed_slot, key, nulls_first[0])) { return handle_null_scan(key->sk_flags, nulls_first[0], Tuple_after); } } - return Tuple_match; } diff --git a/tsl/test/expected/recompression_nullable_orderby.out b/tsl/test/expected/recompression_nullable_orderby.out index 97c2dfb77f8..2b3d5d8a273 100644 --- a/tsl/test/expected/recompression_nullable_orderby.out +++ b/tsl/test/expected/recompression_nullable_orderby.out @@ -1,9 +1,10 @@ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. +\c :TEST_DBNAME :ROLE_SUPERUSER -- #9444: do not recompress when order by columns are nullable, do decompress/compress instead. -- It is due to compression min/max metadata not handling NULLs. --- When we implement chunks with min/max NULL-handling metadata, this restriction can be lifted. +-- For chunks with min/max NULL-handling metadata (i.e. with first/last metadata), this restriction can be lifted. SET timescaledb.enable_direct_compress_insert TO OFF; SET timescaledb.batch_sorted_merge = 'off'; CREATE TABLE t1(time int, dev int, v1 int); @@ -22,12 +23,29 @@ SELECT compress_chunk(show_chunks('t1')); ---------------------------------------- _timescaledb_internal._hyper_1_1_chunk +-- Remove firstlast index from order by columns +update _timescaledb_catalog.compression_settings +set index = '[{"type": "minmax", "column": "v1", "source": "orderby"}, {"type": "minmax", "column": "time", "source": "orderby"}]' +where relid = 't1'::regclass; +update _timescaledb_catalog.compression_settings +set index = '[{"type": "minmax", "column": "v1", "source": "orderby"}, {"type": "minmax", "column": "time", "source": "orderby"}]' +where compress_relid = (select format('%I.%I', schema_name, table_name)::regclass AS chunk_regclass from _timescaledb_catalog.chunk + where id = (select compressed_chunk_id from _timescaledb_catalog.chunk + where hypertable_id = (select id from _timescaledb_catalog.hypertable + where table_name = 't1') limit 1)); +-- Use minmax index on (v1, time DESC) instead +select schema_name || '.' || table_name comp_chunk from _timescaledb_catalog.chunk + where id = (select compressed_chunk_id from _timescaledb_catalog.chunk + where hypertable_id = (select id from _timescaledb_catalog.hypertable + where table_name = 't1') limit 1) +\gset +create index t1_compressed_index_minmax on :comp_chunk (dev, _ts_meta_min_1, _ts_meta_max_1, _ts_meta_min_2 DESC, _ts_meta_max_2 DESC); -- Now this chunk is partial INSERT INTO t1 SELECT 1, 1, g FROM generate_series(901,1400) g; -- Should see an info about not recompressing because of nullable order by SET timescaledb.debug_compression_path_info TO ON; SELECT compress_chunk(show_chunks('t1'), if_not_compressed => true); -INFO: in-memory recompression is disabled due to nullable order by, performing segmentwise decompress/compress on chunk "_timescaledb_internal._hyper_1_1_chunk" +INFO: in-memory recompression is disabled due to nullable order by with no firstlast, performing segmentwise decompress/compress on chunk "_timescaledb_internal._hyper_1_1_chunk" INFO: Using index "compress_hyper_2_2_chunk_dev__ts_meta_v2_first_v1__ts_meta__idx" for recompression compress_chunk ---------------------------------------- @@ -50,7 +68,7 @@ INSERT INTO t1 SELECT 1, 2, g FROM generate_series(901,1400) g; SET client_min_messages = 'DEBUG1'; SELECT compress_chunk(show_chunks('t1'), if_not_compressed => true); LOG: statement: SELECT compress_chunk(show_chunks('t1'), if_not_compressed => true); -DEBUG: in-memory recompression is disabled due to nullable order by, performing segmentwise decompress/compress on chunk "_timescaledb_internal._hyper_1_1_chunk" +DEBUG: in-memory recompression is disabled due to nullable order by with no firstlast, performing segmentwise decompress/compress on chunk "_timescaledb_internal._hyper_1_1_chunk" DEBUG: acquiring locks for recompression: "_timescaledb_internal._hyper_1_1_chunk" DEBUG: Using index "compress_hyper_2_2_chunk_dev__ts_meta_v2_first_v1__ts_meta__idx" for recompression DEBUG: locks acquired for recompression: "_timescaledb_internal._hyper_1_1_chunk" @@ -72,7 +90,7 @@ SELECT show_chunks as chunk_to_recompress FROM show_chunks('t1') LIMIT 1 \gset LOG: statement: SELECT show_chunks as chunk_to_recompress FROM show_chunks('t1') LIMIT 1 SELECT _timescaledb_functions.recompress_chunk_segmentwise(:'chunk_to_recompress'); LOG: statement: SELECT _timescaledb_functions.recompress_chunk_segmentwise('_timescaledb_internal._hyper_1_1_chunk'); -DEBUG: in-memory recompression is disabled due to nullable order by, performing segmentwise decompress/compress on chunk "_timescaledb_internal._hyper_1_1_chunk" +DEBUG: in-memory recompression is disabled due to nullable order by with no firstlast, performing segmentwise decompress/compress on chunk "_timescaledb_internal._hyper_1_1_chunk" DEBUG: acquiring locks for recompression: "_timescaledb_internal._hyper_1_1_chunk" DEBUG: Using index "compress_hyper_2_2_chunk_dev__ts_meta_v2_first_v1__ts_meta__idx" for recompression DEBUG: locks acquired for recompression: "_timescaledb_internal._hyper_1_1_chunk" @@ -84,7 +102,7 @@ DEBUG: cleared chunk status for recompression: "_timescaledb_internal._hyper_1_ RESET client_min_messages; LOG: statement: RESET client_min_messages; --- If only some of order by columns are nullable we still bail out on recompress +-- If only some of order by columns are nullable without firstlast, we still bail out on recompress CREATE TABLE t2(time int NOT NULL, v1 int NOT NULL, v2 int); SELECT create_hypertable('t2', 'time', chunk_time_interval => 10000); create_hypertable @@ -99,6 +117,23 @@ SELECT compress_chunk(show_chunks('t2')); ---------------------------------------- _timescaledb_internal._hyper_3_3_chunk +-- Remove firstlast index from nullable order by column "v2" +update _timescaledb_catalog.compression_settings +set index = '[{"type": "minmax", "column": "v1", "source": "orderby"}, {"type": "firstlast", "column": "v1", "source": "orderby"}, {"type": "minmax", "column": "v2", "source": "orderby"}, {"type": "minmax", "column": "time", "source": "orderby"}, {"type": "firstlast", "column": "time", "source": "orderby"}]' +where relid = 't2'::regclass; +update _timescaledb_catalog.compression_settings +set index = '[{"type": "minmax", "column": "v1", "source": "orderby"}, {"type": "firstlast", "column": "v1", "source": "orderby"}, {"type": "minmax", "column": "v2", "source": "orderby"}, {"type": "minmax", "column": "time", "source": "orderby"}, {"type": "firstlast", "column": "time", "source": "orderby"}]' +where compress_relid = (select format('%I.%I', schema_name, table_name)::regclass AS chunk_regclass from _timescaledb_catalog.chunk + where id = (select compressed_chunk_id from _timescaledb_catalog.chunk + where hypertable_id = (select id from _timescaledb_catalog.hypertable + where table_name = 't2') limit 1)); +-- Use minmax index on (v1, v2, time DESC) instead +select schema_name || '.' || table_name comp_chunk from _timescaledb_catalog.chunk + where id = (select compressed_chunk_id from _timescaledb_catalog.chunk + where hypertable_id = (select id from _timescaledb_catalog.hypertable + where table_name = 't2') limit 1) +\gset +create index t2_compressed_index_minmax on :comp_chunk (_ts_meta_min_1, _ts_meta_max_1, _ts_meta_min_2, _ts_meta_max_2, _ts_meta_min_3 DESC, _ts_meta_max_3 DESC); -- Now this chunk is partial INSERT INTO t2 SELECT 1, 1, g FROM generate_series(901,1400) g; -- Use a function to recompress-segmentwise @@ -106,7 +141,7 @@ INSERT INTO t2 SELECT 1, 1, g FROM generate_series(901,1400) g; SELECT show_chunks as chunk_to_recompress FROM show_chunks('t2') LIMIT 1 \gset SET timescaledb.debug_compression_path_info TO ON; SELECT _timescaledb_functions.recompress_chunk_segmentwise(:'chunk_to_recompress'); -INFO: in-memory recompression is disabled due to nullable order by, performing segmentwise decompress/compress on chunk "_timescaledb_internal._hyper_3_3_chunk" +INFO: in-memory recompression is disabled due to nullable order by with no firstlast, performing segmentwise decompress/compress on chunk "_timescaledb_internal._hyper_3_3_chunk" INFO: Using index "compress_hyper_4_4_chunk__ts_meta_v2_first_v1__ts_meta_v2_l_idx" for recompression recompress_chunk_segmentwise ---------------------------------------- @@ -124,5 +159,142 @@ SELECT count(*) AS wrong_rows FROM ( drop table t1 cascade; drop table t2 cascade; +-- Tests for nullable order by columns with firstlast index +------------------------------------------------------------ +--- NULLS LAST: batch (1, NULL) +CREATE TABLE test1(time int, dev int, v1 int); +-- t1 is declared NOT NULL on a hypertable as partitioning column +SELECT create_hypertable('test1', 'time', chunk_time_interval => 10000); + create_hypertable +-------------------- + (5,public,test1,t) + +ALTER TABLE test1 SET (timescaledb.compress, timescaledb.compress_segmentby='dev', timescaledb.compress_orderby='v1'); +-- Insert data for one segment +INSERT INTO test1 SELECT 1, 1, g FROM generate_series(1,800) g; +INSERT INTO test1 SELECT 1, 1, NULL FROM generate_series(1,200); +SELECT compress_chunk(show_chunks('test1')); + compress_chunk +---------------------------------------- + _timescaledb_internal._hyper_5_5_chunk + +-- Now this chunk is partial, this batch should match [1, NULL] batch instead of sorting after it +INSERT INTO test1 SELECT 1, 1, g FROM generate_series(901,1400) g; +-- Should not see info about nullable order by +SET timescaledb.debug_compression_path_info TO ON; +SELECT compress_chunk(show_chunks('test1'), if_not_compressed => true); +INFO: Using index "compress_hyper_6_6_chunk_dev__ts_meta_v2_first_v1__ts_meta__idx" for recompression + compress_chunk +---------------------------------------- + _timescaledb_internal._hyper_5_5_chunk + +RESET timescaledb.debug_compression_path_info; +-- This query orders NULLS LAST, so we can't have a non-NULL value after a NULL value. +-- Should return 0. +SELECT count(*) AS wrong_rows FROM ( + SELECT v1, lead(v1) OVER (ORDER BY v1) AS next FROM test1 WHERE dev=1 ORDER BY v1 +) t WHERE v1 IS NULL AND next IS NOT NULL; + wrong_rows +------------ + 0 + +-- All NULLs, NULLS LAST, batch (NULL, NULL) +CREATE TABLE test2(time int NOT NULL, val int); +SELECT create_hypertable('test2', 'time', chunk_time_interval => 10000); + create_hypertable +-------------------- + (7,public,test2,t) + +ALTER TABLE test2 SET (timescaledb.compress, timescaledb.compress_orderby='val'); +INSERT INTO test2 SELECT 1, g FROM generate_series(501,1500) g; +INSERT INTO test2 SELECT 1, NULL FROM generate_series(1,1000); +SELECT compress_chunk(show_chunks('test2')); + compress_chunk +---------------------------------------- + _timescaledb_internal._hyper_7_7_chunk + +-- Now this chunk is partial, recompress will be applied +INSERT INTO test2 SELECT 1, g FROM generate_series(1300,1600) g; +SELECT compress_chunk(show_chunks('test2'), if_not_compressed => true); + compress_chunk +---------------------------------------- + _timescaledb_internal._hyper_7_7_chunk + +-- Should return 0 +SELECT count(*) AS wrong_rows FROM ( + SELECT val, lead(val) OVER (ORDER BY val) AS next FROM test2 ORDER BY val +) t WHERE val IS NOT NULL AND next IS NOT NULL AND val > next; + wrong_rows +------------ + 0 + +-- All NULLs, NULLS FIRST, batch (NULL, NULL) +CREATE TABLE test3(time int NOT NULL, val int); +SELECT create_hypertable('test3', 'time', chunk_time_interval => 10000); + create_hypertable +-------------------- + (9,public,test3,t) + +ALTER TABLE test3 SET (timescaledb.compress, timescaledb.compress_orderby='val NULLS FIRST'); +INSERT INTO test3 SELECT 1, NULL FROM generate_series(1,1000); +INSERT INTO test3 SELECT 1, g FROM generate_series(501,1500) g; +SELECT compress_chunk(show_chunks('test3')); + compress_chunk +---------------------------------------- + _timescaledb_internal._hyper_9_9_chunk + +-- Now this chunk is partial, recompress will be applied +INSERT INTO test3 SELECT 1, g FROM generate_series(400,600) g; +SELECT compress_chunk(show_chunks('test3'), if_not_compressed => true); + compress_chunk +---------------------------------------- + _timescaledb_internal._hyper_9_9_chunk + +-- Should return 0 +SELECT count(*) AS wrong_rows FROM ( + SELECT lag(val) OVER (ORDER BY val NULLS FIRST) AS prev, val FROM test3 ORDER BY val NULLS FIRST +) t WHERE val IS NOT NULL AND prev IS NOT NULL AND val < prev; + wrong_rows +------------ + 0 + +--- NULLS FIRST: batch (NULL, 1) +CREATE TABLE test4(time int, dev int, v1 int); +-- t1 is declared NOT NULL on a hypertable as partitioning column +SELECT create_hypertable('test4', 'time', chunk_time_interval => 10000); + create_hypertable +--------------------- + (11,public,test4,t) + +ALTER TABLE test4 SET (timescaledb.compress, timescaledb.compress_segmentby='dev', timescaledb.compress_orderby='v1 NULLS FIRST'); +-- Insert data for one segment +INSERT INTO test4 SELECT 1, 1, NULL FROM generate_series(1,200); +INSERT INTO test4 SELECT 1, 1, g FROM generate_series(1001,1800) g; +SELECT compress_chunk(show_chunks('test4')); + compress_chunk +------------------------------------------ + _timescaledb_internal._hyper_11_11_chunk + +-- Now this chunk is partial, recompress will be applied +-- This batch should match [NULL... 1001...1800] batch instead of going before it +INSERT INTO test4 SELECT 1, 1, g FROM generate_series(500,900) g; +SELECT compress_chunk(show_chunks('test4'), if_not_compressed => true); + compress_chunk +------------------------------------------ + _timescaledb_internal._hyper_11_11_chunk + +-- This query orders NULLS FIRST, so we can't have a non-NULL value before a NULL value. +-- Should return 0. +SELECT count(*) AS wrong_rows FROM ( + SELECT lag(v1) OVER (ORDER BY v1 NULLS FIRST) AS prev, v1 FROM test4 WHERE dev=1 ORDER BY v1 NULLS FIRST +) t WHERE v1 IS NULL AND prev IS NOT NULL; + wrong_rows +------------ + 0 + +drop table test1 cascade; +drop table test2 cascade; +drop table test3 cascade; +drop table test4 cascade; RESET timescaledb.enable_direct_compress_insert; RESET timescaledb.batch_sorted_merge; diff --git a/tsl/test/sql/recompression_nullable_orderby.sql b/tsl/test/sql/recompression_nullable_orderby.sql index faa16b5d70e..bf1c4f8112c 100644 --- a/tsl/test/sql/recompression_nullable_orderby.sql +++ b/tsl/test/sql/recompression_nullable_orderby.sql @@ -2,9 +2,11 @@ -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. +\c :TEST_DBNAME :ROLE_SUPERUSER + -- #9444: do not recompress when order by columns are nullable, do decompress/compress instead. -- It is due to compression min/max metadata not handling NULLs. --- When we implement chunks with min/max NULL-handling metadata, this restriction can be lifted. +-- For chunks with min/max NULL-handling metadata (i.e. with first/last metadata), this restriction can be lifted. SET timescaledb.enable_direct_compress_insert TO OFF; SET timescaledb.batch_sorted_merge = 'off'; @@ -19,6 +21,26 @@ INSERT INTO t1 SELECT 1, 1, g FROM generate_series(1,800) g; INSERT INTO t1 SELECT 1, 1, NULL FROM generate_series(1,200); SELECT compress_chunk(show_chunks('t1')); +-- Remove firstlast index from order by columns +update _timescaledb_catalog.compression_settings +set index = '[{"type": "minmax", "column": "v1", "source": "orderby"}, {"type": "minmax", "column": "time", "source": "orderby"}]' +where relid = 't1'::regclass; + +update _timescaledb_catalog.compression_settings +set index = '[{"type": "minmax", "column": "v1", "source": "orderby"}, {"type": "minmax", "column": "time", "source": "orderby"}]' +where compress_relid = (select format('%I.%I', schema_name, table_name)::regclass AS chunk_regclass from _timescaledb_catalog.chunk + where id = (select compressed_chunk_id from _timescaledb_catalog.chunk + where hypertable_id = (select id from _timescaledb_catalog.hypertable + where table_name = 't1') limit 1)); + +-- Use minmax index on (v1, time DESC) instead +select schema_name || '.' || table_name comp_chunk from _timescaledb_catalog.chunk + where id = (select compressed_chunk_id from _timescaledb_catalog.chunk + where hypertable_id = (select id from _timescaledb_catalog.hypertable + where table_name = 't1') limit 1) +\gset +create index t1_compressed_index_minmax on :comp_chunk (dev, _ts_meta_min_1, _ts_meta_max_1, _ts_meta_min_2 DESC, _ts_meta_max_2 DESC); + -- Now this chunk is partial INSERT INTO t1 SELECT 1, 1, g FROM generate_series(901,1400) g; @@ -53,7 +75,7 @@ SELECT show_chunks as chunk_to_recompress FROM show_chunks('t1') LIMIT 1 \gset SELECT _timescaledb_functions.recompress_chunk_segmentwise(:'chunk_to_recompress'); RESET client_min_messages; --- If only some of order by columns are nullable we still bail out on recompress +-- If only some of order by columns are nullable without firstlast, we still bail out on recompress CREATE TABLE t2(time int NOT NULL, v1 int NOT NULL, v2 int); SELECT create_hypertable('t2', 'time', chunk_time_interval => 10000); ALTER TABLE t2 SET (timescaledb.compress, timescaledb.compress_orderby='v1,v2'); @@ -62,6 +84,26 @@ INSERT INTO t2 SELECT 1, 1, g FROM generate_series(1,800) g; INSERT INTO t2 SELECT 1, 1, NULL FROM generate_series(1,200); SELECT compress_chunk(show_chunks('t2')); +-- Remove firstlast index from nullable order by column "v2" +update _timescaledb_catalog.compression_settings +set index = '[{"type": "minmax", "column": "v1", "source": "orderby"}, {"type": "firstlast", "column": "v1", "source": "orderby"}, {"type": "minmax", "column": "v2", "source": "orderby"}, {"type": "minmax", "column": "time", "source": "orderby"}, {"type": "firstlast", "column": "time", "source": "orderby"}]' +where relid = 't2'::regclass; + +update _timescaledb_catalog.compression_settings +set index = '[{"type": "minmax", "column": "v1", "source": "orderby"}, {"type": "firstlast", "column": "v1", "source": "orderby"}, {"type": "minmax", "column": "v2", "source": "orderby"}, {"type": "minmax", "column": "time", "source": "orderby"}, {"type": "firstlast", "column": "time", "source": "orderby"}]' +where compress_relid = (select format('%I.%I', schema_name, table_name)::regclass AS chunk_regclass from _timescaledb_catalog.chunk + where id = (select compressed_chunk_id from _timescaledb_catalog.chunk + where hypertable_id = (select id from _timescaledb_catalog.hypertable + where table_name = 't2') limit 1)); + +-- Use minmax index on (v1, v2, time DESC) instead +select schema_name || '.' || table_name comp_chunk from _timescaledb_catalog.chunk + where id = (select compressed_chunk_id from _timescaledb_catalog.chunk + where hypertable_id = (select id from _timescaledb_catalog.hypertable + where table_name = 't2') limit 1) +\gset +create index t2_compressed_index_minmax on :comp_chunk (_ts_meta_min_1, _ts_meta_max_1, _ts_meta_min_2, _ts_meta_max_2, _ts_meta_min_3 DESC, _ts_meta_max_3 DESC); + -- Now this chunk is partial INSERT INTO t2 SELECT 1, 1, g FROM generate_series(901,1400) g; @@ -81,5 +123,96 @@ SELECT count(*) AS wrong_rows FROM ( drop table t1 cascade; drop table t2 cascade; +-- Tests for nullable order by columns with firstlast index +------------------------------------------------------------ + +--- NULLS LAST: batch (1, NULL) +CREATE TABLE test1(time int, dev int, v1 int); +-- t1 is declared NOT NULL on a hypertable as partitioning column +SELECT create_hypertable('test1', 'time', chunk_time_interval => 10000); +ALTER TABLE test1 SET (timescaledb.compress, timescaledb.compress_segmentby='dev', timescaledb.compress_orderby='v1'); + +-- Insert data for one segment +INSERT INTO test1 SELECT 1, 1, g FROM generate_series(1,800) g; +INSERT INTO test1 SELECT 1, 1, NULL FROM generate_series(1,200); +SELECT compress_chunk(show_chunks('test1')); + +-- Now this chunk is partial, this batch should match [1, NULL] batch instead of sorting after it +INSERT INTO test1 SELECT 1, 1, g FROM generate_series(901,1400) g; + +-- Should not see info about nullable order by +SET timescaledb.debug_compression_path_info TO ON; +SELECT compress_chunk(show_chunks('test1'), if_not_compressed => true); +RESET timescaledb.debug_compression_path_info; + +-- This query orders NULLS LAST, so we can't have a non-NULL value after a NULL value. +-- Should return 0. +SELECT count(*) AS wrong_rows FROM ( + SELECT v1, lead(v1) OVER (ORDER BY v1) AS next FROM test1 WHERE dev=1 ORDER BY v1 +) t WHERE v1 IS NULL AND next IS NOT NULL; + +-- All NULLs, NULLS LAST, batch (NULL, NULL) +CREATE TABLE test2(time int NOT NULL, val int); +SELECT create_hypertable('test2', 'time', chunk_time_interval => 10000); +ALTER TABLE test2 SET (timescaledb.compress, timescaledb.compress_orderby='val'); + +INSERT INTO test2 SELECT 1, g FROM generate_series(501,1500) g; +INSERT INTO test2 SELECT 1, NULL FROM generate_series(1,1000); +SELECT compress_chunk(show_chunks('test2')); + +-- Now this chunk is partial, recompress will be applied +INSERT INTO test2 SELECT 1, g FROM generate_series(1300,1600) g; +SELECT compress_chunk(show_chunks('test2'), if_not_compressed => true); + +-- Should return 0 +SELECT count(*) AS wrong_rows FROM ( + SELECT val, lead(val) OVER (ORDER BY val) AS next FROM test2 ORDER BY val +) t WHERE val IS NOT NULL AND next IS NOT NULL AND val > next; + +-- All NULLs, NULLS FIRST, batch (NULL, NULL) +CREATE TABLE test3(time int NOT NULL, val int); +SELECT create_hypertable('test3', 'time', chunk_time_interval => 10000); +ALTER TABLE test3 SET (timescaledb.compress, timescaledb.compress_orderby='val NULLS FIRST'); + +INSERT INTO test3 SELECT 1, NULL FROM generate_series(1,1000); +INSERT INTO test3 SELECT 1, g FROM generate_series(501,1500) g; +SELECT compress_chunk(show_chunks('test3')); + +-- Now this chunk is partial, recompress will be applied +INSERT INTO test3 SELECT 1, g FROM generate_series(400,600) g; +SELECT compress_chunk(show_chunks('test3'), if_not_compressed => true); + +-- Should return 0 +SELECT count(*) AS wrong_rows FROM ( + SELECT lag(val) OVER (ORDER BY val NULLS FIRST) AS prev, val FROM test3 ORDER BY val NULLS FIRST +) t WHERE val IS NOT NULL AND prev IS NOT NULL AND val < prev; + +--- NULLS FIRST: batch (NULL, 1) +CREATE TABLE test4(time int, dev int, v1 int); +-- t1 is declared NOT NULL on a hypertable as partitioning column +SELECT create_hypertable('test4', 'time', chunk_time_interval => 10000); +ALTER TABLE test4 SET (timescaledb.compress, timescaledb.compress_segmentby='dev', timescaledb.compress_orderby='v1 NULLS FIRST'); + +-- Insert data for one segment +INSERT INTO test4 SELECT 1, 1, NULL FROM generate_series(1,200); +INSERT INTO test4 SELECT 1, 1, g FROM generate_series(1001,1800) g; +SELECT compress_chunk(show_chunks('test4')); + +-- Now this chunk is partial, recompress will be applied +-- This batch should match [NULL... 1001...1800] batch instead of going before it +INSERT INTO test4 SELECT 1, 1, g FROM generate_series(500,900) g; +SELECT compress_chunk(show_chunks('test4'), if_not_compressed => true); + +-- This query orders NULLS FIRST, so we can't have a non-NULL value before a NULL value. +-- Should return 0. +SELECT count(*) AS wrong_rows FROM ( + SELECT lag(v1) OVER (ORDER BY v1 NULLS FIRST) AS prev, v1 FROM test4 WHERE dev=1 ORDER BY v1 NULLS FIRST +) t WHERE v1 IS NULL AND prev IS NOT NULL; + +drop table test1 cascade; +drop table test2 cascade; +drop table test3 cascade; +drop table test4 cascade; + RESET timescaledb.enable_direct_compress_insert; RESET timescaledb.batch_sorted_merge;