Skip to content

Commit ad266f8

Browse files
natalya-aksmantimescale-automation
natalya-aksman
authored and
timescale-automation
committed
Forbid Batch Sorted Merge on multikey minmax index (#9955)
Fixes #9922 Forbids Batch Sorted Merge when there is no firstlast index on a non-leading order by column. (cherry picked from commit 763e7b1)
1 parent 2a5dcef commit ad266f8

4 files changed

Lines changed: 196 additions & 8 deletions

File tree

‎.unreleased/pr_9955‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Fixes: #9955 Fix wrong results when using Batch Sorted Merge with no first-last index on a non-leading order by column

‎tsl/src/nodes/columnar_scan/columnar_scan.c‎

Lines changed: 26 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2853,15 +2853,33 @@ match_pathkeys_to_compression_orderby(List *pathkeys, List *chunk_em_exprs,
28532853
return false;
28542854
}
28552855

2856-
/* Bail out on BSM if orderby column is nullable,
2857-
* as at the moment the minmax metadata we have doesn't include NULLs,
2858-
* so it's difficult to use it for null-sensitive ordering.
2859-
* But this restriction can be lifted in the future on new type of chunks
2860-
* with NULL-handling metadata.
2861-
*/
2862-
if (for_bsm && !is_var_notnull(compression_info, var))
2856+
if (for_bsm)
28632857
{
2864-
return false;
2858+
/* Bail out on Batch Sorted Merge if orderby column is nullable,
2859+
* as at the moment the minmax metadata we have doesn't include NULLs,
2860+
* so it's difficult to use it for null-sensitive ordering.
2861+
* But this restriction can be lifted in the future on new type of chunks
2862+
* with NULL-handling metadata.
2863+
*/
2864+
if (!is_var_notnull(compression_info, var))
2865+
{
2866+
return false;
2867+
}
2868+
2869+
/* Bail out on Batch Sorted Merge with multiple order by keys
2870+
* if non-leading keys don't use firstlast index.
2871+
* Batches can be sorted incorrectly on multikey minmax index,
2872+
* for example
2873+
* [(1, 20) .. (1, 30), (2,0)...(2,30)] with min(1),(0)
2874+
* will be sorted before [(1,1) .. (1,19)] with min(1),(1)
2875+
* but it should be sorted after as (1,20) > (1,1): correct with firstlast index.
2876+
*/
2877+
if (compressed_pk_index > 1 &&
2878+
orderby_sparse_kind(compression_info->settings, orderby_index) !=
2879+
ORDERBY_SPARSE_FIRSTLAST)
2880+
{
2881+
return false;
2882+
}
28652883
}
28662884

28672885
bool orderby_desc =

‎tsl/test/expected/compression_sorted_merge.out‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1365,6 +1365,7 @@ SELECT "time","hin"::text,"model"::text,"block"::text,"message_name"::text,"sign
13651365
--------------------------+---------+----------+---------+--------------+-------------+----------------------+---------------------
13661366
Wed Jan 01 10:00:00 2020 | hin1111 | model111 | blok111 | message_here | signal1 | 12.34 | 12.34
13671367

1368+
drop table bugtab cascade;
13681369
-- Condition that filter the first tuple of a batch - Issue 5797
13691370
CREATE TABLE test (
13701371
id bigint,
@@ -1528,4 +1529,87 @@ SELECT t.dttm FROM test t ORDER BY t.dttm LIMIT 1;
15281529
Output: compress_hyper_14_26_chunk._ts_meta_count, compress_hyper_14_26_chunk.otherid, compress_hyper_14_26_chunk.valuefk, compress_hyper_14_26_chunk.otherfk, compress_hyper_14_26_chunk.id, compress_hyper_14_26_chunk._ts_meta_min_1, compress_hyper_14_26_chunk._ts_meta_max_1, compress_hyper_14_26_chunk.dttm, compress_hyper_14_26_chunk.measure
15291530

15301531
RESET enable_sort;
1532+
drop table test cascade;
1533+
-- Test issue #9922: wrong sort order for Batch Sorted Merge with multikey minmax index
1534+
\c :TEST_DBNAME :ROLE_SUPERUSER
1535+
CREATE TABLE bsm_segby(ts timestamptz NOT NULL, grp int NOT NULL, name text NOT NULL);
1536+
SELECT table_name FROM create_hypertable('bsm_segby','ts',chunk_time_interval=>interval '100 day');
1537+
table_name
1538+
------------
1539+
bsm_segby
1540+
1541+
ALTER TABLE bsm_segby SET (timescaledb.compress, timescaledb.compress_segmentby='grp', timescaledb.compress_orderby='name asc, ts asc');
1542+
INSERT INTO bsm_segby
1543+
SELECT '2024-08-01'::timestamptz + g*interval '1 min', g%4, 'k'||lpad(((g*5)%17)::text,3,'0')
1544+
FROM generate_series(0,4379) g;
1545+
SELECT count(compress_chunk(c)) FROM show_chunks('bsm_segby') c;
1546+
count
1547+
-------
1548+
1
1549+
1550+
select schema_name || '.' || table_name chunk from _timescaledb_catalog.chunk
1551+
where id = (select compressed_chunk_id from _timescaledb_catalog.chunk
1552+
where hypertable_id = (select id from _timescaledb_catalog.hypertable
1553+
where table_name = 'bsm_segby') limit 1)
1554+
\gset
1555+
-- Firstlast index is used with Batch Sorted Merge: correct result
1556+
set timescaledb.debug_require_batch_sorted_merge = 'force';
1557+
SELECT count(*) misorder FROM (
1558+
SELECT name, ts, lag(name) OVER () pn, lag(ts) OVER () pt
1559+
FROM (SELECT name, ts FROM bsm_segby ORDER BY name, ts) s) z
1560+
WHERE pn IS NOT NULL AND (name<pn OR (name=pn AND ts<pt));
1561+
misorder
1562+
----------
1563+
0
1564+
1565+
-- Remove firstlast index from leading column: still OK to use Batch Sorted Merge
1566+
update _timescaledb_catalog.compression_settings
1567+
set index = '[{"type": "minmax", "column": "name", "source": "orderby"}, {"type": "minmax", "column": "ts", "source": "orderby"}, {"type": "firstlast", "column": "ts", "source": "orderby"}]'
1568+
where relid = 'bsm_segby'::regclass;
1569+
update _timescaledb_catalog.compression_settings
1570+
set index = '[{"type": "minmax", "column": "name", "source": "orderby"}, {"type": "minmax", "column": "ts", "source": "orderby"}, {"type": "firstlast", "column": "ts", "source": "orderby"}]'
1571+
where compress_relid = (select format('%I.%I', schema_name, table_name)::regclass AS chunk_regclass from _timescaledb_catalog.chunk
1572+
where id = (select compressed_chunk_id from _timescaledb_catalog.chunk
1573+
where hypertable_id = (select id from _timescaledb_catalog.hypertable
1574+
where table_name = 'bsm_segby') limit 1));
1575+
create index compressed_index_minmax_firstlast on :chunk (grp, _ts_meta_min_1, _ts_meta_max_1, _ts_meta_v2_first_ts, _ts_meta_v2_last_ts);
1576+
-- Correct result with Batch Sorted Merge
1577+
set timescaledb.debug_require_batch_sorted_merge = 'force';
1578+
SELECT count(*) misorder FROM (
1579+
SELECT name, ts, lag(name) OVER () pn, lag(ts) OVER () pt
1580+
FROM (SELECT name, ts FROM bsm_segby ORDER BY name, ts) s) z
1581+
WHERE pn IS NOT NULL AND (name<pn OR (name=pn AND ts<pt));
1582+
misorder
1583+
----------
1584+
0
1585+
1586+
-- Now use minmax index on non-leading column "ts"
1587+
drop index _timescaledb_internal.compressed_index_minmax_firstlast;
1588+
create index compressed_index_minmax_minmax on :chunk (grp, _ts_meta_min_1, _ts_meta_max_1, _ts_meta_min_2, _ts_meta_max_2);
1589+
-- Remove firstlast index from all columns
1590+
update _timescaledb_catalog.compression_settings
1591+
set index = '[{"type": "minmax", "column": "name", "source": "orderby"}, {"type": "minmax", "column": "ts", "source": "orderby"}]'
1592+
where relid = 'bsm_segby'::regclass;
1593+
update _timescaledb_catalog.compression_settings
1594+
set index = '[{"type": "minmax", "column": "name", "source": "orderby"}, {"type": "minmax", "column": "ts", "source": "orderby"}]'
1595+
where compress_relid = (select format('%I.%I', schema_name, table_name)::regclass AS chunk_regclass from _timescaledb_catalog.chunk
1596+
where id = (select compressed_chunk_id from _timescaledb_catalog.chunk
1597+
where hypertable_id = (select id from _timescaledb_catalog.hypertable
1598+
where table_name = 'bsm_segby') limit 1));
1599+
-- Correct result only when cannot use Batch Sorted Merge
1600+
set timescaledb.debug_require_batch_sorted_merge = 'forbid';
1601+
SELECT count(*) misorder FROM (
1602+
SELECT name, ts, lag(name) OVER () pn, lag(ts) OVER () pt
1603+
FROM (SELECT name, ts FROM bsm_segby ORDER BY name, ts) s) z
1604+
WHERE pn IS NOT NULL AND (name<pn OR (name=pn AND ts<pt));
1605+
misorder
1606+
----------
1607+
0
1608+
1609+
drop table bsm_segby cascade;
1610+
drop table test1 cascade;
1611+
drop table test2 cascade;
1612+
drop table test_with_defined_null cascade;
1613+
drop table test_costs cascade;
1614+
drop table insert_test cascade;
15311615
reset timescaledb.debug_require_batch_sorted_merge;

‎tsl/test/sql/compression_sorted_merge.sql‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -555,6 +555,8 @@ SELECT "time","hin"::text,"model"::text,"block"::text,"message_name"::text,"sign
555555

556556
SELECT "time","hin"::text,"model"::text,"block"::text,"message_name"::text,"signal_name"::text,"signal_numeric_value","signal_string_value"::text FROM bugtab ORDER BY "time" DESC;
557557

558+
drop table bugtab cascade;
559+
558560
-- Condition that filter the first tuple of a batch - Issue 5797
559561
CREATE TABLE test (
560562
id bigint,
@@ -666,5 +668,88 @@ set timescaledb.debug_require_batch_sorted_merge = 'require';
666668
SELECT t.dttm FROM test t ORDER BY t.dttm LIMIT 1;
667669
RESET enable_sort;
668670

671+
drop table test cascade;
672+
673+
-- Test issue #9922: wrong sort order for Batch Sorted Merge with multikey minmax index
674+
\c :TEST_DBNAME :ROLE_SUPERUSER
675+
676+
CREATE TABLE bsm_segby(ts timestamptz NOT NULL, grp int NOT NULL, name text NOT NULL);
677+
SELECT table_name FROM create_hypertable('bsm_segby','ts',chunk_time_interval=>interval '100 day');
678+
ALTER TABLE bsm_segby SET (timescaledb.compress, timescaledb.compress_segmentby='grp', timescaledb.compress_orderby='name asc, ts asc');
679+
680+
INSERT INTO bsm_segby
681+
SELECT '2024-08-01'::timestamptz + g*interval '1 min', g%4, 'k'||lpad(((g*5)%17)::text,3,'0')
682+
FROM generate_series(0,4379) g;
683+
684+
SELECT count(compress_chunk(c)) FROM show_chunks('bsm_segby') c;
685+
686+
select schema_name || '.' || table_name chunk from _timescaledb_catalog.chunk
687+
where id = (select compressed_chunk_id from _timescaledb_catalog.chunk
688+
where hypertable_id = (select id from _timescaledb_catalog.hypertable
689+
where table_name = 'bsm_segby') limit 1)
690+
\gset
691+
692+
-- Firstlast index is used with Batch Sorted Merge: correct result
693+
set timescaledb.debug_require_batch_sorted_merge = 'force';
694+
695+
SELECT count(*) misorder FROM (
696+
SELECT name, ts, lag(name) OVER () pn, lag(ts) OVER () pt
697+
FROM (SELECT name, ts FROM bsm_segby ORDER BY name, ts) s) z
698+
WHERE pn IS NOT NULL AND (name<pn OR (name=pn AND ts<pt));
699+
700+
-- Remove firstlast index from leading column: still OK to use Batch Sorted Merge
701+
update _timescaledb_catalog.compression_settings
702+
set index = '[{"type": "minmax", "column": "name", "source": "orderby"}, {"type": "minmax", "column": "ts", "source": "orderby"}, {"type": "firstlast", "column": "ts", "source": "orderby"}]'
703+
where relid = 'bsm_segby'::regclass;
704+
705+
update _timescaledb_catalog.compression_settings
706+
set index = '[{"type": "minmax", "column": "name", "source": "orderby"}, {"type": "minmax", "column": "ts", "source": "orderby"}, {"type": "firstlast", "column": "ts", "source": "orderby"}]'
707+
where compress_relid = (select format('%I.%I', schema_name, table_name)::regclass AS chunk_regclass from _timescaledb_catalog.chunk
708+
where id = (select compressed_chunk_id from _timescaledb_catalog.chunk
709+
where hypertable_id = (select id from _timescaledb_catalog.hypertable
710+
where table_name = 'bsm_segby') limit 1));
711+
712+
create index compressed_index_minmax_firstlast on :chunk (grp, _ts_meta_min_1, _ts_meta_max_1, _ts_meta_v2_first_ts, _ts_meta_v2_last_ts);
713+
714+
-- Correct result with Batch Sorted Merge
715+
set timescaledb.debug_require_batch_sorted_merge = 'force';
716+
717+
SELECT count(*) misorder FROM (
718+
SELECT name, ts, lag(name) OVER () pn, lag(ts) OVER () pt
719+
FROM (SELECT name, ts FROM bsm_segby ORDER BY name, ts) s) z
720+
WHERE pn IS NOT NULL AND (name<pn OR (name=pn AND ts<pt));
721+
722+
-- Now use minmax index on non-leading column "ts"
723+
drop index _timescaledb_internal.compressed_index_minmax_firstlast;
724+
create index compressed_index_minmax_minmax on :chunk (grp, _ts_meta_min_1, _ts_meta_max_1, _ts_meta_min_2, _ts_meta_max_2);
725+
726+
-- Remove firstlast index from all columns
727+
update _timescaledb_catalog.compression_settings
728+
set index = '[{"type": "minmax", "column": "name", "source": "orderby"}, {"type": "minmax", "column": "ts", "source": "orderby"}]'
729+
where relid = 'bsm_segby'::regclass;
730+
731+
update _timescaledb_catalog.compression_settings
732+
set index = '[{"type": "minmax", "column": "name", "source": "orderby"}, {"type": "minmax", "column": "ts", "source": "orderby"}]'
733+
where compress_relid = (select format('%I.%I', schema_name, table_name)::regclass AS chunk_regclass from _timescaledb_catalog.chunk
734+
where id = (select compressed_chunk_id from _timescaledb_catalog.chunk
735+
where hypertable_id = (select id from _timescaledb_catalog.hypertable
736+
where table_name = 'bsm_segby') limit 1));
737+
738+
-- Correct result only when cannot use Batch Sorted Merge
739+
set timescaledb.debug_require_batch_sorted_merge = 'forbid';
740+
741+
SELECT count(*) misorder FROM (
742+
SELECT name, ts, lag(name) OVER () pn, lag(ts) OVER () pt
743+
FROM (SELECT name, ts FROM bsm_segby ORDER BY name, ts) s) z
744+
WHERE pn IS NOT NULL AND (name<pn OR (name=pn AND ts<pt));
745+
746+
drop table bsm_segby cascade;
747+
748+
drop table test1 cascade;
749+
drop table test2 cascade;
750+
drop table test_with_defined_null cascade;
751+
drop table test_costs cascade;
752+
drop table insert_test cascade;
753+
669754
reset timescaledb.debug_require_batch_sorted_merge;
670755

0 commit comments

Comments
 (0)