Skip to content

Forbid Batch Sorted Merge on multikey minmax index #9955

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
natalya-aksman merged 1 commit into from
Jun 3, 2026
+196 −8
Merged

Forbid Batch Sorted Merge on multikey minmax index #9955

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .unreleased/pr_9955
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Fixes: #9955 Fix wrong results when using Batch Sorted Merge with no first-last index on a non-leading order by column
34 changes: 26 additions & 8 deletions tsl/src/nodes/columnar_scan/columnar_scan.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2925,15 +2925,33 @@ match_pathkeys_to_compression_orderby(List *pathkeys, List *chunk_em_exprs,
return false;
}

/* Bail out on BSM if orderby column is nullable,
* as at the moment the minmax metadata we have doesn't include NULLs,
* so it's difficult to use it for null-sensitive ordering.
* But this restriction can be lifted in the future on new type of chunks
* with NULL-handling metadata.
*/
if (for_bsm && !is_var_notnull(compression_info, var))
if (for_bsm)
{
return false;
/* Bail out on Batch Sorted Merge if orderby column is nullable,
* as at the moment the minmax metadata we have doesn't include NULLs,
* so it's difficult to use it for null-sensitive ordering.
* But this restriction can be lifted in the future on new type of chunks
* with NULL-handling metadata.
*/
if (!is_var_notnull(compression_info, var))
Comment thread
natalya-aksman marked this conversation as resolved.
{
return false;
}

/* Bail out on Batch Sorted Merge with multiple order by keys
* if non-leading keys don't use firstlast index.
* Batches can be sorted incorrectly on multikey minmax index,
* for example
* [(1, 20) .. (1, 30), (2,0)...(2,30)] with min(1),(0)
* will be sorted before [(1,1) .. (1,19)] with min(1),(1)

@akuzm akuzm Jun 4, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shouldn't we just continue adding the batches to the heap if there's a tie on leading columns? Disabling it altogether looks too heavy-handed.

@natalya-aksman natalya-aksman Jun 4, 2026 •
edited
Loading

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is only disabled if there are no firstlast indexes on the non-leading columns. No firstlast on leading column is OK, BSM is not disabled in this case, there is a unit test for it.

Shouldn't we just continue adding the batches to the heap if there's a tie on leading columns?

Batch Sorted Merge depends on batches added in correct sort order of their first tuple, adding [(1, 20) .. (1, 30), (2,0)...(2,30)] before [(1,1) .. (1,19)] when we sort on 2 keys may lead to wrong result order as in the issue reproducer.

* but it should be sorted after as (1,20) > (1,1): correct with firstlast index.
*/
if (compressed_pk_index > 1 &&
orderby_sparse_kind(compression_info->settings, orderby_index) !=
ORDERBY_SPARSE_FIRSTLAST)
{
return false;
}
}

bool orderby_desc =
Expand Down
84 changes: 84 additions & 0 deletions tsl/test/expected/compression_sorted_merge.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1365,6 +1365,7 @@ SELECT "time","hin"::text,"model"::text,"block"::text,"message_name"::text,"sign
--------------------------+---------+----------+---------+--------------+-------------+----------------------+---------------------
Wed Jan 01 10:00:00 2020 | hin1111 | model111 | blok111 | message_here | signal1 | 12.34 | 12.34

drop table bugtab cascade;
-- Condition that filter the first tuple of a batch - Issue 5797
CREATE TABLE test (
id bigint,
Expand Down Expand Up @@ -1528,4 +1529,87 @@ SELECT t.dttm FROM test t ORDER BY t.dttm LIMIT 1;
Output: compress_hyper_14_26_chunk._ts_meta_count, compress_hyper_14_26_chunk.otherid, compress_hyper_14_26_chunk.valuefk, compress_hyper_14_26_chunk.otherfk, compress_hyper_14_26_chunk.id, compress_hyper_14_26_chunk._ts_meta_min_1, compress_hyper_14_26_chunk._ts_meta_max_1, compress_hyper_14_26_chunk._ts_meta_v2_first_dttm, compress_hyper_14_26_chunk._ts_meta_v2_last_dttm, compress_hyper_14_26_chunk.dttm, compress_hyper_14_26_chunk.measure

RESET enable_sort;
drop table test cascade;
-- Test issue #9922: wrong sort order for Batch Sorted Merge with multikey minmax index
\c :TEST_DBNAME :ROLE_SUPERUSER
CREATE TABLE bsm_segby(ts timestamptz NOT NULL, grp int NOT NULL, name text NOT NULL);
SELECT table_name FROM create_hypertable('bsm_segby','ts',chunk_time_interval=>interval '100 day');
table_name
------------
bsm_segby

ALTER TABLE bsm_segby SET (timescaledb.compress, timescaledb.compress_segmentby='grp', timescaledb.compress_orderby='name asc, ts asc');
INSERT INTO bsm_segby
SELECT '2024-08-01'::timestamptz + g*interval '1 min', g%4, 'k'||lpad(((g*5)%17)::text,3,'0')
FROM generate_series(0,4379) g;
SELECT count(compress_chunk(c)) FROM show_chunks('bsm_segby') c;
count
-------
1

select schema_name || '.' || table_name chunk from _timescaledb_catalog.chunk
where id = (select compressed_chunk_id from _timescaledb_catalog.chunk
where hypertable_id = (select id from _timescaledb_catalog.hypertable
where table_name = 'bsm_segby') limit 1)
\gset
-- Firstlast index is used with Batch Sorted Merge: correct result
set timescaledb.debug_require_batch_sorted_merge = 'force';
SELECT count(*) misorder FROM (
SELECT name, ts, lag(name) OVER () pn, lag(ts) OVER () pt
FROM (SELECT name, ts FROM bsm_segby ORDER BY name, ts) s) z
WHERE pn IS NOT NULL AND (name<pn OR (name=pn AND ts<pt));
misorder
----------
0

-- Remove firstlast index from leading column: still OK to use Batch Sorted Merge
update _timescaledb_catalog.compression_settings
set index = '[{"type": "minmax", "column": "name", "source": "orderby"}, {"type": "minmax", "column": "ts", "source": "orderby"}, {"type": "firstlast", "column": "ts", "source": "orderby"}]'
where relid = 'bsm_segby'::regclass;
update _timescaledb_catalog.compression_settings
set index = '[{"type": "minmax", "column": "name", "source": "orderby"}, {"type": "minmax", "column": "ts", "source": "orderby"}, {"type": "firstlast", "column": "ts", "source": "orderby"}]'
where compress_relid = (select format('%I.%I', schema_name, table_name)::regclass AS chunk_regclass from _timescaledb_catalog.chunk
where id = (select compressed_chunk_id from _timescaledb_catalog.chunk
where hypertable_id = (select id from _timescaledb_catalog.hypertable
where table_name = 'bsm_segby') limit 1));
create index compressed_index_minmax_firstlast on :chunk (grp, _ts_meta_min_1, _ts_meta_max_1, _ts_meta_v2_first_ts, _ts_meta_v2_last_ts);
-- Correct result with Batch Sorted Merge
set timescaledb.debug_require_batch_sorted_merge = 'force';
SELECT count(*) misorder FROM (
SELECT name, ts, lag(name) OVER () pn, lag(ts) OVER () pt
FROM (SELECT name, ts FROM bsm_segby ORDER BY name, ts) s) z
WHERE pn IS NOT NULL AND (name<pn OR (name=pn AND ts<pt));
misorder
----------
0

-- Now use minmax index on non-leading column "ts"
drop index _timescaledb_internal.compressed_index_minmax_firstlast;
create index compressed_index_minmax_minmax on :chunk (grp, _ts_meta_min_1, _ts_meta_max_1, _ts_meta_min_2, _ts_meta_max_2);
-- Remove firstlast index from all columns
update _timescaledb_catalog.compression_settings
set index = '[{"type": "minmax", "column": "name", "source": "orderby"}, {"type": "minmax", "column": "ts", "source": "orderby"}]'
where relid = 'bsm_segby'::regclass;
update _timescaledb_catalog.compression_settings
set index = '[{"type": "minmax", "column": "name", "source": "orderby"}, {"type": "minmax", "column": "ts", "source": "orderby"}]'
where compress_relid = (select format('%I.%I', schema_name, table_name)::regclass AS chunk_regclass from _timescaledb_catalog.chunk
where id = (select compressed_chunk_id from _timescaledb_catalog.chunk
where hypertable_id = (select id from _timescaledb_catalog.hypertable
where table_name = 'bsm_segby') limit 1));
-- Correct result only when cannot use Batch Sorted Merge
set timescaledb.debug_require_batch_sorted_merge = 'forbid';
SELECT count(*) misorder FROM (
SELECT name, ts, lag(name) OVER () pn, lag(ts) OVER () pt
FROM (SELECT name, ts FROM bsm_segby ORDER BY name, ts) s) z
WHERE pn IS NOT NULL AND (name<pn OR (name=pn AND ts<pt));
misorder
----------
0

drop table bsm_segby cascade;
drop table test1 cascade;
drop table test2 cascade;
drop table test_with_defined_null cascade;
drop table test_costs cascade;
drop table insert_test cascade;
reset timescaledb.debug_require_batch_sorted_merge;
85 changes: 85 additions & 0 deletions tsl/test/sql/compression_sorted_merge.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -555,6 +555,8 @@ SELECT "time","hin"::text,"model"::text,"block"::text,"message_name"::text,"sign

SELECT "time","hin"::text,"model"::text,"block"::text,"message_name"::text,"signal_name"::text,"signal_numeric_value","signal_string_value"::text FROM bugtab ORDER BY "time" DESC;

drop table bugtab cascade;

-- Condition that filter the first tuple of a batch - Issue 5797
CREATE TABLE test (
id bigint,
Expand Down Expand Up @@ -666,5 +668,88 @@ set timescaledb.debug_require_batch_sorted_merge = 'require';
SELECT t.dttm FROM test t ORDER BY t.dttm LIMIT 1;
RESET enable_sort;

drop table test cascade;

-- Test issue #9922: wrong sort order for Batch Sorted Merge with multikey minmax index
\c :TEST_DBNAME :ROLE_SUPERUSER

CREATE TABLE bsm_segby(ts timestamptz NOT NULL, grp int NOT NULL, name text NOT NULL);
SELECT table_name FROM create_hypertable('bsm_segby','ts',chunk_time_interval=>interval '100 day');
ALTER TABLE bsm_segby SET (timescaledb.compress, timescaledb.compress_segmentby='grp', timescaledb.compress_orderby='name asc, ts asc');

INSERT INTO bsm_segby
SELECT '2024-08-01'::timestamptz + g*interval '1 min', g%4, 'k'||lpad(((g*5)%17)::text,3,'0')
FROM generate_series(0,4379) g;

SELECT count(compress_chunk(c)) FROM show_chunks('bsm_segby') c;

select schema_name || '.' || table_name chunk from _timescaledb_catalog.chunk
where id = (select compressed_chunk_id from _timescaledb_catalog.chunk
where hypertable_id = (select id from _timescaledb_catalog.hypertable
where table_name = 'bsm_segby') limit 1)
\gset

-- Firstlast index is used with Batch Sorted Merge: correct result
set timescaledb.debug_require_batch_sorted_merge = 'force';

SELECT count(*) misorder FROM (
SELECT name, ts, lag(name) OVER () pn, lag(ts) OVER () pt
FROM (SELECT name, ts FROM bsm_segby ORDER BY name, ts) s) z
WHERE pn IS NOT NULL AND (name<pn OR (name=pn AND ts<pt));

-- Remove firstlast index from leading column: still OK to use Batch Sorted Merge
update _timescaledb_catalog.compression_settings
set index = '[{"type": "minmax", "column": "name", "source": "orderby"}, {"type": "minmax", "column": "ts", "source": "orderby"}, {"type": "firstlast", "column": "ts", "source": "orderby"}]'
where relid = 'bsm_segby'::regclass;

update _timescaledb_catalog.compression_settings
set index = '[{"type": "minmax", "column": "name", "source": "orderby"}, {"type": "minmax", "column": "ts", "source": "orderby"}, {"type": "firstlast", "column": "ts", "source": "orderby"}]'
where compress_relid = (select format('%I.%I', schema_name, table_name)::regclass AS chunk_regclass from _timescaledb_catalog.chunk
where id = (select compressed_chunk_id from _timescaledb_catalog.chunk
where hypertable_id = (select id from _timescaledb_catalog.hypertable
where table_name = 'bsm_segby') limit 1));

create index compressed_index_minmax_firstlast on :chunk (grp, _ts_meta_min_1, _ts_meta_max_1, _ts_meta_v2_first_ts, _ts_meta_v2_last_ts);

-- Correct result with Batch Sorted Merge
set timescaledb.debug_require_batch_sorted_merge = 'force';

SELECT count(*) misorder FROM (
SELECT name, ts, lag(name) OVER () pn, lag(ts) OVER () pt
FROM (SELECT name, ts FROM bsm_segby ORDER BY name, ts) s) z
WHERE pn IS NOT NULL AND (name<pn OR (name=pn AND ts<pt));

-- Now use minmax index on non-leading column "ts"
drop index _timescaledb_internal.compressed_index_minmax_firstlast;
create index compressed_index_minmax_minmax on :chunk (grp, _ts_meta_min_1, _ts_meta_max_1, _ts_meta_min_2, _ts_meta_max_2);

-- Remove firstlast index from all columns
update _timescaledb_catalog.compression_settings
set index = '[{"type": "minmax", "column": "name", "source": "orderby"}, {"type": "minmax", "column": "ts", "source": "orderby"}]'
where relid = 'bsm_segby'::regclass;

update _timescaledb_catalog.compression_settings
set index = '[{"type": "minmax", "column": "name", "source": "orderby"}, {"type": "minmax", "column": "ts", "source": "orderby"}]'
where compress_relid = (select format('%I.%I', schema_name, table_name)::regclass AS chunk_regclass from _timescaledb_catalog.chunk
where id = (select compressed_chunk_id from _timescaledb_catalog.chunk
where hypertable_id = (select id from _timescaledb_catalog.hypertable
where table_name = 'bsm_segby') limit 1));

-- Correct result only when cannot use Batch Sorted Merge
set timescaledb.debug_require_batch_sorted_merge = 'forbid';

SELECT count(*) misorder FROM (
SELECT name, ts, lag(name) OVER () pn, lag(ts) OVER () pt
FROM (SELECT name, ts FROM bsm_segby ORDER BY name, ts) s) z
WHERE pn IS NOT NULL AND (name<pn OR (name=pn AND ts<pt));

drop table bsm_segby cascade;

drop table test1 cascade;
drop table test2 cascade;
drop table test_with_defined_null cascade;
drop table test_costs cascade;
drop table insert_test cascade;

reset timescaledb.debug_require_batch_sorted_merge;

Loading