Skip to content

Commit 9fc21d8

Browse files
antekresicantekresic
committed
Match tuples against leading orderby column
Segmentwise recompression matches new tuples against existing batches by checking each orderby column's metadata range. When a tuple's leading orderby column sat inside an existing batch's range but a later column sat outside it, the matcher would miss the overlap and a new batch was written. This would create overlapping batches on ordered chunks. Making the matching more aggressive by only checking the first orderby match avoids creating these scenarios. Fixes #9731
1 parent cf1132e commit 9fc21d8

4 files changed

Lines changed: 165 additions & 6 deletions

File tree

‎.unreleased/pr_9733‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Fixes: #9731 Avoid creating overlapping batches during recompression for multi orderby configurations
2+
Thanks: @h0rn3t for reporting issue with recompression creating overlapping batches

‎tsl/src/compression/recompress.c‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1059,19 +1059,26 @@ static enum Batch_match_result
10591059
match_tuple_batch(TupleTableSlot *compressed_slot, int num_orderby, ScanKey orderby_scankeys,
10601060
bool *nulls_first)
10611061
{
1062-
ScanKey key;
1063-
for (int i = 0; i < num_orderby; i++)
1062+
/*
1063+
* Only the leading orderby column gives a sound before/after verdict from
1064+
* batch metadata. The min/max for later orderby columns are aggregated
1065+
* over all rows in the batch, not conditional on the leading column, so a
1066+
* tuple whose leading column is in range but whose later column is out of
1067+
* range is interleaved with the batch in multi-column sort order — not
1068+
* strictly before or after it.
1069+
*/
1070+
if (num_orderby >= 1)
10641071
{
1065-
key = &orderby_scankeys[i * 2];
1072+
ScanKey key = &orderby_scankeys[0];
10661073
if (!slot_key_test(compressed_slot, key))
10671074
{
1068-
return handle_null_scan(key->sk_flags, nulls_first[i], Tuple_before);
1075+
return handle_null_scan(key->sk_flags, nulls_first[0], Tuple_before);
10691076
}
10701077

1071-
key = &orderby_scankeys[i * 2 + 1];
1078+
key = &orderby_scankeys[1];
10721079
if (!slot_key_test(compressed_slot, key))
10731080
{
1074-
return handle_null_scan(key->sk_flags, nulls_first[i], Tuple_after);
1081+
return handle_null_scan(key->sk_flags, nulls_first[0], Tuple_after);
10751082
}
10761083
}
10771084

‎tsl/test/expected/recompress_chunk_segmentwise.out‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -763,3 +763,83 @@ EXPLAIN (COSTS OFF) SELECT a, time FROM segwise_unordered WHERE a = 2 ORDER BY a
763763

764764
RESET timescaledb.debug_require_batch_sorted_merge;
765765
DROP TABLE segwise_unordered;
766+
----------------------------------------------------------------------------
767+
-- Issue #9731: segmentwise recompression must not write a new batch whose
768+
-- leading-orderby range is contained inside an existing batch's range.
769+
-- The matcher previously evaluated each orderby column independently and
770+
-- could return Tuple_after based on a non-leading column even when the
771+
-- leading column sat inside the existing batch's range. The result was a
772+
-- new batch shifted up inside the original, which the planner's
773+
-- use_compressed_sort path then concatenated out of order.
774+
----------------------------------------------------------------------------
775+
CREATE TABLE segwise_no_overlap (
776+
ts timestamptz NOT NULL,
777+
series_id bigint NOT NULL,
778+
value int
779+
);
780+
SELECT create_hypertable('segwise_no_overlap', 'ts', chunk_time_interval => interval '30 day');
781+
create_hypertable
782+
----------------------------------
783+
(27,public,segwise_no_overlap,t)
784+
785+
ALTER TABLE segwise_no_overlap SET (
786+
timescaledb.compress,
787+
timescaledb.compress_segmentby = '',
788+
timescaledb.compress_orderby = 'series_id DESC, ts ASC'
789+
);
790+
-- Initial compression: 1000 rows for series_id 1..100, ts 00:01..00:10.
791+
-- Produces a single batch with series_id range [1,100].
792+
INSERT INTO segwise_no_overlap
793+
SELECT '2026-05-01 00:00:00+00'::timestamptz + (t * interval '1 second'), s, s * 100 + t
794+
FROM generate_series(1, 100) s
795+
CROSS JOIN generate_series(1, 10) t;
796+
SELECT compress_chunk(c) FROM show_chunks('segwise_no_overlap') c;
797+
compress_chunk
798+
------------------------------------------
799+
_timescaledb_internal._hyper_27_29_chunk
800+
801+
-- Add rows for series_id = 50 (inside the existing batch's series_id range)
802+
-- with ts outside the existing batch's ts range. Pre-fix, the matcher would
803+
-- decide Tuple_after on ts and write a new batch [50,50] inside [1,100].
804+
INSERT INTO segwise_no_overlap
805+
SELECT '2026-05-01 06:00:00+00'::timestamptz + (t * interval '1 second'), 50, 99000 + t
806+
FROM generate_series(1, 500) t;
807+
-- Triggers segmentwise recompression (chunk is partial).
808+
SELECT compress_chunk(c) FROM show_chunks('segwise_no_overlap') c;
809+
compress_chunk
810+
------------------------------------------
811+
_timescaledb_internal._hyper_27_29_chunk
812+
813+
-- Locate the compressed chunk for the metadata check.
814+
SELECT cc.schema_name || '.' || cc.table_name AS comp_table_segwise
815+
FROM _timescaledb_catalog.chunk uc
816+
JOIN _timescaledb_catalog.chunk cc ON cc.id = uc.compressed_chunk_id
817+
JOIN _timescaledb_catalog.hypertable h ON h.id = uc.hypertable_id
818+
WHERE h.table_name = 'segwise_no_overlap' \gset
819+
-- Resulting batches: must be at most adjacent on series_id, never
820+
-- shifted-up contained. Consecutive pairs in (min DESC, max DESC) order
821+
-- must satisfy prev_min >= curr_max.
822+
SELECT count(*) AS overlap_violations FROM (
823+
SELECT _ts_meta_max_1 AS curr_hi,
824+
lag(_ts_meta_min_1) OVER (
825+
ORDER BY _ts_meta_min_1 DESC, _ts_meta_max_1 DESC
826+
) AS prev_lo
827+
FROM :comp_table_segwise
828+
) t WHERE prev_lo IS NOT NULL AND prev_lo < curr_hi;
829+
overlap_violations
830+
--------------------
831+
0
832+
833+
-- Row-level stream must be monotone DESC on series_id.
834+
SELECT count(*) AS desc_violations FROM (
835+
SELECT series_id, lag(series_id) OVER () AS prev
836+
FROM (
837+
SELECT series_id FROM segwise_no_overlap
838+
ORDER BY series_id DESC, ts ASC
839+
) ordered
840+
) lagged WHERE prev IS NOT NULL AND prev < series_id;
841+
desc_violations
842+
-----------------
843+
0
844+
845+
DROP TABLE segwise_no_overlap CASCADE;

‎tsl/test/sql/recompress_chunk_segmentwise.sql‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -447,3 +447,73 @@ EXPLAIN (COSTS OFF) SELECT a, time FROM segwise_unordered WHERE a = 2 ORDER BY a
447447
RESET timescaledb.debug_require_batch_sorted_merge;
448448

449449
DROP TABLE segwise_unordered;
450+
451+
----------------------------------------------------------------------------
452+
-- Issue #9731: segmentwise recompression must not write a new batch whose
453+
-- leading-orderby range is contained inside an existing batch's range.
454+
-- The matcher previously evaluated each orderby column independently and
455+
-- could return Tuple_after based on a non-leading column even when the
456+
-- leading column sat inside the existing batch's range. The result was a
457+
-- new batch shifted up inside the original, which the planner's
458+
-- use_compressed_sort path then concatenated out of order.
459+
----------------------------------------------------------------------------
460+
CREATE TABLE segwise_no_overlap (
461+
ts timestamptz NOT NULL,
462+
series_id bigint NOT NULL,
463+
value int
464+
);
465+
SELECT create_hypertable('segwise_no_overlap', 'ts', chunk_time_interval => interval '30 day');
466+
467+
ALTER TABLE segwise_no_overlap SET (
468+
timescaledb.compress,
469+
timescaledb.compress_segmentby = '',
470+
timescaledb.compress_orderby = 'series_id DESC, ts ASC'
471+
);
472+
473+
-- Initial compression: 1000 rows for series_id 1..100, ts 00:01..00:10.
474+
-- Produces a single batch with series_id range [1,100].
475+
INSERT INTO segwise_no_overlap
476+
SELECT '2026-05-01 00:00:00+00'::timestamptz + (t * interval '1 second'), s, s * 100 + t
477+
FROM generate_series(1, 100) s
478+
CROSS JOIN generate_series(1, 10) t;
479+
480+
SELECT compress_chunk(c) FROM show_chunks('segwise_no_overlap') c;
481+
482+
-- Add rows for series_id = 50 (inside the existing batch's series_id range)
483+
-- with ts outside the existing batch's ts range. Pre-fix, the matcher would
484+
-- decide Tuple_after on ts and write a new batch [50,50] inside [1,100].
485+
INSERT INTO segwise_no_overlap
486+
SELECT '2026-05-01 06:00:00+00'::timestamptz + (t * interval '1 second'), 50, 99000 + t
487+
FROM generate_series(1, 500) t;
488+
489+
-- Triggers segmentwise recompression (chunk is partial).
490+
SELECT compress_chunk(c) FROM show_chunks('segwise_no_overlap') c;
491+
492+
-- Locate the compressed chunk for the metadata check.
493+
SELECT cc.schema_name || '.' || cc.table_name AS comp_table_segwise
494+
FROM _timescaledb_catalog.chunk uc
495+
JOIN _timescaledb_catalog.chunk cc ON cc.id = uc.compressed_chunk_id
496+
JOIN _timescaledb_catalog.hypertable h ON h.id = uc.hypertable_id
497+
WHERE h.table_name = 'segwise_no_overlap' \gset
498+
499+
-- Resulting batches: must be at most adjacent on series_id, never
500+
-- shifted-up contained. Consecutive pairs in (min DESC, max DESC) order
501+
-- must satisfy prev_min >= curr_max.
502+
SELECT count(*) AS overlap_violations FROM (
503+
SELECT _ts_meta_max_1 AS curr_hi,
504+
lag(_ts_meta_min_1) OVER (
505+
ORDER BY _ts_meta_min_1 DESC, _ts_meta_max_1 DESC
506+
) AS prev_lo
507+
FROM :comp_table_segwise
508+
) t WHERE prev_lo IS NOT NULL AND prev_lo < curr_hi;
509+
510+
-- Row-level stream must be monotone DESC on series_id.
511+
SELECT count(*) AS desc_violations FROM (
512+
SELECT series_id, lag(series_id) OVER () AS prev
513+
FROM (
514+
SELECT series_id FROM segwise_no_overlap
515+
ORDER BY series_id DESC, ts ASC
516+
) ordered
517+
) lagged WHERE prev IS NOT NULL AND prev < series_id;
518+
519+
DROP TABLE segwise_no_overlap CASCADE;

0 commit comments

Comments
 (0)