Skip to content

Commit 2f9881f

Browse files
akuzmakuzm
authored
Constify the query parameters in compressed batch filters for DML (#9917)
Currently we don't do that, and this means DML on compressed hypertable in a prepared statement using a generic plan cannot use the parameter values for skipping the decompression of batches. This leads to excessive decompression. Fix this by evaluating the query parameters. Fixes #9916
1 parent 0dfe7fa commit 2f9881f

6 files changed

Lines changed: 355 additions & 122 deletions

File tree

‎.unreleased/param-eval‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Implements: #9917 Decompress less data in DML on compressed hypertable by accounting for the prepared statement parameters
2+
Thanks: @MaximeEthon for reporting an issue with prepared statement parameters in DML decompression

‎tsl/src/compression/compression_dml.c‎

Lines changed: 11 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -822,6 +822,15 @@ decompress_batches_for_update_delete(ModifyHypertableState *ht_state, Chunk *chu
822822
List *predicates, EState *estate,
823823
bool plan_requires_decompression)
824824
{
825+
/*
826+
* Constify the stable functions and query parameters in the predicates.
827+
* This doesn't evaluate the join parameters (PARAM_EXEC) which are supplied
828+
* differently and would have to be evaluated on every rescan.
829+
*/
830+
PlannerGlobal glob = { .boundParams = estate->es_param_list_info };
831+
PlannerInfo root = { .glob = &glob };
832+
predicates = (List *) estimate_expression_value(&root, (Node *) predicates);
833+
825834
/* process each chunk with its corresponding predicates */
826835

827836
List *heap_filters = NIL;
@@ -1958,15 +1967,6 @@ process_predicates(Chunk *ch, CompressionSettings *settings, List *predicates,
19581967
*num_mem_scankeys = 0;
19591968
List *eq_preds = NIL;
19601969

1961-
/*
1962-
* We dont want to forward boundParams from the execution state here
1963-
* as we dont want to constify join params in the predicates.
1964-
* Constifying JOIN params would not be safe as we don't redo
1965-
* this part in rescan.
1966-
*/
1967-
PlannerGlobal glob = { .boundParams = NULL };
1968-
PlannerInfo root = { .glob = &glob };
1969-
19701970
foreach (lc, predicates)
19711971
{
19721972
Node *node = copyObject(lfirst(lc));
@@ -1991,12 +1991,7 @@ process_predicates(Chunk *ch, CompressionSettings *settings, List *predicates,
19911991

19921992
if (!IsA(expr, Const))
19931993
{
1994-
expr = (Expr *) estimate_expression_value(&root, (Node *) expr);
1995-
1996-
if (!IsA(expr, Const))
1997-
{
1998-
continue;
1999-
}
1994+
continue;
20001995
}
20011996

20021997
arg_value = castNode(Const, expr);
@@ -2196,11 +2191,7 @@ process_predicates(Chunk *ch, CompressionSettings *settings, List *predicates,
21962191

21972192
if (!IsA(expr, Const))
21982193
{
2199-
expr = (Expr *) estimate_expression_value(&root, (Node *) expr);
2200-
if (!IsA(expr, Const))
2201-
{
2202-
continue;
2203-
}
2194+
continue;
22042195
}
22052196

22062197
Const *arg_value = castNode(Const, expr);

‎tsl/test/expected/compression_update_delete-16.out‎

Lines changed: 103 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -2851,13 +2851,12 @@ BEGIN; :EXPLAIN DELETE FROM test_pushdown WHERE device = substring(CURRENT_USER,
28512851
--- QUERY PLAN ---
28522852
Custom Scan (ModifyHypertable) (actual rows=0.00 loops=1)
28532853
Batches scanned: 1
2854-
Batches decompressed: 1
2855-
Tuples decompressed: 1
2854+
Batches deleted: 1
28562855
-> Delete on test_pushdown (actual rows=0.00 loops=1)
28572856
Delete on _hyper_39_76_chunk test_pushdown_1
2858-
-> Custom Scan (ChunkAppend) on test_pushdown (actual rows=1.00 loops=1)
2857+
-> Custom Scan (ChunkAppend) on test_pushdown (actual rows=0.00 loops=1)
28592858
Chunks excluded during startup: 0
2860-
-> Seq Scan on _hyper_39_76_chunk test_pushdown_1 (actual rows=1.00 loops=1)
2859+
-> Seq Scan on _hyper_39_76_chunk test_pushdown_1 (actual rows=0.00 loops=1)
28612860
Filter: (device = ("substring"((CURRENT_USER)::text, (length((CURRENT_USER)::text) + 1)) || 'c'::text))
28622861

28632862
-- JOIN tests
@@ -2990,6 +2989,77 @@ BEGIN; :EXPLAIN EXECUTE q1('not here'); ROLLBACK;
29902989
-> Seq Scan on _hyper_39_76_chunk test_pushdown_1 (actual rows=0.00 loops=1)
29912990
Filter: (device = 'not here'::text)
29922991

2992+
-- prepared statement with generic plan (param should be constified)
2993+
SET plan_cache_mode = force_generic_plan;
2994+
BEGIN; :EXPLAIN EXECUTE q1('a'); ROLLBACK;
2995+
--- QUERY PLAN ---
2996+
Custom Scan (ModifyHypertable) (actual rows=0.00 loops=1)
2997+
Batches scanned: 1
2998+
Batches deleted: 1
2999+
-> Delete on test_pushdown (actual rows=0.00 loops=1)
3000+
Delete on _hyper_39_76_chunk test_pushdown_1
3001+
-> Custom Scan (ChunkAppend) on test_pushdown (actual rows=0.00 loops=1)
3002+
Chunks excluded during startup: 0
3003+
-> Seq Scan on _hyper_39_76_chunk test_pushdown_1 (actual rows=0.00 loops=1)
3004+
Filter: (device = $1)
3005+
3006+
BEGIN; :EXPLAIN EXECUTE q1('not here'); ROLLBACK;
3007+
--- QUERY PLAN ---
3008+
Custom Scan (ModifyHypertable) (actual rows=0.00 loops=1)
3009+
-> Delete on test_pushdown (actual rows=0.00 loops=1)
3010+
Delete on _hyper_39_76_chunk test_pushdown_1
3011+
-> Custom Scan (ChunkAppend) on test_pushdown (actual rows=0.00 loops=1)
3012+
Chunks excluded during startup: 0
3013+
-> Seq Scan on _hyper_39_76_chunk test_pushdown_1 (actual rows=0.00 loops=1)
3014+
Filter: (device = $1)
3015+
3016+
RESET plan_cache_mode;
3017+
-- ensure that we're not incorrectly constifying the nestloop join params
3018+
CREATE TABLE nl_param(time timestamptz NOT NULL, device int, PRIMARY KEY(device, time));
3019+
SELECT table_name FROM create_hypertable('nl_param', 'time', chunk_time_interval => INTERVAL '10 years');
3020+
table_name
3021+
------------
3022+
nl_param
3023+
3024+
INSERT INTO nl_param
3025+
SELECT '2020-01-01'::timestamptz + (i || ' seconds')::interval, (i % 100) + 1
3026+
FROM generate_series(1, 50000) i;
3027+
ALTER TABLE nl_param SET (timescaledb.compress, timescaledb.compress_segmentby='device');
3028+
SELECT compress_chunk(show_chunks('nl_param'));
3029+
compress_chunk
3030+
------------------------------------------
3031+
_timescaledb_internal._hyper_41_78_chunk
3032+
3033+
-- Uncompressed rows so PG sees a non-empty heap and considers indexed paths
3034+
INSERT INTO nl_param
3035+
SELECT '2025-01-01'::timestamptz + (i || ' seconds')::interval, (i % 100) + 1
3036+
FROM generate_series(1, 50000) i;
3037+
CREATE TABLE nl_devices(device int);
3038+
INSERT INTO nl_devices VALUES (1);
3039+
ANALYZE nl_param, nl_devices;
3040+
SET enable_hashjoin = off;
3041+
SET enable_mergejoin = off;
3042+
SET enable_seqscan = off;
3043+
BEGIN; :EXPLAIN DELETE FROM nl_param p USING nl_devices d WHERE p.device=d.device; SELECT count(*) FROM nl_param; ROLLBACK;
3044+
--- QUERY PLAN ---
3045+
Custom Scan (ModifyHypertable) (actual rows=0.00 loops=1)
3046+
Batches scanned: 100
3047+
Batches decompressed: 100
3048+
Tuples decompressed: 50000
3049+
-> Delete on nl_param p (actual rows=0.00 loops=1)
3050+
Delete on _hyper_41_78_chunk p_1
3051+
-> Nested Loop (actual rows=1000.00 loops=1)
3052+
-> Seq Scan on nl_devices d (actual rows=1.00 loops=1)
3053+
-> Index Scan using "78_nl_param_pkey" on _hyper_41_78_chunk p_1 (actual rows=1000.00 loops=1)
3054+
Index Cond: (device = d.device)
3055+
3056+
count
3057+
-------
3058+
99000
3059+
3060+
RESET enable_hashjoin;
3061+
RESET enable_mergejoin;
3062+
RESET enable_seqscan;
29933063
-- test arrayop pushdown less than 3 decompressions are expected for successful pushdown
29943064
BEGIN; :EXPLAIN DELETE FROM test_pushdown WHERE device IN ('a','d'); ROLLBACK;
29953065
--- QUERY PLAN ---
@@ -3015,13 +3085,12 @@ BEGIN; :EXPLAIN DELETE FROM test_pushdown WHERE device IN ('a',CURRENT_USER); RO
30153085
--- QUERY PLAN ---
30163086
Custom Scan (ModifyHypertable) (actual rows=0.00 loops=1)
30173087
Batches scanned: 1
3018-
Batches decompressed: 1
3019-
Tuples decompressed: 1
3088+
Batches deleted: 1
30203089
-> Delete on test_pushdown (actual rows=0.00 loops=1)
30213090
Delete on _hyper_39_76_chunk test_pushdown_1
3022-
-> Custom Scan (ChunkAppend) on test_pushdown (actual rows=1.00 loops=1)
3091+
-> Custom Scan (ChunkAppend) on test_pushdown (actual rows=0.00 loops=1)
30233092
Chunks excluded during startup: 0
3024-
-> Seq Scan on _hyper_39_76_chunk test_pushdown_1 (actual rows=1.00 loops=1)
3093+
-> Seq Scan on _hyper_39_76_chunk test_pushdown_1 (actual rows=0.00 loops=1)
30253094
Filter: (device = ANY (ARRAY['a'::text, (CURRENT_USER)::text]))
30263095

30273096
-- arroyop pushdown only works for segmentby columns atm so 3 decompressions are expected for now
@@ -3129,7 +3198,7 @@ CREATE TABLE update_trigger_test (
31293198
SELECT create_hypertable('update_trigger_test', 'effective_date_time');
31303199
create_hypertable
31313200
-----------------------------------
3132-
(41,public,update_trigger_test,t)
3201+
(43,public,update_trigger_test,t)
31333202

31343203
CREATE OR REPLACE FUNCTION update_modified_at_test()
31353204
RETURNS TRIGGER
@@ -3155,7 +3224,7 @@ ALTER TABLE update_trigger_test SET (timescaledb.compress);
31553224
SELECT compress_chunk(show_chunks('update_trigger_test'));
31563225
compress_chunk
31573226
------------------------------------------
3158-
_timescaledb_internal._hyper_41_78_chunk
3227+
_timescaledb_internal._hyper_43_80_chunk
31593228

31603229
BEGIN;
31613230
UPDATE update_trigger_test SET measurement = measurement + 2
@@ -3165,7 +3234,7 @@ ROLLBACK;
31653234
SELECT decompress_chunk(show_chunks('update_trigger_test'));
31663235
decompress_chunk
31673236
------------------------------------------
3168-
_timescaledb_internal._hyper_41_78_chunk
3237+
_timescaledb_internal._hyper_43_80_chunk
31693238

31703239
ALTER TABLE update_trigger_test SET (timescaledb.compress, timescaledb.compress_segmentby='entity_id');
31713240
NOTICE: updated compression settings will only apply to future compressions
@@ -3174,7 +3243,7 @@ HINT: Use compress_chunk(chunk, recompress => true) to recompress.
31743243
SELECT compress_chunk(show_chunks('update_trigger_test'));
31753244
compress_chunk
31763245
------------------------------------------
3177-
_timescaledb_internal._hyper_41_78_chunk
3246+
_timescaledb_internal._hyper_43_80_chunk
31783247

31793248
BEGIN;
31803249
UPDATE update_trigger_test SET measurement = measurement + 2
@@ -3204,13 +3273,13 @@ BEGIN
32043273
Batches scanned: 3
32053274
Batches deleted: 3
32063275
-> Delete on delete_counter (actual rows=0.00 loops=1)
3207-
Delete on _hyper_43_81_chunk delete_counter_1
3208-
Delete on _hyper_43_82_chunk delete_counter_2
3209-
Delete on _hyper_43_85_chunk delete_counter_3
3276+
Delete on _hyper_45_83_chunk delete_counter_1
3277+
Delete on _hyper_45_84_chunk delete_counter_2
3278+
Delete on _hyper_45_87_chunk delete_counter_3
32103279
-> Append (actual rows=246.00 loops=1)
3211-
-> Seq Scan on _hyper_43_81_chunk delete_counter_1 (actual rows=0.00 loops=1)
3212-
-> Seq Scan on _hyper_43_82_chunk delete_counter_2 (actual rows=123.00 loops=1)
3213-
-> Seq Scan on _hyper_43_85_chunk delete_counter_3 (actual rows=123.00 loops=1)
3280+
-> Seq Scan on _hyper_45_83_chunk delete_counter_1 (actual rows=0.00 loops=1)
3281+
-> Seq Scan on _hyper_45_84_chunk delete_counter_2 (actual rows=123.00 loops=1)
3282+
-> Seq Scan on _hyper_45_87_chunk delete_counter_3 (actual rows=123.00 loops=1)
32143283

32153284
ROLLBACK
32163285
BEGIN; DELETE FROM delete_counter; ROLLBACK;
@@ -3225,16 +3294,16 @@ BEGIN
32253294
Batches decompressed: 3
32263295
Tuples decompressed: 369
32273296
-> Delete on delete_counter (actual rows=0.00 loops=1)
3228-
Delete on _hyper_43_81_chunk delete_counter_1
3229-
Delete on _hyper_43_82_chunk delete_counter_2
3230-
Delete on _hyper_43_85_chunk delete_counter_3
3297+
Delete on _hyper_45_83_chunk delete_counter_1
3298+
Delete on _hyper_45_84_chunk delete_counter_2
3299+
Delete on _hyper_45_87_chunk delete_counter_3
32313300
-> Custom Scan (ChunkAppend) on delete_counter (actual rows=615.00 loops=1)
32323301
Chunks excluded during startup: 0
3233-
-> Seq Scan on _hyper_43_81_chunk delete_counter_1 (actual rows=246.00 loops=1)
3302+
-> Seq Scan on _hyper_45_83_chunk delete_counter_1 (actual rows=246.00 loops=1)
32343303
Filter: (random() < '1'::double precision)
3235-
-> Seq Scan on _hyper_43_82_chunk delete_counter_2 (actual rows=246.00 loops=1)
3304+
-> Seq Scan on _hyper_45_84_chunk delete_counter_2 (actual rows=246.00 loops=1)
32363305
Filter: (random() < '1'::double precision)
3237-
-> Seq Scan on _hyper_43_85_chunk delete_counter_3 (actual rows=123.00 loops=1)
3306+
-> Seq Scan on _hyper_45_87_chunk delete_counter_3 (actual rows=123.00 loops=1)
32383307
Filter: (random() < '1'::double precision)
32393308

32403309
ROLLBACK
@@ -3265,11 +3334,11 @@ EXPLAIN (analyze,costs off, timing off,summary off, buffers off) DELETE FROM cag
32653334
Batches scanned: 2
32663335
Batches deleted: 2
32673336
-> Delete on cagg_inval (actual rows=0.00 loops=1)
3268-
Delete on _hyper_45_86_chunk cagg_inval_1
3269-
Delete on _hyper_45_87_chunk cagg_inval_2
3337+
Delete on _hyper_47_88_chunk cagg_inval_1
3338+
Delete on _hyper_47_89_chunk cagg_inval_2
32703339
-> Append (actual rows=0.00 loops=1)
3271-
-> Seq Scan on _hyper_45_86_chunk cagg_inval_1 (actual rows=0.00 loops=1)
3272-
-> Seq Scan on _hyper_45_87_chunk cagg_inval_2 (actual rows=0.00 loops=1)
3340+
-> Seq Scan on _hyper_47_88_chunk cagg_inval_1 (actual rows=0.00 loops=1)
3341+
-> Seq Scan on _hyper_47_89_chunk cagg_inval_2 (actual rows=0.00 loops=1)
32733342

32743343
ROLLBACK;
32753344
ROLLBACK
@@ -3284,18 +3353,18 @@ EXPLAIN (analyze,costs off, timing off,summary off, buffers off) DELETE FROM cag
32843353
Batches scanned: 2
32853354
Batches deleted: 2
32863355
-> Delete on cagg_inval (actual rows=0.00 loops=1)
3287-
Delete on _hyper_45_86_chunk cagg_inval_1
3288-
Delete on _hyper_45_87_chunk cagg_inval_2
3356+
Delete on _hyper_47_88_chunk cagg_inval_1
3357+
Delete on _hyper_47_89_chunk cagg_inval_2
32893358
-> Append (actual rows=0.00 loops=1)
3290-
-> Seq Scan on _hyper_45_86_chunk cagg_inval_1 (actual rows=0.00 loops=1)
3291-
-> Seq Scan on _hyper_45_87_chunk cagg_inval_2 (actual rows=0.00 loops=1)
3359+
-> Seq Scan on _hyper_47_88_chunk cagg_inval_1 (actual rows=0.00 loops=1)
3360+
-> Seq Scan on _hyper_47_89_chunk cagg_inval_2 (actual rows=0.00 loops=1)
32923361

32933362
-- should have invalidation entry from the direct batch delete
32943363
SELECT * FROM _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log ORDER BY 1,2;
32953364
hypertable_id | lowest_modified_value | greatest_modified_value
32963365
---------------+-----------------------+-------------------------
3297-
45 | 1577865600000000 | 1577865600000000
3298-
45 | 1609488000000000 | 1609488000000000
3366+
47 | 1577865600000000 | 1577865600000000
3367+
47 | 1609488000000000 | 1609488000000000
32993368

33003369
-- direct batch delete with time column as segmentby. Segmentby columns don't
33013370
-- have _ts_meta_min/max, so the invalidation context must read the segmentby

0 commit comments

Comments
 (0)