Skip to content

Commit 069a94d

Browse files
svenklemmsvenklemm
committed
Add DeferredChunkScan custom scan
Add a new custom scan node for LIMIT queries that leaves the hypertable unexpanded during initial planning and defers chunk enumeration and scanning to execution time. This gives huge speedup to LIMIT queries on hypertables with many chunks. Warm planning time (ms), median of 15, ORDER BY ts LIMIT 1 ┌────────────────────────────┬───────┬───────┬───────┬───────┬───────┬───────┬────────┐ │ config │ 1 │ 10 │ 50 │ 100 │ 1,000 │ 5,000 │ 10,000 │ ├────────────────────────────┼───────┼───────┼───────┼───────┼───────┼───────┼────────┤ │ uncompr, expanded │ 0.010 │ 0.037 │ 0.173 │ 0.372 │ 4.39 │ 35.9 │ 84.3 │ ├────────────────────────────┼───────┼───────┼───────┼───────┼───────┼───────┼────────┤ │ uncompr, DeferredChunkScan │ 0.003 │ 0.003 │ 0.003 │ 0.002 │ 0.002 │ 0.002 │ 0.002 │ ├────────────────────────────┼───────┼───────┼───────┼───────┼───────┼───────┼────────┤ │ compr, expanded │ 0.019 │ 0.136 │ 0.636 │ 1.361 │ 27.1 │ 169.9 │ 363.1 │ ├────────────────────────────┼───────┼───────┼───────┼───────┼───────┼───────┼────────┤ │ compr, DeferredChunkScan │ 0.003 │ 0.003 │ 0.003 │ 0.002 │ 0.002 │ 0.003 │ 0.003 │ └────────────────────────────┴───────┴───────┴───────┴───────┴───────┴───────┴────────┘ Warm execution time (ms), median of 15, ORDER BY ts LIMIT 1 ┌────────────────────────────┬───────┬───────┬───────┬───────┬────────┬───────┬────────┐ │ config │ 1 │ 10 │ 50 │ 100 │ 1,000 │ 5,000 │ 10,000 │ ├────────────────────────────┼───────┼───────┼───────┼───────┼────────┼───────┼────────┤ │ uncompr, expanded │ 0.002 │ 0.005 │ 0.015 │ 0.093 │ 0.88 │ 9.97 │ 23.6 │ ├────────────────────────────┼───────┼───────┼───────┼───────┼────────┼───────┼────────┤ │ uncompr, DeferredChunkScan │ 0.021 │ 0.022 │ 0.018 │ 0.017 │ 0.018 │ 0.017 │ 0.018 │ ├────────────────────────────┼───────┼───────┼───────┼───────┼────────┼───────┼────────┤ │ compr, expanded │ 0.004 │ 0.015 │ 0.143 │ 0.301 │ 6.33 │ 42.4 │ 92.3 │ ├────────────────────────────┼───────┼───────┼───────┼───────┼────────┼───────┼────────┤ │ compr, DeferredChunkScan │ 0.037 │ 0.037 │ 0.029 │ 0.029 │ 6.1e-2 │ 0.030 │ 0.031 │ └────────────────────────────┴───────┴───────┴───────┴───────┴────────┴───────┴────────┘
1 parent c9baf4b commit 069a94d

13 files changed

Lines changed: 2434 additions & 3 deletions

File tree

‎src/guc.c‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,7 @@ bool ts_guc_enable_parallel_chunk_append = true;
9595
bool ts_guc_enable_runtime_exclusion = true;
9696
bool ts_guc_enable_constraint_exclusion = true;
9797
bool ts_guc_enable_hypertable_expansion_for_dml = true;
98+
TSDLLEXPORT bool ts_guc_enable_deferredchunkscan = false;
9899
bool ts_guc_enable_qual_propagation = true;
99100
TSDLLEXPORT bool ts_guc_enable_columnar_scan_filter_pushdown = true;
100101
bool ts_guc_enable_qual_filtering = true;
@@ -211,6 +212,8 @@ bool ts_guc_debug_have_int128;
211212
DebugRequireOption ts_guc_debug_require_vector_qual = DRO_Allow;
212213

213214
DebugRequireOption ts_guc_debug_require_vector_agg = DRO_Allow;
215+
216+
DebugRequireOption ts_guc_debug_require_deferred_chunk_scan = DRO_Allow;
214217
#endif
215218

216219
DebugRequireOption ts_guc_debug_require_batch_sorted_merge = DRO_Allow;
@@ -778,6 +781,18 @@ _guc_init(void)
778781
NULL,
779782
NULL);
780783

784+
DefineCustomBoolVariable(MAKE_EXTOPTION("enable_deferredchunkscan"),
785+
"Enable DeferredChunkScan for LIMIT queries",
786+
"Custom scan node for hypertables that iterates chunks at"
787+
"execution instead of expanding every chunk at plan time.",
788+
&ts_guc_enable_deferredchunkscan,
789+
false,
790+
PGC_USERSET,
791+
0,
792+
NULL,
793+
NULL,
794+
NULL);
795+
781796
DefineCustomBoolVariable(MAKE_EXTOPTION("enable_foreign_key_propagation"),
782797
"Enable foreign key propagation",
783798
"Adjust foreign key lookup queries to target whole hypertable",
@@ -1711,6 +1726,22 @@ _guc_init(void)
17111726
/* assign_hook= */ NULL,
17121727
/* show_hook= */ NULL);
17131728

1729+
DefineCustomEnumVariable(/* name= */ MAKE_EXTOPTION("debug_require_deferred_chunk_scan"),
1730+
/* short_desc= */
1731+
"ensure that DeferredChunkScan is used or not",
1732+
/* long_desc= */
1733+
"this is for debugging purposes, to check whether a query uses the "
1734+
"DeferredChunkScan node without depending on version-specific EXPLAIN "
1735+
"output",
1736+
/* valueAddr= */ (int *) &ts_guc_debug_require_deferred_chunk_scan,
1737+
/* bootValue= */ DRO_Allow,
1738+
/* options = */ debug_require_options,
1739+
/* context= */ PGC_USERSET,
1740+
/* flags= */ 0,
1741+
/* check_hook= */ NULL,
1742+
/* assign_hook= */ NULL,
1743+
/* show_hook= */ NULL);
1744+
17141745
#endif
17151746

17161747
/* register feature flags */

‎src/guc.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ extern bool ts_guc_enable_qual_filtering;
2828
extern bool ts_guc_enable_runtime_exclusion;
2929
extern bool ts_guc_enable_constraint_exclusion;
3030
extern bool ts_guc_enable_hypertable_expansion_for_dml;
31+
extern TSDLLEXPORT bool ts_guc_enable_deferredchunkscan;
3132
extern bool ts_guc_enable_cagg_reorder_groupby;
3233
extern TSDLLEXPORT bool ts_guc_enable_cagg_window_functions;
3334
extern TSDLLEXPORT bool ts_guc_skip_cagg_invalidation;
@@ -150,6 +151,8 @@ extern TSDLLEXPORT DebugRequireOption ts_guc_debug_require_vector_qual;
150151

151152
extern TSDLLEXPORT DebugRequireOption ts_guc_debug_require_vector_agg;
152153

154+
extern TSDLLEXPORT DebugRequireOption ts_guc_debug_require_deferred_chunk_scan;
155+
153156
#endif
154157

155158
extern TSDLLEXPORT bool ts_guc_debug_compression_path_info;

‎src/init.c‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ extern void _conn_mock_fini();
5757
#endif
5858

5959
extern void _chunk_append_init();
60+
extern void _deferred_chunk_scan_init(void);
6061

6162
TS_FUNCTION_INFO_V1(ts_post_load_init);
6263

@@ -111,6 +112,7 @@ _PG_init(void)
111112
_planner_init();
112113
_constraint_aware_append_init();
113114
_chunk_append_init();
115+
_deferred_chunk_scan_init();
114116
_event_trigger_init();
115117
_process_utility_init();
116118
_guc_init();

‎src/nodes/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,4 @@ set(SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/modify_hypertable.c
33
target_sources(${PROJECT_NAME} PRIVATE ${SOURCES})
44
add_subdirectory(chunk_append)
55
add_subdirectory(constraint_aware_append)
6+
add_subdirectory(deferred_chunk_scan)

‎src/nodes/deferred_chunk_scan/CMakeLists.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
set(SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/deferred_chunk_scan.c)
2+
target_sources(${PROJECT_NAME} PRIVATE ${SOURCES})

0 commit comments

Comments
 (0)