Skip to content

Commit 87a42d8

Browse files
author
zhaixiangli
committed
[opt](scan) Prune sequence mapping candidate keys
Add an optional two-phase scan for sequence-mapping UNIQUE MOR tables. Indexed positive value predicates collect and intersect candidate keys before the normal MOR merge and residual filter run over exact point keys. Share one immutable point-key payload and schema across rowsets and segments, build monotonic point ranges linearly, and preserve the original FE key ranges as an additional constraint. Bound candidate memory with an adaptive 32 MiB budget and memory reservation fallback. Expose profile counters and cover budget enforcement, shared schemas, exact range construction, cancellation, and composite-key prefix ranges. Closes #66879
1 parent f1cfc38 commit 87a42d8

26 files changed

Lines changed: 1677 additions & 20 deletions

be/src/exec/operator/olap_scan_operator.cpp

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -292,6 +292,38 @@ Status OlapScanLocalState::_init_profile() {
292292
ADD_TIMER_WITH_LEVEL(_segment_profile, "InvertedIndexAnalyzerTime", 1);
293293
_inverted_index_lookup_timer =
294294
ADD_TIMER_WITH_LEVEL(_segment_profile, "InvertedIndexLookupTimer", 1);
295+
_seq_map_candidate_driver_groups_counter =
296+
ADD_COUNTER(_segment_profile, "SeqMapCandidateDriverGroups", TUnit::UNIT);
297+
_seq_map_candidate_driver_predicates_counter =
298+
ADD_COUNTER(_segment_profile, "SeqMapCandidateDriverPredicates", TUnit::UNIT);
299+
_seq_map_candidate_rows_counter =
300+
ADD_COUNTER(_segment_profile, "SeqMapCandidateRows", TUnit::UNIT);
301+
_seq_map_candidate_scan_rows_counter =
302+
ADD_COUNTER(_segment_profile, "SeqMapCandidateScanRows", TUnit::UNIT);
303+
_seq_map_candidate_scan_bytes_counter =
304+
ADD_COUNTER(_segment_profile, "SeqMapCandidateScanBytes", TUnit::BYTES);
305+
_seq_map_candidate_index_filtered_rows_counter =
306+
ADD_COUNTER(_segment_profile, "SeqMapCandidateIndexFilteredRows", TUnit::UNIT);
307+
_seq_map_candidate_index_downgrades_counter =
308+
ADD_COUNTER(_segment_profile, "SeqMapCandidateIndexDowngrades", TUnit::UNIT);
309+
_seq_map_candidate_index_lookup_timer =
310+
ADD_TIMER(_segment_profile, "SeqMapCandidateIndexLookupTime");
311+
_seq_map_candidate_cache_local_bytes_counter =
312+
ADD_COUNTER(_segment_profile, "SeqMapCandidateCacheLocalBytes", TUnit::BYTES);
313+
_seq_map_candidate_cache_remote_bytes_counter =
314+
ADD_COUNTER(_segment_profile, "SeqMapCandidateCacheRemoteBytes", TUnit::BYTES);
315+
_seq_map_candidate_keys_before_intersect_counter =
316+
ADD_COUNTER(_segment_profile, "SeqMapCandidateKeysBeforeIntersect", TUnit::UNIT);
317+
_seq_map_candidate_keys_after_intersect_counter =
318+
ADD_COUNTER(_segment_profile, "SeqMapCandidateKeysAfterIntersect", TUnit::UNIT);
319+
_seq_map_candidate_key_bytes_counter =
320+
ADD_COUNTER(_segment_profile, "SeqMapCandidateKeyBytes", TUnit::BYTES);
321+
_seq_map_candidate_build_timer = ADD_TIMER(_segment_profile, "SeqMapCandidateBuildTime");
322+
_seq_map_point_range_build_timer = ADD_TIMER(_segment_profile, "SeqMapPointRangeBuildTime");
323+
_seq_map_candidate_fallbacks_counter =
324+
ADD_COUNTER(_segment_profile, "SeqMapCandidateFallbacks", TUnit::UNIT);
325+
_seq_map_candidate_pruned_tablets_counter =
326+
ADD_COUNTER(_segment_profile, "SeqMapCandidatePrunedTablets", TUnit::UNIT);
295327

296328
_output_index_result_column_timer = ADD_TIMER(_segment_profile, "OutputIndexResultColumnTime");
297329
_filtered_segment_counter = ADD_COUNTER(_segment_profile, "NumSegmentFiltered", TUnit::UNIT);

be/src/exec/operator/olap_scan_operator.h

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -259,6 +259,23 @@ class OlapScanLocalState final : public ScanLocalState<OlapScanLocalState> {
259259
RuntimeProfile::Counter* _inverted_index_downgrade_count_counter = nullptr;
260260
RuntimeProfile::Counter* _inverted_index_analyzer_timer = nullptr;
261261
RuntimeProfile::Counter* _inverted_index_lookup_timer = nullptr;
262+
RuntimeProfile::Counter* _seq_map_candidate_driver_groups_counter = nullptr;
263+
RuntimeProfile::Counter* _seq_map_candidate_driver_predicates_counter = nullptr;
264+
RuntimeProfile::Counter* _seq_map_candidate_rows_counter = nullptr;
265+
RuntimeProfile::Counter* _seq_map_candidate_scan_rows_counter = nullptr;
266+
RuntimeProfile::Counter* _seq_map_candidate_scan_bytes_counter = nullptr;
267+
RuntimeProfile::Counter* _seq_map_candidate_index_filtered_rows_counter = nullptr;
268+
RuntimeProfile::Counter* _seq_map_candidate_index_downgrades_counter = nullptr;
269+
RuntimeProfile::Counter* _seq_map_candidate_index_lookup_timer = nullptr;
270+
RuntimeProfile::Counter* _seq_map_candidate_cache_local_bytes_counter = nullptr;
271+
RuntimeProfile::Counter* _seq_map_candidate_cache_remote_bytes_counter = nullptr;
272+
RuntimeProfile::Counter* _seq_map_candidate_keys_before_intersect_counter = nullptr;
273+
RuntimeProfile::Counter* _seq_map_candidate_keys_after_intersect_counter = nullptr;
274+
RuntimeProfile::Counter* _seq_map_candidate_key_bytes_counter = nullptr;
275+
RuntimeProfile::Counter* _seq_map_candidate_build_timer = nullptr;
276+
RuntimeProfile::Counter* _seq_map_point_range_build_timer = nullptr;
277+
RuntimeProfile::Counter* _seq_map_candidate_fallbacks_counter = nullptr;
278+
RuntimeProfile::Counter* _seq_map_candidate_pruned_tablets_counter = nullptr;
262279

263280
RuntimeProfile::Counter* _ann_topn_filter_counter = nullptr;
264281
// topn_search_costs = index_load_costs + engine_search_costs + pre_process_costs + post_process_costs

0 commit comments

Comments
 (0)