From 24675705deee49818bc5141398ccf612a71a4883 Mon Sep 17 00:00:00 2001 From: Natalya Aksman Date: Tue, 21 Jul 2026 14:31:29 -0400 Subject: [PATCH] Do not sort batches for Batch sorted merge over unordered chunks if query sort matches compressed sort --- tsl/src/nodes/columnar_scan/columnar_scan.c | 293 ++++--- tsl/src/nodes/columnar_scan/exec.c | 6 +- tsl/src/nodes/columnar_scan/exec.h | 7 + tsl/src/nodes/columnar_scan/planner.c | 139 +--- tsl/test/expected/compress_unordered_sort.out | 6 +- .../compression_sorted_merge_unordered.out | 737 +++++++++++++++++- .../expected/recompress_chunk_segmentwise.out | 6 +- .../compression_sorted_merge_unordered.sql | 228 +++++- 8 files changed, 1168 insertions(+), 254 deletions(-) diff --git a/tsl/src/nodes/columnar_scan/columnar_scan.c b/tsl/src/nodes/columnar_scan/columnar_scan.c index 3a4d462c764..14cb564f700 100644 --- a/tsl/src/nodes/columnar_scan/columnar_scan.c +++ b/tsl/src/nodes/columnar_scan/columnar_scan.c @@ -241,7 +241,7 @@ build_compressed_scan_pathkeys(const SortInfo *sort_info, PlannerInfo *root, Lis * If pathkeys contains non-segmentby columns the rest of the ordering * requirements will be satisfied by ordering by sequence_num. */ - if (sort_info->needs_sequence_num) + if (sort_info->needs_sequence_num || sort_info->use_batch_sorted_merge) { /* TODO: split up legacy sequence number path and non-sequence number path into dedicated * functions. */ @@ -317,60 +317,10 @@ build_compressed_scan_pathkeys(const SortInfo *sort_info, PlannerInfo *root, Lis column_name = get_attname(info->chunk_rte->relid, var->varattno, false); int16 orderby_index = ts_array_position(info->settings->fd.orderby, column_name); Assert(orderby_index != 0); - AttrNumber leading_attno; - AttrNumber trailing_attno; - orderby_sparse_metadata_attnos(info->settings, - info->compressed_rte->relid, - orderby_index, - &leading_attno, - &trailing_attno); + bool orderby_desc = ts_array_get_element_bool(info->settings->fd.orderby_desc, orderby_index); - /* - * Compressed chunk indexes based on firstlast sparse indexes can have two - * orderings. New chunks index them as (first, last); chunks compressed before that - * change index a DESC column as (last, first). Only DESC columns can differ, so for - * those we check the chunk's index and follow whichever order it has. - */ - if (orderby_desc && - orderby_sparse_kind(info->settings, orderby_index) == ORDERBY_SPARSE_FIRSTLAST) - { - orderby_firstlast_metadata_attnos(info->settings, - info->compressed_rte->relid, - orderby_index, - &leading_attno, - &trailing_attno); - - ListCell *index_lc; - foreach (index_lc, info->compressed_rel->indexlist) - { - IndexOptInfo *index = lfirst(index_lc); - int leading_pos = -1; - int trailing_pos = -1; - for (int k = 0; k < index->nkeycolumns; k++) - { - if (index->indexkeys[k] == leading_attno) - { - leading_pos = k; - } - else if (index->indexkeys[k] == trailing_attno) - { - trailing_pos = k; - } - } - if (leading_pos >= 0 && trailing_pos >= 0) - { - if (trailing_pos < leading_pos) - { - AttrNumber tmp = leading_attno; - leading_attno = trailing_attno; - trailing_attno = tmp; - } - break; - } - } - } bool orderby_nullsfirst = ts_array_get_element_bool(info->settings->fd.orderby_nullsfirst, orderby_index); @@ -388,43 +338,151 @@ build_compressed_scan_pathkeys(const SortInfo *sort_info, PlannerInfo *root, Lis nulls_first = orderby_nullsfirst; } - Var *metadata_var = makeVar(info->compressed_rel->relid, - leading_attno, - var->vartype, - var->vartypmod, - var->varcollid, - var->varlevelsup); - Expr *leading_expr = - canonicalize_ec_expression((Expr *) metadata_var, opcintype, collation); - EquivalenceClass *leading_ec = - append_ec_for_metadata_col(root, info, leading_expr, pk, opcintype); - PathKey *leading_pk = make_canonical_pathkey(root, - leading_ec, - pk->pk_opfamily, - strategy, - nulls_first); - required_compressed_pathkeys = lappend(required_compressed_pathkeys, leading_pk); - - metadata_var = makeVar(info->compressed_rel->relid, - trailing_attno, - var->vartype, - var->vartypmod, - var->varcollid, - var->varlevelsup); - Expr *trailing_expr = - canonicalize_ec_expression((Expr *) metadata_var, opcintype, collation); - EquivalenceClass *trailing_ec = - append_ec_for_metadata_col(root, info, trailing_expr, pk, opcintype); - PathKey *trailing_pk = make_canonical_pathkey(root, - trailing_ec, - pk->pk_opfamily, - strategy, - nulls_first); - - required_compressed_pathkeys = lappend(required_compressed_pathkeys, trailing_pk); + /* For Batch sorted merge we need to sort on specially chosen leading attribute for + * each pathkey */ + if (sort_info->use_batch_sorted_merge) + { + Oid sortop = + get_opfamily_member(pk->pk_opfamily, opcintype, opcintype, pk->pk_cmptype); + Oid opfamily, optype; + CompareType bsm_strategy; + if (!get_ordering_op_properties(sortop, &opfamily, &optype, &bsm_strategy)) + { + elog(ERROR, "operator %u is not a valid ordering operator", sortop); + } + Assert(bsm_strategy == BTLessStrategyNumber || + bsm_strategy == BTGreaterStrategyNumber); + char *leading_name; + char *trailing_name; + orderby_sparse_metadata_names(info->settings, + orderby_index, + &leading_name, + &trailing_name); + char *meta_col_name = + strategy == BTLessStrategyNumber ? leading_name : trailing_name; + + AttrNumber attr_position = + get_attnum(info->compressed_rte->relid, meta_col_name); + + if (attr_position == InvalidAttrNumber) + { + elog(ERROR, "couldn't find metadata column \"%s\"", meta_col_name); + } + Var *metadata_var = makeVar(info->compressed_rel->relid, + attr_position, + var->vartype, + var->vartypmod, + var->varcollid, + var->varlevelsup); + Expr *leading_expr = + canonicalize_ec_expression((Expr *) metadata_var, opcintype, collation); + EquivalenceClass *leading_ec = + append_ec_for_metadata_col(root, info, leading_expr, pk, opcintype); + PathKey *leading_pk = make_canonical_pathkey(root, + leading_ec, + pk->pk_opfamily, + strategy, + nulls_first); + required_compressed_pathkeys = + lappend(required_compressed_pathkeys, leading_pk); + } + /* Need to sort on compressed pathkeys matching compressed indexscan order */ + else + { + AttrNumber leading_attno; + AttrNumber trailing_attno; + orderby_sparse_metadata_attnos(info->settings, + info->compressed_rte->relid, + orderby_index, + &leading_attno, + &trailing_attno); + /* + * Compressed chunk indexes based on firstlast sparse indexes can have two + * orderings. New chunks index them as (first, last); chunks compressed before + * that change index a DESC column as (last, first). Only DESC columns can + * differ, so for those we check the chunk's index and follow whichever order it + * has. + */ + if (orderby_desc && orderby_sparse_kind(info->settings, orderby_index) == + ORDERBY_SPARSE_FIRSTLAST) + { + orderby_firstlast_metadata_attnos(info->settings, + info->compressed_rte->relid, + orderby_index, + &leading_attno, + &trailing_attno); + + ListCell *index_lc; + foreach (index_lc, info->compressed_rel->indexlist) + { + IndexOptInfo *index = lfirst(index_lc); + int leading_pos = -1; + int trailing_pos = -1; + for (int k = 0; k < index->nkeycolumns; k++) + { + if (index->indexkeys[k] == leading_attno) + { + leading_pos = k; + } + else if (index->indexkeys[k] == trailing_attno) + { + trailing_pos = k; + } + } + if (leading_pos >= 0 && trailing_pos >= 0) + { + if (trailing_pos < leading_pos) + { + AttrNumber tmp = leading_attno; + leading_attno = trailing_attno; + trailing_attno = tmp; + } + break; + } + } + } + Var *metadata_var; + metadata_var = makeVar(info->compressed_rel->relid, + leading_attno, + var->vartype, + var->vartypmod, + var->varcollid, + var->varlevelsup); + Expr *leading_expr = + canonicalize_ec_expression((Expr *) metadata_var, opcintype, collation); + EquivalenceClass *leading_ec = + append_ec_for_metadata_col(root, info, leading_expr, pk, opcintype); + PathKey *leading_pk = make_canonical_pathkey(root, + leading_ec, + pk->pk_opfamily, + strategy, + nulls_first); + required_compressed_pathkeys = + lappend(required_compressed_pathkeys, leading_pk); + + metadata_var = makeVar(info->compressed_rel->relid, + trailing_attno, + var->vartype, + var->vartypmod, + var->varcollid, + var->varlevelsup); + Expr *trailing_expr = + canonicalize_ec_expression((Expr *) metadata_var, opcintype, collation); + EquivalenceClass *trailing_ec = + append_ec_for_metadata_col(root, info, trailing_expr, pk, opcintype); + PathKey *trailing_pk = make_canonical_pathkey(root, + trailing_ec, + pk->pk_opfamily, + strategy, + nulls_first); + + required_compressed_pathkeys = + lappend(required_compressed_pathkeys, trailing_pk); + } } } } + return required_compressed_pathkeys; } @@ -978,27 +1036,45 @@ cost_batch_sorted_merge(PlannerInfo *root, const CompressionInfo *compression_in { Path sort_path; /* dummy for result of cost_sort */ - /* - * Don't disable the compressed batch sorted merge plan with the enable_sort - * GUC. We have a separate GUC for it, and this way you can try to force the - * batch sorted merge plan by disabling sort. - */ - const bool old_enable_sort = enable_sort; - enable_sort = true; - cost_sort(&sort_path, - root, - dcpath->required_compressed_pathkeys, + /* We are utilizing compressed sort for batch sorted merge: do not need extra sort */ + if (dcpath->required_compressed_pathkeys && + pathkeys_contained_in(dcpath->required_compressed_pathkeys, compressed_path->pathkeys)) + { + sort_path.rows = compressed_path->rows; + sort_path.startup_cost = compressed_path->startup_cost; + sort_path.total_cost = compressed_path->total_cost; #if PG18_GE - compressed_path->disabled_nodes, + /* PG18 changes the way we handle disabled nodes so we + * need to take those into account as well. + * + * https://github.com/postgres/postgres/commit/e2225346 + */ + sort_path.disabled_nodes = compressed_path->disabled_nodes; #endif - compressed_path->total_cost, - compressed_path->rows, - compressed_path->pathtarget->width, - 0.0, - work_mem, - -1); - enable_sort = old_enable_sort; - + } + else + { + /* + * Don't disable the compressed batch sorted merge plan with the enable_sort + * GUC. We have a separate GUC for it, and this way you can try to force the + * batch sorted merge plan by disabling sort. + */ + const bool old_enable_sort = enable_sort; + enable_sort = true; + cost_sort(&sort_path, + root, + dcpath->required_compressed_pathkeys, +#if PG18_GE + compressed_path->disabled_nodes, +#endif + compressed_path->total_cost, + compressed_path->rows, + compressed_path->pathtarget->width, + 0.0, + work_mem, + -1); + enable_sort = old_enable_sort; + } /* * In compressed batch sorted merge, for each distinct segmentby value we * have to keep the corresponding latest batch open. Estimate the number of @@ -1277,7 +1353,7 @@ ts_columnar_scan_generate_paths(PlannerInfo *root, RelOptInfo *chunk_rel, const chunk_rel, sort_info.needs_sequence_num); - if (sort_info.use_compressed_sort) + if (sort_info.use_compressed_sort || sort_info.use_batch_sorted_merge) { sort_info.required_compressed_pathkeys = build_compressed_scan_pathkeys(&sort_info, @@ -1492,6 +1568,8 @@ build_on_single_compressed_path(PlannerInfo *root, const Chunk *chunk, RelOptInf * query here. */ path_copy->custom_path.path.pathkeys = sort_info->decompressed_sort_pathkeys; + path_copy->required_compressed_pathkeys = sort_info->required_compressed_pathkeys; + cost_batch_sorted_merge(root, compression_info, path_copy, compressed_path); if (ts_guc_debug_require_batch_sorted_merge == DRO_Force) @@ -2646,7 +2724,8 @@ create_compressed_scan_paths(PlannerInfo *root, RelOptInfo *compressed_rel, } } - if (sort_info->use_compressed_sort) + /* We can use sorted input before decompression in both cases */ + if (sort_info->use_compressed_sort || sort_info->use_batch_sorted_merge) { /* * If we can push down sort below decompression we temporarily switch diff --git a/tsl/src/nodes/columnar_scan/exec.c b/tsl/src/nodes/columnar_scan/exec.c index 8dd3ee4dc03..38e4685739c 100644 --- a/tsl/src/nodes/columnar_scan/exec.c +++ b/tsl/src/nodes/columnar_scan/exec.c @@ -98,6 +98,7 @@ columnar_scan_state_create(CustomScan *cscan) Assert(list_length(chunk_state->decompression_map) == list_length(chunk_state->is_segmentby_column)); + chunk_state->done_fetching_batches = false; return (Node *) chunk_state; } @@ -201,6 +202,7 @@ columnar_scan_begin(CustomScanState *node, EState *estate, int eflags) Plan *compressed_scan = linitial(cscan->custom_plans); Assert(list_length(cscan->custom_plans) == 1); + chunk_state->done_fetching_batches = false; ts_stats_compression_acc_init(&dcontext->observ_acc); PlanState *ps = &node->ss.ps; @@ -451,12 +453,13 @@ columnar_scan_exec_impl(ColumnarScanState *chunk_state, const BatchQueueFunction bqfuncs->pop(bq, dcontext); - while (bqfuncs->needs_next_batch(bq)) + while (!chunk_state->done_fetching_batches && bqfuncs->needs_next_batch(bq)) { TupleTableSlot *subslot = ExecProcNode(linitial(chunk_state->csstate.custom_ps)); if (TupIsNull(subslot)) { /* Won't have more compressed tuples. */ + chunk_state->done_fetching_batches = true; break; } @@ -491,6 +494,7 @@ static void columnar_scan_rescan(CustomScanState *node) { ColumnarScanState *chunk_state = (ColumnarScanState *) node; + chunk_state->done_fetching_batches = false; BatchQueue *bq = chunk_state->batch_queue; bq->funcs->reset(bq); diff --git a/tsl/src/nodes/columnar_scan/exec.h b/tsl/src/nodes/columnar_scan/exec.h index 3b47bc3cc0d..0d641849205 100644 --- a/tsl/src/nodes/columnar_scan/exec.h +++ b/tsl/src/nodes/columnar_scan/exec.h @@ -42,6 +42,13 @@ typedef struct ColumnarScanState * evaluate to constant false, hence the flag. */ List *vectorized_quals_original; + + /* + * After the compressed scan has ended, we cannot continue scanning + * it, otherwise some scan types will restart. This node can still have more tuples + * to return, so it doesn't just stop when the input has ended. + */ + bool done_fetching_batches; } ColumnarScanState; extern Node *columnar_scan_state_create(CustomScan *cscan); diff --git a/tsl/src/nodes/columnar_scan/planner.c b/tsl/src/nodes/columnar_scan/planner.c index 8c56d60b640..b1fbbd0fde2 100644 --- a/tsl/src/nodes/columnar_scan/planner.c +++ b/tsl/src/nodes/columnar_scan/planner.c @@ -586,39 +586,6 @@ replace_compressed_vars(Node *node, const CompressionInfo *info) return expression_tree_mutator(node, replace_compressed_vars, (void *) info); } -/* - * Find the resno of the given attribute in the provided target list - */ -static AttrNumber -find_attr_pos_in_tlist(List *targetlist, AttrNumber pos) -{ - ListCell *lc; - - Assert(targetlist != NIL); - Assert(pos > 0 && pos != InvalidAttrNumber); - - foreach (lc, targetlist) - { - TargetEntry *target = (TargetEntry *) lfirst(lc); - - if (!IsA(target->expr, Var)) - { - elog(ERROR, "compressed scan targetlist entries must be Vars"); - } - - Var *var = castNode(Var, target->expr); - AttrNumber compressed_attno = var->varattno; - - if (compressed_attno == pos) - { - return target->resno; - } - } - - elog(ERROR, "Unable to locate var %d in targetlist", pos); - pg_unreachable(); -} - static bool is_not_runtime_constant_walker(Node *node, void *context) { @@ -1133,7 +1100,6 @@ columnar_scan_plan_create(PlannerInfo *root, RelOptInfo *rel, CustomPath *path, * paths of the Custom path, so we won't automatically get a physical tlist * here. */ - bool target_list_compressed_is_physical = false; if (compressed_path->pathtype == T_IndexOnlyScan) { compressed_scan->plan.targetlist = ((IndexPath *) compressed_path)->indexinfo->indextlist; @@ -1145,7 +1111,6 @@ columnar_scan_plan_create(PlannerInfo *root, RelOptInfo *rel, CustomPath *path, if (physical_tlist) { compressed_scan->plan.targetlist = physical_tlist; - target_list_compressed_is_physical = true; } } @@ -1189,8 +1154,6 @@ columnar_scan_plan_create(PlannerInfo *root, RelOptInfo *rel, CustomPath *path, * we must match the pathkeys to the decompressed chunk tupdesc. */ - int numsortkeys = list_length(dcpath->custom_path.path.pathkeys); - List *sort_col_idx = NIL; List *sort_ops = NIL; List *sort_collations = NIL; @@ -1227,7 +1190,6 @@ columnar_scan_plan_create(PlannerInfo *root, RelOptInfo *rel, CustomPath *path, { em = lfirst(membercell); #endif - if (em->em_is_const) { continue; @@ -1297,79 +1259,17 @@ columnar_scan_plan_create(PlannerInfo *root, RelOptInfo *rel, CustomPath *path, } sort_options = list_make4(sort_col_idx, sort_ops, sort_collations, sort_nulls); + } - /* - * Build a sort node for the compressed batches. The sort function is - * derived from the sort function of the pathkeys, except that it refers - * to the min and max metadata columns of the batches. We have already - * verified that the pathkeys match the compression order_by, so this - * mapping is possible. - */ - AttrNumber *sortColIdx = palloc(sizeof(AttrNumber) * numsortkeys); - Oid *sortOperators = palloc(sizeof(Oid) * numsortkeys); - Oid *collations = palloc(sizeof(Oid) * numsortkeys); - bool *nullsFirst = palloc(sizeof(bool) * numsortkeys); - for (int i = 0; i < numsortkeys; i++) - { - Oid sortop = list_nth_oid(sort_ops, i); - - /* Find the operator in pg_amop --- failure shouldn't happen */ - Oid opfamily, opcintype; - CompareType strategy; - if (!get_ordering_op_properties(list_nth_oid(sort_ops, i), - &opfamily, - &opcintype, - &strategy)) - { - elog(ERROR, "operator %u is not a valid ordering operator", sortOperators[i]); - } - - /* - * This way to determine the matching metadata column works, because - * we have already verified that the pathkeys match the compression - * orderby. - */ - Assert(strategy == BTLessStrategyNumber || strategy == BTGreaterStrategyNumber); - char *lower_name; - char *upper_name; - orderby_sparse_metadata_names(dcpath->info->settings, i + 1, &lower_name, &upper_name); - char *meta_col_name = strategy == BTLessStrategyNumber ? lower_name : upper_name; - - AttrNumber attr_position = - get_attnum(dcpath->info->compressed_rte->relid, meta_col_name); - - if (attr_position == InvalidAttrNumber) - { - elog(ERROR, "couldn't find metadata column \"%s\"", meta_col_name); - } - - /* - * If the compressed target list is not based on the layout of - * the uncompressed chunk (see comment for physical_tlist above), - * adjust the position of the attribute. - */ - if (target_list_compressed_is_physical) - { - sortColIdx[i] = attr_position; - } - else - { - sortColIdx[i] = - find_attr_pos_in_tlist(compressed_scan->plan.targetlist, attr_position); - } - - sortOperators[i] = sortop; - collations[i] = list_nth_oid(sort_collations, i); - nullsFirst[i] = list_nth_oid(sort_nulls, i); - } - - /* Now build the compressed batches sort node */ - Sort *sort = ts_make_sort((Plan *) compressed_scan, - numsortkeys, - sortColIdx, - sortOperators, - collations, - nullsFirst); + /* + * Add a sort if the compressed scan is not ordered appropriately. + */ + if (!pathkeys_contained_in(dcpath->required_compressed_pathkeys, compressed_path->pathkeys)) + { + List *compressed_pks = dcpath->required_compressed_pathkeys; + Sort *sort = ts_make_sort_from_pathkeys((Plan *) compressed_scan, + compressed_pks, + bms_make_singleton(compressed_scan->scanrelid)); ts_label_sort_with_costsize(root, sort, /* limit_tuples = */ -1.0); @@ -1377,24 +1277,7 @@ columnar_scan_plan_create(PlannerInfo *root, RelOptInfo *rel, CustomPath *path, } else { - /* - * Add a sort if the compressed scan is not ordered appropriately. - */ - if (!pathkeys_contained_in(dcpath->required_compressed_pathkeys, compressed_path->pathkeys)) - { - List *compressed_pks = dcpath->required_compressed_pathkeys; - Sort *sort = ts_make_sort_from_pathkeys((Plan *) compressed_scan, - compressed_pks, - bms_make_singleton(compressed_scan->scanrelid)); - - ts_label_sort_with_costsize(root, sort, /* limit_tuples = */ -1.0); - - decompress_plan->custom_plans = list_make1(sort); - } - else - { - decompress_plan->custom_plans = custom_plans; - } + decompress_plan->custom_plans = custom_plans; } Assert(list_length(custom_plans) == 1); diff --git a/tsl/test/expected/compress_unordered_sort.out b/tsl/test/expected/compress_unordered_sort.out index 7c8408f8412..151c4a31b5d 100644 --- a/tsl/test/expected/compress_unordered_sort.out +++ b/tsl/test/expected/compress_unordered_sort.out @@ -369,10 +369,8 @@ SET timescaledb.debug_require_batch_sorted_merge = 'force'; GroupAggregate Group Key: _hyper_1_1_chunk."time" -> Custom Scan (ColumnarScan) on _hyper_1_1_chunk - -> Sort - Sort Key: _hyper_1_1_chunk_compressed._ts_meta_v2_first_time DESC - -> Index Scan using _hyper_1_1_chunk_compressed_device_sensor__ts_meta_v2_first_idx on _hyper_1_1_chunk_compressed - Index Cond: ((device = 'd1'::text) AND (sensor = 'A'::text)) + -> Index Scan using _hyper_1_1_chunk_compressed_device_sensor__ts_meta_v2_first_idx on _hyper_1_1_chunk_compressed + Index Cond: ((device = 'd1'::text) AND (sensor = 'A'::text)) select time, avg(value) from metrics where device = 'd1' and sensor='A' group by time order by time DESC; time | avg diff --git a/tsl/test/expected/compression_sorted_merge_unordered.out b/tsl/test/expected/compression_sorted_merge_unordered.out index 2e4d4081211..1ad8e071472 100644 --- a/tsl/test/expected/compression_sorted_merge_unordered.out +++ b/tsl/test/expected/compression_sorted_merge_unordered.out @@ -1669,6 +1669,71 @@ drop table t cascade; -------------------------------------- -- Should be optimized (all segmentby columns are pinned to a Const, orderby columns match) SET timescaledb.debug_require_batch_sorted_merge = 'force'; +-- Encourage compressed indexscan as query sort keys may match compressed indexscan order +SET enable_seqscan=0; +SET enable_bitmapscan=0; +:PREFIX +SELECT * FROM test_segby WHERE segby = 1 ORDER BY time DESC; +--- QUERY PLAN --- + Custom Scan (ColumnarScan) on _timescaledb_internal._hyper_4_4_chunk (actual rows=9.00 loops=1) + Output: _hyper_4_4_chunk.segby, _hyper_4_4_chunk."time", _hyper_4_4_chunk.val + Chunk Status: UNORDERED + Batch Sorted Merge: true + Bulk Decompression: false + -> Index Scan using _hyper_4_4_chunk_compressed_segby__ts_meta_v2_first_time__t_idx on _timescaledb_internal._hyper_4_4_chunk_compressed (actual rows=1.00 loops=1) + Output: _hyper_4_4_chunk_compressed._ts_meta_count, _hyper_4_4_chunk_compressed.segby, _hyper_4_4_chunk_compressed._ts_meta_min_1, _hyper_4_4_chunk_compressed._ts_meta_max_1, _hyper_4_4_chunk_compressed._ts_meta_v2_first_time, _hyper_4_4_chunk_compressed._ts_meta_v2_last_time, _hyper_4_4_chunk_compressed."time", _hyper_4_4_chunk_compressed.val + Index Cond: (_hyper_4_4_chunk_compressed.segby = 1) + +:PREFIX +SELECT * FROM test1 WHERE x1 = 1 AND x2 = 2 AND x5 = 0 ORDER BY x1, x2, x5, time DESC; +--- QUERY PLAN --- + Custom Scan (ColumnarScan) on _timescaledb_internal._hyper_1_1_chunk (actual rows=6.00 loops=1) + Output: _hyper_1_1_chunk."time", _hyper_1_1_chunk.x1, _hyper_1_1_chunk.x2, _hyper_1_1_chunk.x3, _hyper_1_1_chunk.x4, _hyper_1_1_chunk.x5, _hyper_1_1_chunk.c1, _hyper_1_1_chunk.c2 + Chunk Status: UNORDERED + Batch Sorted Merge: true + Bulk Decompression: false + -> Index Scan using _hyper_1_1_chunk_compressed_x1_x2_x5__ts_meta_v2_first_time_idx on _timescaledb_internal._hyper_1_1_chunk_compressed (actual rows=1.00 loops=1) + Output: _hyper_1_1_chunk_compressed."time", _hyper_1_1_chunk_compressed._ts_meta_v2_last_time, _hyper_1_1_chunk_compressed._ts_meta_v2_first_time, _hyper_1_1_chunk_compressed.x1, _hyper_1_1_chunk_compressed.x2, _hyper_1_1_chunk_compressed.x3, _hyper_1_1_chunk_compressed._ts_meta_v2_first_x3, _hyper_1_1_chunk_compressed._ts_meta_v2_last_x3, _hyper_1_1_chunk_compressed.x4, _hyper_1_1_chunk_compressed._ts_meta_v2_first_x4, _hyper_1_1_chunk_compressed._ts_meta_v2_last_x4, _hyper_1_1_chunk_compressed.x5, _hyper_1_1_chunk_compressed.c1, _hyper_1_1_chunk_compressed.c2, _hyper_1_1_chunk_compressed._ts_meta_count + Index Cond: ((_hyper_1_1_chunk_compressed.x1 = 1) AND (_hyper_1_1_chunk_compressed.x2 = 2) AND (_hyper_1_1_chunk_compressed.x5 = 0)) + +-- multikey orderby: can't use compressed indexscan order, need explicit sort +:PREFIX +SELECT * FROM test2 WHERE x1 = 1 AND x2 = 2 AND x5 = 0 ORDER BY time ASC, x3 DESC; +--- QUERY PLAN --- + Custom Scan (ColumnarScan) on _timescaledb_internal._hyper_2_2_chunk (actual rows=6.00 loops=1) + Output: _hyper_2_2_chunk."time", _hyper_2_2_chunk.x1, _hyper_2_2_chunk.x2, _hyper_2_2_chunk.x3, _hyper_2_2_chunk.x4, _hyper_2_2_chunk.x5 + Chunk Status: UNORDERED + Batch Sorted Merge: true + Bulk Decompression: false + -> Sort (actual rows=1.00 loops=1) + Output: _hyper_2_2_chunk_compressed._ts_meta_count, _hyper_2_2_chunk_compressed.x1, _hyper_2_2_chunk_compressed.x2, _hyper_2_2_chunk_compressed.x5, _hyper_2_2_chunk_compressed._ts_meta_min_1, _hyper_2_2_chunk_compressed._ts_meta_max_1, _hyper_2_2_chunk_compressed._ts_meta_v2_first_time, _hyper_2_2_chunk_compressed._ts_meta_v2_last_time, _hyper_2_2_chunk_compressed."time", _hyper_2_2_chunk_compressed._ts_meta_min_2, _hyper_2_2_chunk_compressed._ts_meta_max_2, _hyper_2_2_chunk_compressed._ts_meta_v2_first_x3, _hyper_2_2_chunk_compressed._ts_meta_v2_last_x3, _hyper_2_2_chunk_compressed.x3, _hyper_2_2_chunk_compressed._ts_meta_min_3, _hyper_2_2_chunk_compressed._ts_meta_max_3, _hyper_2_2_chunk_compressed._ts_meta_v2_first_x4, _hyper_2_2_chunk_compressed._ts_meta_v2_last_x4, _hyper_2_2_chunk_compressed.x4 + Sort Key: _hyper_2_2_chunk_compressed._ts_meta_v2_first_time, _hyper_2_2_chunk_compressed._ts_meta_v2_first_x3 DESC + Sort Method: quicksort + -> Index Scan using _hyper_2_2_chunk_compressed_x1_x2_x5__ts_meta_v2_first_time_idx on _timescaledb_internal._hyper_2_2_chunk_compressed (actual rows=1.00 loops=1) + Output: _hyper_2_2_chunk_compressed._ts_meta_count, _hyper_2_2_chunk_compressed.x1, _hyper_2_2_chunk_compressed.x2, _hyper_2_2_chunk_compressed.x5, _hyper_2_2_chunk_compressed._ts_meta_min_1, _hyper_2_2_chunk_compressed._ts_meta_max_1, _hyper_2_2_chunk_compressed._ts_meta_v2_first_time, _hyper_2_2_chunk_compressed._ts_meta_v2_last_time, _hyper_2_2_chunk_compressed."time", _hyper_2_2_chunk_compressed._ts_meta_min_2, _hyper_2_2_chunk_compressed._ts_meta_max_2, _hyper_2_2_chunk_compressed._ts_meta_v2_first_x3, _hyper_2_2_chunk_compressed._ts_meta_v2_last_x3, _hyper_2_2_chunk_compressed.x3, _hyper_2_2_chunk_compressed._ts_meta_min_3, _hyper_2_2_chunk_compressed._ts_meta_max_3, _hyper_2_2_chunk_compressed._ts_meta_v2_first_x4, _hyper_2_2_chunk_compressed._ts_meta_v2_last_x4, _hyper_2_2_chunk_compressed.x4 + Index Cond: ((_hyper_2_2_chunk_compressed.x1 = 1) AND (_hyper_2_2_chunk_compressed.x2 = 2) AND (_hyper_2_2_chunk_compressed.x5 = 0)) + +:PREFIX +SELECT * FROM test2 WHERE x1 = 1 AND x2 = 2 AND x5 = 0 ORDER BY x1 DESC, x2 DESC, x5 DESC, time DESC, x3 ASC; +--- QUERY PLAN --- + Custom Scan (ColumnarScan) on _timescaledb_internal._hyper_2_2_chunk (actual rows=6.00 loops=1) + Output: _hyper_2_2_chunk."time", _hyper_2_2_chunk.x1, _hyper_2_2_chunk.x2, _hyper_2_2_chunk.x3, _hyper_2_2_chunk.x4, _hyper_2_2_chunk.x5 + Chunk Status: UNORDERED + Batch Sorted Merge: true + Reverse: true + Bulk Decompression: false + -> Sort (actual rows=1.00 loops=1) + Output: _hyper_2_2_chunk_compressed._ts_meta_count, _hyper_2_2_chunk_compressed.x1, _hyper_2_2_chunk_compressed.x2, _hyper_2_2_chunk_compressed.x5, _hyper_2_2_chunk_compressed._ts_meta_min_1, _hyper_2_2_chunk_compressed._ts_meta_max_1, _hyper_2_2_chunk_compressed._ts_meta_v2_first_time, _hyper_2_2_chunk_compressed._ts_meta_v2_last_time, _hyper_2_2_chunk_compressed."time", _hyper_2_2_chunk_compressed._ts_meta_min_2, _hyper_2_2_chunk_compressed._ts_meta_max_2, _hyper_2_2_chunk_compressed._ts_meta_v2_first_x3, _hyper_2_2_chunk_compressed._ts_meta_v2_last_x3, _hyper_2_2_chunk_compressed.x3, _hyper_2_2_chunk_compressed._ts_meta_min_3, _hyper_2_2_chunk_compressed._ts_meta_max_3, _hyper_2_2_chunk_compressed._ts_meta_v2_first_x4, _hyper_2_2_chunk_compressed._ts_meta_v2_last_x4, _hyper_2_2_chunk_compressed.x4 + Sort Key: _hyper_2_2_chunk_compressed._ts_meta_v2_last_time DESC, _hyper_2_2_chunk_compressed._ts_meta_v2_last_x3 + Sort Method: quicksort + -> Index Scan using _hyper_2_2_chunk_compressed_x1_x2_x5__ts_meta_v2_first_time_idx on _timescaledb_internal._hyper_2_2_chunk_compressed (actual rows=1.00 loops=1) + Output: _hyper_2_2_chunk_compressed._ts_meta_count, _hyper_2_2_chunk_compressed.x1, _hyper_2_2_chunk_compressed.x2, _hyper_2_2_chunk_compressed.x5, _hyper_2_2_chunk_compressed._ts_meta_min_1, _hyper_2_2_chunk_compressed._ts_meta_max_1, _hyper_2_2_chunk_compressed._ts_meta_v2_first_time, _hyper_2_2_chunk_compressed._ts_meta_v2_last_time, _hyper_2_2_chunk_compressed."time", _hyper_2_2_chunk_compressed._ts_meta_min_2, _hyper_2_2_chunk_compressed._ts_meta_max_2, _hyper_2_2_chunk_compressed._ts_meta_v2_first_x3, _hyper_2_2_chunk_compressed._ts_meta_v2_last_x3, _hyper_2_2_chunk_compressed.x3, _hyper_2_2_chunk_compressed._ts_meta_min_3, _hyper_2_2_chunk_compressed._ts_meta_max_3, _hyper_2_2_chunk_compressed._ts_meta_v2_first_x4, _hyper_2_2_chunk_compressed._ts_meta_v2_last_x4, _hyper_2_2_chunk_compressed.x4 + Index Cond: ((_hyper_2_2_chunk_compressed.x1 = 1) AND (_hyper_2_2_chunk_compressed.x2 = 2) AND (_hyper_2_2_chunk_compressed.x5 = 0)) + +-- We should choose Batch sorted merge with explicit sort on the leading orderby metadata column +SET enable_indexscan=0; +SET enable_bitmapscan=0; +SET enable_seqscan=1; :PREFIX SELECT * FROM test_segby WHERE segby = 1 ORDER BY time DESC; --- QUERY PLAN --- @@ -1681,9 +1746,10 @@ SELECT * FROM test_segby WHERE segby = 1 ORDER BY time DESC; Output: _hyper_4_4_chunk_compressed._ts_meta_count, _hyper_4_4_chunk_compressed.segby, _hyper_4_4_chunk_compressed._ts_meta_min_1, _hyper_4_4_chunk_compressed._ts_meta_max_1, _hyper_4_4_chunk_compressed._ts_meta_v2_first_time, _hyper_4_4_chunk_compressed._ts_meta_v2_last_time, _hyper_4_4_chunk_compressed."time", _hyper_4_4_chunk_compressed.val Sort Key: _hyper_4_4_chunk_compressed._ts_meta_v2_first_time DESC Sort Method: quicksort - -> Index Scan using _hyper_4_4_chunk_compressed_segby__ts_meta_v2_first_time__t_idx on _timescaledb_internal._hyper_4_4_chunk_compressed (actual rows=1.00 loops=1) + -> Seq Scan on _timescaledb_internal._hyper_4_4_chunk_compressed (actual rows=1.00 loops=1) Output: _hyper_4_4_chunk_compressed._ts_meta_count, _hyper_4_4_chunk_compressed.segby, _hyper_4_4_chunk_compressed._ts_meta_min_1, _hyper_4_4_chunk_compressed._ts_meta_max_1, _hyper_4_4_chunk_compressed._ts_meta_v2_first_time, _hyper_4_4_chunk_compressed._ts_meta_v2_last_time, _hyper_4_4_chunk_compressed."time", _hyper_4_4_chunk_compressed.val - Index Cond: (_hyper_4_4_chunk_compressed.segby = 1) + Filter: (_hyper_4_4_chunk_compressed.segby = 1) + Rows Removed by Filter: 1 :PREFIX SELECT * FROM test1 WHERE x1 = 1 AND x2 = 2 AND x5 = 0 ORDER BY x1, x2, x5, time DESC; @@ -1737,6 +1803,9 @@ SELECT * FROM test2 WHERE x1 = 1 AND x2 = 2 AND x5 = 0 ORDER BY x1 DESC, x2 DESC Filter: ((_hyper_2_2_chunk_compressed.x1 = 1) AND (_hyper_2_2_chunk_compressed.x2 = 2) AND (_hyper_2_2_chunk_compressed.x5 = 0)) Rows Removed by Filter: 2 +SET enable_seqscan=0; +SET enable_bitmapscan=0; +SET enable_indexscan=1; -- No segmentby CREATE TABLE test_nosegby ( segby int NOT NULL, @@ -1765,6 +1834,9 @@ SELECT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('test_no {COMPRESSED,UNORDERED} -- Should be optimized (implicit NULLS first) +-- but can't use compressed indexscan order, need explicit sort: multikey orderby +-- Batch sort merge needs sort by (first_col1, first_col2) +-- while compressed input is sorted on (first_col1, last_col1, first_col2, last_col2) :PREFIX SELECT * FROM test_nosegby ORDER BY segby, time DESC; --- QUERY PLAN --- @@ -1777,10 +1849,10 @@ SELECT * FROM test_nosegby ORDER BY segby, time DESC; Output: _hyper_6_6_chunk_compressed._ts_meta_count, _hyper_6_6_chunk_compressed._ts_meta_min_1, _hyper_6_6_chunk_compressed._ts_meta_max_1, _hyper_6_6_chunk_compressed._ts_meta_v2_first_segby, _hyper_6_6_chunk_compressed._ts_meta_v2_last_segby, _hyper_6_6_chunk_compressed.segby, _hyper_6_6_chunk_compressed._ts_meta_min_2, _hyper_6_6_chunk_compressed._ts_meta_max_2, _hyper_6_6_chunk_compressed._ts_meta_v2_first_time, _hyper_6_6_chunk_compressed._ts_meta_v2_last_time, _hyper_6_6_chunk_compressed."time", _hyper_6_6_chunk_compressed.val Sort Key: _hyper_6_6_chunk_compressed._ts_meta_v2_first_segby, _hyper_6_6_chunk_compressed._ts_meta_v2_first_time DESC Sort Method: quicksort - -> Seq Scan on _timescaledb_internal._hyper_6_6_chunk_compressed (actual rows=1.00 loops=1) + -> Index Scan using _hyper_6_6_chunk_compressed__ts_meta_v2_first_segby__ts_met_idx on _timescaledb_internal._hyper_6_6_chunk_compressed (actual rows=1.00 loops=1) Output: _hyper_6_6_chunk_compressed._ts_meta_count, _hyper_6_6_chunk_compressed._ts_meta_min_1, _hyper_6_6_chunk_compressed._ts_meta_max_1, _hyper_6_6_chunk_compressed._ts_meta_v2_first_segby, _hyper_6_6_chunk_compressed._ts_meta_v2_last_segby, _hyper_6_6_chunk_compressed.segby, _hyper_6_6_chunk_compressed._ts_meta_min_2, _hyper_6_6_chunk_compressed._ts_meta_max_2, _hyper_6_6_chunk_compressed._ts_meta_v2_first_time, _hyper_6_6_chunk_compressed._ts_meta_v2_last_time, _hyper_6_6_chunk_compressed."time", _hyper_6_6_chunk_compressed.val --- Should be optimized +-- Should be optimized but can't use compressed indexscan order: multikey orderby :PREFIX SELECT * FROM test_nosegby ORDER BY segby, time DESC NULLS FIRST; --- QUERY PLAN --- @@ -1793,10 +1865,10 @@ SELECT * FROM test_nosegby ORDER BY segby, time DESC NULLS FIRST; Output: _hyper_6_6_chunk_compressed._ts_meta_count, _hyper_6_6_chunk_compressed._ts_meta_min_1, _hyper_6_6_chunk_compressed._ts_meta_max_1, _hyper_6_6_chunk_compressed._ts_meta_v2_first_segby, _hyper_6_6_chunk_compressed._ts_meta_v2_last_segby, _hyper_6_6_chunk_compressed.segby, _hyper_6_6_chunk_compressed._ts_meta_min_2, _hyper_6_6_chunk_compressed._ts_meta_max_2, _hyper_6_6_chunk_compressed._ts_meta_v2_first_time, _hyper_6_6_chunk_compressed._ts_meta_v2_last_time, _hyper_6_6_chunk_compressed."time", _hyper_6_6_chunk_compressed.val Sort Key: _hyper_6_6_chunk_compressed._ts_meta_v2_first_segby, _hyper_6_6_chunk_compressed._ts_meta_v2_first_time DESC Sort Method: quicksort - -> Seq Scan on _timescaledb_internal._hyper_6_6_chunk_compressed (actual rows=1.00 loops=1) + -> Index Scan using _hyper_6_6_chunk_compressed__ts_meta_v2_first_segby__ts_met_idx on _timescaledb_internal._hyper_6_6_chunk_compressed (actual rows=1.00 loops=1) Output: _hyper_6_6_chunk_compressed._ts_meta_count, _hyper_6_6_chunk_compressed._ts_meta_min_1, _hyper_6_6_chunk_compressed._ts_meta_max_1, _hyper_6_6_chunk_compressed._ts_meta_v2_first_segby, _hyper_6_6_chunk_compressed._ts_meta_v2_last_segby, _hyper_6_6_chunk_compressed.segby, _hyper_6_6_chunk_compressed._ts_meta_min_2, _hyper_6_6_chunk_compressed._ts_meta_max_2, _hyper_6_6_chunk_compressed._ts_meta_v2_first_time, _hyper_6_6_chunk_compressed._ts_meta_v2_last_time, _hyper_6_6_chunk_compressed."time", _hyper_6_6_chunk_compressed.val --- Should be optimized (backward scan, NULLS last) +-- Should be optimized but can't use compressed indexscan order (backward scan, NULLS last) :PREFIX SELECT * FROM test_nosegby ORDER BY segby DESC, time ASC NULLS LAST; --- QUERY PLAN --- @@ -1813,7 +1885,7 @@ SELECT * FROM test_nosegby ORDER BY segby DESC, time ASC NULLS LAST; -> Seq Scan on _timescaledb_internal._hyper_6_6_chunk_compressed (actual rows=1.00 loops=1) Output: _hyper_6_6_chunk_compressed._ts_meta_count, _hyper_6_6_chunk_compressed._ts_meta_min_1, _hyper_6_6_chunk_compressed._ts_meta_max_1, _hyper_6_6_chunk_compressed._ts_meta_v2_first_segby, _hyper_6_6_chunk_compressed._ts_meta_v2_last_segby, _hyper_6_6_chunk_compressed.segby, _hyper_6_6_chunk_compressed._ts_meta_min_2, _hyper_6_6_chunk_compressed._ts_meta_max_2, _hyper_6_6_chunk_compressed._ts_meta_v2_first_time, _hyper_6_6_chunk_compressed._ts_meta_v2_last_time, _hyper_6_6_chunk_compressed."time", _hyper_6_6_chunk_compressed.val --- Should be optimized (backward scan) +-- Should be optimized but can't use compressed indexscan order (backward scan) :PREFIX SELECT * FROM test_nosegby ORDER BY segby DESC; --- QUERY PLAN --- @@ -1830,6 +1902,18 @@ SELECT * FROM test_nosegby ORDER BY segby DESC; -> Seq Scan on _timescaledb_internal._hyper_6_6_chunk_compressed (actual rows=1.00 loops=1) Output: _hyper_6_6_chunk_compressed._ts_meta_count, _hyper_6_6_chunk_compressed._ts_meta_min_1, _hyper_6_6_chunk_compressed._ts_meta_max_1, _hyper_6_6_chunk_compressed._ts_meta_v2_first_segby, _hyper_6_6_chunk_compressed._ts_meta_v2_last_segby, _hyper_6_6_chunk_compressed.segby, _hyper_6_6_chunk_compressed._ts_meta_min_2, _hyper_6_6_chunk_compressed._ts_meta_max_2, _hyper_6_6_chunk_compressed._ts_meta_v2_first_time, _hyper_6_6_chunk_compressed._ts_meta_v2_last_time, _hyper_6_6_chunk_compressed."time", _hyper_6_6_chunk_compressed.val +-- Should be optimized and can use compressed indexscan order by (first_col1) +:PREFIX +SELECT * FROM test_nosegby ORDER BY segby; +--- QUERY PLAN --- + Custom Scan (ColumnarScan) on _timescaledb_internal._hyper_6_6_chunk (actual rows=12.00 loops=1) + Output: _hyper_6_6_chunk.segby, _hyper_6_6_chunk."time", _hyper_6_6_chunk.val + Chunk Status: UNORDERED + Batch Sorted Merge: true + Bulk Decompression: false + -> Index Scan using _hyper_6_6_chunk_compressed__ts_meta_v2_first_segby__ts_met_idx on _timescaledb_internal._hyper_6_6_chunk_compressed (actual rows=1.00 loops=1) + Output: _hyper_6_6_chunk_compressed._ts_meta_count, _hyper_6_6_chunk_compressed._ts_meta_min_1, _hyper_6_6_chunk_compressed._ts_meta_max_1, _hyper_6_6_chunk_compressed._ts_meta_v2_first_segby, _hyper_6_6_chunk_compressed._ts_meta_v2_last_segby, _hyper_6_6_chunk_compressed.segby, _hyper_6_6_chunk_compressed._ts_meta_min_2, _hyper_6_6_chunk_compressed._ts_meta_max_2, _hyper_6_6_chunk_compressed._ts_meta_v2_first_time, _hyper_6_6_chunk_compressed._ts_meta_v2_last_time, _hyper_6_6_chunk_compressed."time", _hyper_6_6_chunk_compressed.val + set timescaledb.debug_require_batch_sorted_merge to 'forbid'; -- Should not be optimized (NULL order wrong) :PREFIX @@ -1861,9 +1945,648 @@ SELECT * FROM test_nosegby ORDER BY segby DESC NULLS LAST; -> Seq Scan on _timescaledb_internal._hyper_6_6_chunk_compressed (actual rows=1.00 loops=1) Output: _hyper_6_6_chunk_compressed._ts_meta_count, _hyper_6_6_chunk_compressed._ts_meta_min_1, _hyper_6_6_chunk_compressed._ts_meta_max_1, _hyper_6_6_chunk_compressed._ts_meta_v2_first_segby, _hyper_6_6_chunk_compressed._ts_meta_v2_last_segby, _hyper_6_6_chunk_compressed.segby, _hyper_6_6_chunk_compressed._ts_meta_min_2, _hyper_6_6_chunk_compressed._ts_meta_max_2, _hyper_6_6_chunk_compressed._ts_meta_v2_first_time, _hyper_6_6_chunk_compressed._ts_meta_v2_last_time, _hyper_6_6_chunk_compressed."time", _hyper_6_6_chunk_compressed.val +-- Test for correct results on table with overlapping batches and segmentby pinned to a const +CREATE TABLE t2(time int NOT NULL, dev int NOT NULL, v int); +SELECT table_name FROM create_hypertable('t2', 'time', chunk_time_interval => 10000); + table_name +------------ + t2 + +ALTER TABLE t2 SET (timescaledb.compress, timescaledb.compress_orderby='time DESC', timescaledb.compress_segmentby='dev'); +INSERT INTO t2 (time, dev, v) values +(1, 1, 20), +(1, 2, 300), +(2, 3, 3000), +(3, 1, 40), +(4, 1, 10), +(4, 2, 100), +(6, 3, 2000), +(6, 1, 10), +(6, 2, 300), +(8, 1, 60), +(8, 2, 100), +(8, 3, 7000); +INSERT INTO t2 (time, dev, v) values +(3, 1, 20), +(3, 2, 300), +(3, 3, 3000), +(5, 1, 40), +(5, 2, 100), +(5, 3, 1000), +(6, 1, 2000), +(6, 1, 10), +(6, 1, 200), +(7, 1, 60), +(7, 2, 100), +(7, 3, 7000); +INSERT INTO t2 (time, dev, v) values +(5, 1, 20), +(5, 3, 3000), +(5, 2, 300), +(7, 2, 400), +(8, 1, 10), +(8, 2, 100), +(8, 3, 2000), +(10, 1, 10), +(11, 1, 300), +(14, 1, 60), +(14, 2, 100), +(14, 3, 7000); +INSERT INTO t2 (time, dev, v) values +(9, 1, 20), +(9, 3, 300), +(9, 2, 30), +(9, 2, 40), +(10, 1, 10), +(10, 2, 100), +(10, 3, 2000), +(11, 1, 10), +(12, 1, 300), +(13, 1, 60), +(13, 2, 100), +(13, 3, 7000); +SELECT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('t2') chunk; + chunk_status_text +------------------------ + {COMPRESSED,UNORDERED} + +SET timescaledb.debug_require_batch_sorted_merge = 'force'; +:PREFIX +SELECT dev, time FROM t2 WHERE dev = 1 ORDER BY time DESC; +--- QUERY PLAN --- + Custom Scan (ColumnarScan) on _timescaledb_internal._hyper_7_7_chunk (actual rows=21.00 loops=1) + Output: _hyper_7_7_chunk.dev, _hyper_7_7_chunk."time" + Chunk Status: UNORDERED + Batch Sorted Merge: true + Bulk Decompression: false + -> Index Scan using _hyper_7_7_chunk_compressed_dev__ts_meta_v2_first_time__ts__idx on _timescaledb_internal._hyper_7_7_chunk_compressed (actual rows=4.00 loops=1) + Output: _hyper_7_7_chunk_compressed._ts_meta_count, _hyper_7_7_chunk_compressed.dev, _hyper_7_7_chunk_compressed._ts_meta_min_1, _hyper_7_7_chunk_compressed._ts_meta_max_1, _hyper_7_7_chunk_compressed._ts_meta_v2_first_time, _hyper_7_7_chunk_compressed._ts_meta_v2_last_time, _hyper_7_7_chunk_compressed."time", _hyper_7_7_chunk_compressed.v + Index Cond: (_hyper_7_7_chunk_compressed.dev = 1) + +SELECT dev, time FROM t2 WHERE dev = 1 ORDER BY time DESC; + dev | time +-----+------ + 1 | 14 + 1 | 13 + 1 | 12 + 1 | 11 + 1 | 11 + 1 | 10 + 1 | 10 + 1 | 9 + 1 | 8 + 1 | 8 + 1 | 7 + 1 | 6 + 1 | 6 + 1 | 6 + 1 | 6 + 1 | 5 + 1 | 5 + 1 | 4 + 1 | 3 + 1 | 3 + 1 | 1 + +-- Can't use compressed indexscan order if orderby direction does not match: +-- we will need to sort on "last" metadata column instead of "first" in this case. +:PREFIX +SELECT dev, time FROM t2 WHERE dev = 1 ORDER BY time; +--- QUERY PLAN --- + Custom Scan (ColumnarScan) on _timescaledb_internal._hyper_7_7_chunk (actual rows=21.00 loops=1) + Output: _hyper_7_7_chunk.dev, _hyper_7_7_chunk."time" + Chunk Status: UNORDERED + Batch Sorted Merge: true + Reverse: true + Bulk Decompression: false + -> Sort (actual rows=4.00 loops=1) + Output: _hyper_7_7_chunk_compressed._ts_meta_count, _hyper_7_7_chunk_compressed.dev, _hyper_7_7_chunk_compressed._ts_meta_min_1, _hyper_7_7_chunk_compressed._ts_meta_max_1, _hyper_7_7_chunk_compressed._ts_meta_v2_first_time, _hyper_7_7_chunk_compressed._ts_meta_v2_last_time, _hyper_7_7_chunk_compressed."time", _hyper_7_7_chunk_compressed.v + Sort Key: _hyper_7_7_chunk_compressed._ts_meta_v2_last_time + Sort Method: quicksort + -> Index Scan using _hyper_7_7_chunk_compressed_dev__ts_meta_v2_first_time__ts__idx on _timescaledb_internal._hyper_7_7_chunk_compressed (actual rows=4.00 loops=1) + Output: _hyper_7_7_chunk_compressed._ts_meta_count, _hyper_7_7_chunk_compressed.dev, _hyper_7_7_chunk_compressed._ts_meta_min_1, _hyper_7_7_chunk_compressed._ts_meta_max_1, _hyper_7_7_chunk_compressed._ts_meta_v2_first_time, _hyper_7_7_chunk_compressed._ts_meta_v2_last_time, _hyper_7_7_chunk_compressed."time", _hyper_7_7_chunk_compressed.v + Index Cond: (_hyper_7_7_chunk_compressed.dev = 1) + +SELECT dev, time FROM t2 WHERE dev = 1 ORDER BY time; + dev | time +-----+------ + 1 | 1 + 1 | 3 + 1 | 3 + 1 | 4 + 1 | 5 + 1 | 5 + 1 | 6 + 1 | 6 + 1 | 6 + 1 | 6 + 1 | 7 + 1 | 8 + 1 | 8 + 1 | 9 + 1 | 10 + 1 | 10 + 1 | 11 + 1 | 11 + 1 | 12 + 1 | 13 + 1 | 14 + +-- Predicates on non-segmentby columns +:PREFIX +SELECT dev, time FROM t2 WHERE time > 8 AND dev=1 ORDER BY dev, time DESC; +--- QUERY PLAN --- + Custom Scan (ColumnarScan) on _timescaledb_internal._hyper_7_7_chunk (actual rows=8.00 loops=1) + Output: _hyper_7_7_chunk.dev, _hyper_7_7_chunk."time" + Filter: (_hyper_7_7_chunk."time" > 8) + Rows Removed by Filter: 2 + Chunk Status: UNORDERED + Batch Sorted Merge: true + Bulk Decompression: false + -> Index Scan using _hyper_7_7_chunk_compressed_dev__ts_meta_v2_first_time__ts__idx on _timescaledb_internal._hyper_7_7_chunk_compressed (actual rows=2.00 loops=1) + Output: _hyper_7_7_chunk_compressed._ts_meta_count, _hyper_7_7_chunk_compressed.dev, _hyper_7_7_chunk_compressed._ts_meta_min_1, _hyper_7_7_chunk_compressed._ts_meta_max_1, _hyper_7_7_chunk_compressed._ts_meta_v2_first_time, _hyper_7_7_chunk_compressed._ts_meta_v2_last_time, _hyper_7_7_chunk_compressed."time", _hyper_7_7_chunk_compressed.v + Index Cond: ((_hyper_7_7_chunk_compressed.dev = 1) AND (_hyper_7_7_chunk_compressed._ts_meta_v2_first_time > 8)) + +SELECT dev, time FROM t2 WHERE time > 8 AND dev=1 ORDER BY dev, time DESC; + dev | time +-----+------ + 1 | 14 + 1 | 13 + 1 | 12 + 1 | 11 + 1 | 11 + 1 | 10 + 1 | 10 + 1 | 9 + +:PREFIX +SELECT dev, time, v FROM t2 WHERE v = 20 AND dev=1 ORDER BY dev, time DESC; +--- QUERY PLAN --- + Custom Scan (ColumnarScan) on _timescaledb_internal._hyper_7_7_chunk (actual rows=4.00 loops=1) + Output: _hyper_7_7_chunk.dev, _hyper_7_7_chunk."time", _hyper_7_7_chunk.v + Filter: (_hyper_7_7_chunk.v = 20) + Rows Removed by Filter: 17 + Chunk Status: UNORDERED + Batch Sorted Merge: true + Bulk Decompression: false + -> Index Scan using _hyper_7_7_chunk_compressed_dev__ts_meta_v2_first_time__ts__idx on _timescaledb_internal._hyper_7_7_chunk_compressed (actual rows=4.00 loops=1) + Output: _hyper_7_7_chunk_compressed._ts_meta_count, _hyper_7_7_chunk_compressed.dev, _hyper_7_7_chunk_compressed._ts_meta_min_1, _hyper_7_7_chunk_compressed._ts_meta_max_1, _hyper_7_7_chunk_compressed._ts_meta_v2_first_time, _hyper_7_7_chunk_compressed._ts_meta_v2_last_time, _hyper_7_7_chunk_compressed."time", _hyper_7_7_chunk_compressed.v + Index Cond: (_hyper_7_7_chunk_compressed.dev = 1) + +SELECT dev, time, v FROM t2 WHERE v = 20 AND dev=1 ORDER BY dev, time DESC; + dev | time | v +-----+------+---- + 1 | 9 | 20 + 1 | 5 | 20 + 1 | 3 | 20 + 1 | 1 | 20 + +-- Rescan with lateral subquery +:PREFIX +SELECT dev, time +FROM (VALUES (1), (2), (3)) a(dv), + LATERAL (SELECT dev, time FROM t2 WHERE dev = a.dv ORDER BY time DESC) b; +--- QUERY PLAN --- + Nested Loop (actual rows=48.00 loops=1) + Output: t2.dev, t2."time" + -> Values Scan on "*VALUES*" (actual rows=3.00 loops=1) + Output: "*VALUES*".column1 + -> Custom Scan (ChunkAppend) on public.t2 (actual rows=16.00 loops=3) + Output: t2.dev, t2."time" + Order: t2."time" DESC + Startup Exclusion: false + Runtime Exclusion: true + Hypertables excluded during runtime: 0 + -> Custom Scan (ColumnarScan) on _timescaledb_internal._hyper_7_7_chunk (actual rows=16.00 loops=3) + Output: _hyper_7_7_chunk.dev, _hyper_7_7_chunk."time" + Chunk Status: UNORDERED + Batch Sorted Merge: true + Bulk Decompression: false + -> Index Scan using _hyper_7_7_chunk_compressed_dev__ts_meta_v2_first_time__ts__idx on _timescaledb_internal._hyper_7_7_chunk_compressed (actual rows=4.00 loops=3) + Output: _hyper_7_7_chunk_compressed._ts_meta_count, _hyper_7_7_chunk_compressed.dev, _hyper_7_7_chunk_compressed._ts_meta_min_1, _hyper_7_7_chunk_compressed._ts_meta_max_1, _hyper_7_7_chunk_compressed._ts_meta_v2_first_time, _hyper_7_7_chunk_compressed._ts_meta_v2_last_time, _hyper_7_7_chunk_compressed."time", _hyper_7_7_chunk_compressed.v + Index Cond: (_hyper_7_7_chunk_compressed.dev = "*VALUES*".column1) + +SELECT dev, time +FROM (VALUES (1), (2), (3)) a(dv), + LATERAL (SELECT dev, time FROM t2 WHERE dev = a.dv ORDER BY time DESC) b; + dev | time +-----+------ + 1 | 14 + 1 | 13 + 1 | 12 + 1 | 11 + 1 | 11 + 1 | 10 + 1 | 10 + 1 | 9 + 1 | 8 + 1 | 8 + 1 | 7 + 1 | 6 + 1 | 6 + 1 | 6 + 1 | 6 + 1 | 5 + 1 | 5 + 1 | 4 + 1 | 3 + 1 | 3 + 1 | 1 + 2 | 14 + 2 | 13 + 2 | 10 + 2 | 9 + 2 | 9 + 2 | 8 + 2 | 8 + 2 | 7 + 2 | 7 + 2 | 6 + 2 | 5 + 2 | 5 + 2 | 4 + 2 | 3 + 2 | 1 + 3 | 14 + 3 | 13 + 3 | 10 + 3 | 9 + 3 | 8 + 3 | 8 + 3 | 7 + 3 | 6 + 3 | 5 + 3 | 5 + 3 | 3 + 3 | 2 + +-- Test for correct results on table with overlapping batches and no segmentby +CREATE TABLE t2_noseg(time int NOT NULL, v int); +SELECT table_name FROM create_hypertable('t2_noseg', 'time', chunk_time_interval => 10000); + table_name +------------ + t2_noseg + +ALTER TABLE t2_noseg SET (timescaledb.compress, timescaledb.compress_orderby='time DESC'); +INSERT INTO t2_noseg (time, v) values +(1, 20), +(1, 20), +(2, 20), +(3, 20), +(4, 10), +(4, 10), +(6, 10), +(6, 10), +(6, 10), +(8, 30), +(8, 30), +(8, 30); +INSERT INTO t2_noseg (time, v) values +(3, 20), +(3, 20), +(3, 20), +(5, 10), +(5, 10), +(5, 10), +(6, 20), +(6, 20), +(6, 20), +(7, 10), +(7, 10), +(7, 10); +INSERT INTO t2_noseg (time, v) values +(5, 10), +(5, 10), +(5, 10), +(7, 10), +(8, 30), +(8, 30), +(8, 30), +(10, 10), +(11, 30), +(14, 20), +(14, 20), +(14, 20); +INSERT INTO t2_noseg (time, v) values +(9, 30), +(9, 30), +(9, 30), +(9, 30), +(10, 10), +(10, 10), +(10, 10), +(11, 30), +(12, 30), +(13, 30), +(13, 30), +(13, 30); +SELECT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('t2_noseg') chunk; + chunk_status_text +------------------------ + {COMPRESSED,UNORDERED} + +:PREFIX +SELECT time, v FROM t2_noseg ORDER BY time DESC; +--- QUERY PLAN --- + Custom Scan (ColumnarScan) on _timescaledb_internal._hyper_8_8_chunk (actual rows=48.00 loops=1) + Output: _hyper_8_8_chunk."time", _hyper_8_8_chunk.v + Chunk Status: UNORDERED + Batch Sorted Merge: true + Bulk Decompression: false + -> Index Scan using _hyper_8_8_chunk_compressed__ts_meta_v2_first_time__ts_meta_idx on _timescaledb_internal._hyper_8_8_chunk_compressed (actual rows=4.00 loops=1) + Output: _hyper_8_8_chunk_compressed._ts_meta_count, _hyper_8_8_chunk_compressed._ts_meta_min_1, _hyper_8_8_chunk_compressed._ts_meta_max_1, _hyper_8_8_chunk_compressed._ts_meta_v2_first_time, _hyper_8_8_chunk_compressed._ts_meta_v2_last_time, _hyper_8_8_chunk_compressed."time", _hyper_8_8_chunk_compressed.v + +SELECT time, v FROM t2_noseg ORDER BY time DESC; + time | v +------+---- + 14 | 20 + 14 | 20 + 14 | 20 + 13 | 30 + 13 | 30 + 13 | 30 + 12 | 30 + 11 | 30 + 11 | 30 + 10 | 10 + 10 | 10 + 10 | 10 + 10 | 10 + 9 | 30 + 9 | 30 + 9 | 30 + 9 | 30 + 8 | 30 + 8 | 30 + 8 | 30 + 8 | 30 + 8 | 30 + 8 | 30 + 7 | 10 + 7 | 10 + 7 | 10 + 7 | 10 + 6 | 20 + 6 | 20 + 6 | 20 + 6 | 10 + 6 | 10 + 6 | 10 + 5 | 10 + 5 | 10 + 5 | 10 + 5 | 10 + 5 | 10 + 5 | 10 + 4 | 10 + 4 | 10 + 3 | 20 + 3 | 20 + 3 | 20 + 3 | 20 + 2 | 20 + 1 | 20 + 1 | 20 + +-- Can't use compressed indexscan order if orderby direction does not match: +-- we will need to sort on "last" metadata column instead of "first" in this case. +:PREFIX +SELECT time, v FROM t2_noseg ORDER BY time; +--- QUERY PLAN --- + Custom Scan (ColumnarScan) on _timescaledb_internal._hyper_8_8_chunk (actual rows=48.00 loops=1) + Output: _hyper_8_8_chunk."time", _hyper_8_8_chunk.v + Chunk Status: UNORDERED + Batch Sorted Merge: true + Reverse: true + Bulk Decompression: false + -> Sort (actual rows=4.00 loops=1) + Output: _hyper_8_8_chunk_compressed._ts_meta_count, _hyper_8_8_chunk_compressed._ts_meta_min_1, _hyper_8_8_chunk_compressed._ts_meta_max_1, _hyper_8_8_chunk_compressed._ts_meta_v2_first_time, _hyper_8_8_chunk_compressed._ts_meta_v2_last_time, _hyper_8_8_chunk_compressed."time", _hyper_8_8_chunk_compressed.v + Sort Key: _hyper_8_8_chunk_compressed._ts_meta_v2_last_time + Sort Method: quicksort + -> Seq Scan on _timescaledb_internal._hyper_8_8_chunk_compressed (actual rows=4.00 loops=1) + Output: _hyper_8_8_chunk_compressed._ts_meta_count, _hyper_8_8_chunk_compressed._ts_meta_min_1, _hyper_8_8_chunk_compressed._ts_meta_max_1, _hyper_8_8_chunk_compressed._ts_meta_v2_first_time, _hyper_8_8_chunk_compressed._ts_meta_v2_last_time, _hyper_8_8_chunk_compressed."time", _hyper_8_8_chunk_compressed.v + +SELECT time, v FROM t2_noseg ORDER BY time; + time | v +------+---- + 1 | 20 + 1 | 20 + 2 | 20 + 3 | 20 + 3 | 20 + 3 | 20 + 3 | 20 + 4 | 10 + 4 | 10 + 5 | 10 + 5 | 10 + 5 | 10 + 5 | 10 + 5 | 10 + 5 | 10 + 6 | 10 + 6 | 10 + 6 | 10 + 6 | 20 + 6 | 20 + 6 | 20 + 7 | 10 + 7 | 10 + 7 | 10 + 7 | 10 + 8 | 30 + 8 | 30 + 8 | 30 + 8 | 30 + 8 | 30 + 8 | 30 + 9 | 30 + 9 | 30 + 9 | 30 + 9 | 30 + 10 | 10 + 10 | 10 + 10 | 10 + 10 | 10 + 11 | 30 + 11 | 30 + 12 | 30 + 13 | 30 + 13 | 30 + 13 | 30 + 14 | 20 + 14 | 20 + 14 | 20 + +-- Predicates +:PREFIX +SELECT time, v FROM t2_noseg WHERE time > 8 ORDER BY time DESC; +--- QUERY PLAN --- + Custom Scan (ColumnarScan) on _timescaledb_internal._hyper_8_8_chunk (actual rows=17.00 loops=1) + Output: _hyper_8_8_chunk."time", _hyper_8_8_chunk.v + Filter: (_hyper_8_8_chunk."time" > 8) + Rows Removed by Filter: 7 + Chunk Status: UNORDERED + Batch Sorted Merge: true + Bulk Decompression: false + -> Index Scan using _hyper_8_8_chunk_compressed__ts_meta_v2_first_time__ts_meta_idx on _timescaledb_internal._hyper_8_8_chunk_compressed (actual rows=2.00 loops=1) + Output: _hyper_8_8_chunk_compressed._ts_meta_count, _hyper_8_8_chunk_compressed._ts_meta_min_1, _hyper_8_8_chunk_compressed._ts_meta_max_1, _hyper_8_8_chunk_compressed._ts_meta_v2_first_time, _hyper_8_8_chunk_compressed._ts_meta_v2_last_time, _hyper_8_8_chunk_compressed."time", _hyper_8_8_chunk_compressed.v + Index Cond: (_hyper_8_8_chunk_compressed._ts_meta_v2_first_time > 8) + +SELECT time, v FROM t2_noseg WHERE time > 8 ORDER BY time DESC; + time | v +------+---- + 14 | 20 + 14 | 20 + 14 | 20 + 13 | 30 + 13 | 30 + 13 | 30 + 12 | 30 + 11 | 30 + 11 | 30 + 10 | 10 + 10 | 10 + 10 | 10 + 10 | 10 + 9 | 30 + 9 | 30 + 9 | 30 + 9 | 30 + +:PREFIX +SELECT time, v FROM t2_noseg WHERE v > 10 ORDER BY time DESC; +--- QUERY PLAN --- + Custom Scan (ColumnarScan) on _timescaledb_internal._hyper_8_8_chunk (actual rows=29.00 loops=1) + Output: _hyper_8_8_chunk."time", _hyper_8_8_chunk.v + Filter: (_hyper_8_8_chunk.v > 10) + Rows Removed by Filter: 19 + Chunk Status: UNORDERED + Batch Sorted Merge: true + Bulk Decompression: false + -> Index Scan using _hyper_8_8_chunk_compressed__ts_meta_v2_first_time__ts_meta_idx on _timescaledb_internal._hyper_8_8_chunk_compressed (actual rows=4.00 loops=1) + Output: _hyper_8_8_chunk_compressed._ts_meta_count, _hyper_8_8_chunk_compressed._ts_meta_min_1, _hyper_8_8_chunk_compressed._ts_meta_max_1, _hyper_8_8_chunk_compressed._ts_meta_v2_first_time, _hyper_8_8_chunk_compressed._ts_meta_v2_last_time, _hyper_8_8_chunk_compressed."time", _hyper_8_8_chunk_compressed.v + +SELECT time, v FROM t2_noseg WHERE v > 10 ORDER BY time DESC; + time | v +------+---- + 14 | 20 + 14 | 20 + 14 | 20 + 13 | 30 + 13 | 30 + 13 | 30 + 12 | 30 + 11 | 30 + 11 | 30 + 9 | 30 + 9 | 30 + 9 | 30 + 9 | 30 + 8 | 30 + 8 | 30 + 8 | 30 + 8 | 30 + 8 | 30 + 8 | 30 + 6 | 20 + 6 | 20 + 6 | 20 + 3 | 20 + 3 | 20 + 3 | 20 + 3 | 20 + 2 | 20 + 1 | 20 + 1 | 20 + +-- Rescan with lateral subquery +:PREFIX +SELECT time, v +FROM (VALUES (10), (20), (30)) a(dv), + LATERAL (SELECT time, v FROM t2_noseg WHERE v = a.dv ORDER BY time DESC) b; +--- QUERY PLAN --- + Nested Loop (actual rows=48.00 loops=1) + Output: t2_noseg."time", t2_noseg.v + -> Values Scan on "*VALUES*" (actual rows=3.00 loops=1) + Output: "*VALUES*".column1 + -> Custom Scan (ChunkAppend) on public.t2_noseg (actual rows=16.00 loops=3) + Output: t2_noseg."time", t2_noseg.v + Order: t2_noseg."time" DESC + Startup Exclusion: false + Runtime Exclusion: true + Hypertables excluded during runtime: 0 + -> Custom Scan (ColumnarScan) on _timescaledb_internal._hyper_8_8_chunk (actual rows=16.00 loops=3) + Output: _hyper_8_8_chunk."time", _hyper_8_8_chunk.v + Filter: (_hyper_8_8_chunk.v = "*VALUES*".column1) + Rows Removed by Filter: 32 + Chunk Status: UNORDERED + Batch Sorted Merge: true + Bulk Decompression: false + -> Index Scan using _hyper_8_8_chunk_compressed__ts_meta_v2_first_time__ts_meta_idx on _timescaledb_internal._hyper_8_8_chunk_compressed (actual rows=4.00 loops=3) + Output: _hyper_8_8_chunk_compressed._ts_meta_count, _hyper_8_8_chunk_compressed._ts_meta_min_1, _hyper_8_8_chunk_compressed._ts_meta_max_1, _hyper_8_8_chunk_compressed._ts_meta_v2_first_time, _hyper_8_8_chunk_compressed._ts_meta_v2_last_time, _hyper_8_8_chunk_compressed."time", _hyper_8_8_chunk_compressed.v + +SELECT time, v +FROM (VALUES (10), (20), (30)) a(dv), + LATERAL (SELECT time, v FROM t2_noseg WHERE v = a.dv ORDER BY time DESC) b; + time | v +------+---- + 10 | 10 + 10 | 10 + 10 | 10 + 10 | 10 + 7 | 10 + 7 | 10 + 7 | 10 + 7 | 10 + 6 | 10 + 6 | 10 + 6 | 10 + 5 | 10 + 5 | 10 + 5 | 10 + 5 | 10 + 5 | 10 + 5 | 10 + 4 | 10 + 4 | 10 + 14 | 20 + 14 | 20 + 14 | 20 + 6 | 20 + 6 | 20 + 6 | 20 + 3 | 20 + 3 | 20 + 3 | 20 + 3 | 20 + 2 | 20 + 1 | 20 + 1 | 20 + 13 | 30 + 13 | 30 + 13 | 30 + 12 | 30 + 11 | 30 + 11 | 30 + 9 | 30 + 9 | 30 + 9 | 30 + 9 | 30 + 8 | 30 + 8 | 30 + 8 | 30 + 8 | 30 + 8 | 30 + 8 | 30 + drop table test1 cascade; drop table test2 cascade; drop table test_segby cascade; drop table test_nosegby cascade; +drop table t2 cascade; +drop table t2_noseg cascade; RESET timescaledb.enable_direct_compress_insert; RESET timescaledb.debug_require_batch_sorted_merge; +RESET enable_seqscan; +RESET enable_bitmapscan; diff --git a/tsl/test/expected/recompress_chunk_segmentwise.out b/tsl/test/expected/recompress_chunk_segmentwise.out index 199d2a20078..ac40b878fea 100644 --- a/tsl/test/expected/recompress_chunk_segmentwise.out +++ b/tsl/test/expected/recompress_chunk_segmentwise.out @@ -774,10 +774,8 @@ SET timescaledb.debug_require_batch_sorted_merge = 'force'; EXPLAIN (COSTS OFF) SELECT a, time FROM segwise_unordered WHERE a = 2 ORDER BY a, time; --- QUERY PLAN --- Custom Scan (ColumnarScan) on _hyper_13_13_chunk - -> Sort - Sort Key: _hyper_13_13_chunk_compressed._ts_meta_v2_first_time - -> Index Scan using _hyper_13_13_chunk_compressed_a__ts_meta_v2_first_time__ts__idx on _hyper_13_13_chunk_compressed - Index Cond: (a = 2) + -> Index Scan using _hyper_13_13_chunk_compressed_a__ts_meta_v2_first_time__ts__idx on _hyper_13_13_chunk_compressed + Index Cond: (a = 2) RESET timescaledb.debug_require_batch_sorted_merge; DROP TABLE segwise_unordered; diff --git a/tsl/test/sql/compression_sorted_merge_unordered.sql b/tsl/test/sql/compression_sorted_merge_unordered.sql index 69578900ceb..f8c3b80702d 100644 --- a/tsl/test/sql/compression_sorted_merge_unordered.sql +++ b/tsl/test/sql/compression_sorted_merge_unordered.sql @@ -548,18 +548,44 @@ drop table t cascade; -- Should be optimized (all segmentby columns are pinned to a Const, orderby columns match) SET timescaledb.debug_require_batch_sorted_merge = 'force'; + +-- Encourage compressed indexscan as query sort keys may match compressed indexscan order +SET enable_seqscan=0; +SET enable_bitmapscan=0; + :PREFIX SELECT * FROM test_segby WHERE segby = 1 ORDER BY time DESC; :PREFIX SELECT * FROM test1 WHERE x1 = 1 AND x2 = 2 AND x5 = 0 ORDER BY x1, x2, x5, time DESC; +-- multikey orderby: can't use compressed indexscan order, need explicit sort :PREFIX SELECT * FROM test2 WHERE x1 = 1 AND x2 = 2 AND x5 = 0 ORDER BY time ASC, x3 DESC; :PREFIX SELECT * FROM test2 WHERE x1 = 1 AND x2 = 2 AND x5 = 0 ORDER BY x1 DESC, x2 DESC, x5 DESC, time DESC, x3 ASC; +-- We should choose Batch sorted merge with explicit sort on the leading orderby metadata column +SET enable_indexscan=0; +SET enable_bitmapscan=0; +SET enable_seqscan=1; +:PREFIX +SELECT * FROM test_segby WHERE segby = 1 ORDER BY time DESC; + +:PREFIX +SELECT * FROM test1 WHERE x1 = 1 AND x2 = 2 AND x5 = 0 ORDER BY x1, x2, x5, time DESC; + +:PREFIX +SELECT * FROM test2 WHERE x1 = 1 AND x2 = 2 AND x5 = 0 ORDER BY time ASC, x3 DESC; + +:PREFIX +SELECT * FROM test2 WHERE x1 = 1 AND x2 = 2 AND x5 = 0 ORDER BY x1 DESC, x2 DESC, x5 DESC, time DESC, x3 ASC; + +SET enable_seqscan=0; +SET enable_bitmapscan=0; +SET enable_indexscan=1; + -- No segmentby CREATE TABLE test_nosegby ( segby int NOT NULL, @@ -587,21 +613,28 @@ INSERT INTO test_nosegby (time, segby, val) values SELECT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('test_nosegby') chunk; -- Should be optimized (implicit NULLS first) +-- but can't use compressed indexscan order, need explicit sort: multikey orderby +-- Batch sort merge needs sort by (first_col1, first_col2) +-- while compressed input is sorted on (first_col1, last_col1, first_col2, last_col2) :PREFIX SELECT * FROM test_nosegby ORDER BY segby, time DESC; --- Should be optimized +-- Should be optimized but can't use compressed indexscan order: multikey orderby :PREFIX SELECT * FROM test_nosegby ORDER BY segby, time DESC NULLS FIRST; --- Should be optimized (backward scan, NULLS last) +-- Should be optimized but can't use compressed indexscan order (backward scan, NULLS last) :PREFIX SELECT * FROM test_nosegby ORDER BY segby DESC, time ASC NULLS LAST; --- Should be optimized (backward scan) +-- Should be optimized but can't use compressed indexscan order (backward scan) :PREFIX SELECT * FROM test_nosegby ORDER BY segby DESC; +-- Should be optimized and can use compressed indexscan order by (first_col1) +:PREFIX +SELECT * FROM test_nosegby ORDER BY segby; + set timescaledb.debug_require_batch_sorted_merge to 'forbid'; -- Should not be optimized (NULL order wrong) @@ -612,10 +645,199 @@ SELECT * FROM test_nosegby ORDER BY segby, time DESC NULLS LAST; :PREFIX SELECT * FROM test_nosegby ORDER BY segby DESC NULLS LAST; +-- Test for correct results on table with overlapping batches and segmentby pinned to a const +CREATE TABLE t2(time int NOT NULL, dev int NOT NULL, v int); +SELECT table_name FROM create_hypertable('t2', 'time', chunk_time_interval => 10000); +ALTER TABLE t2 SET (timescaledb.compress, timescaledb.compress_orderby='time DESC', timescaledb.compress_segmentby='dev'); + +INSERT INTO t2 (time, dev, v) values +(1, 1, 20), +(1, 2, 300), +(2, 3, 3000), +(3, 1, 40), +(4, 1, 10), +(4, 2, 100), +(6, 3, 2000), +(6, 1, 10), +(6, 2, 300), +(8, 1, 60), +(8, 2, 100), +(8, 3, 7000); + +INSERT INTO t2 (time, dev, v) values +(3, 1, 20), +(3, 2, 300), +(3, 3, 3000), +(5, 1, 40), +(5, 2, 100), +(5, 3, 1000), +(6, 1, 2000), +(6, 1, 10), +(6, 1, 200), +(7, 1, 60), +(7, 2, 100), +(7, 3, 7000); + +INSERT INTO t2 (time, dev, v) values +(5, 1, 20), +(5, 3, 3000), +(5, 2, 300), +(7, 2, 400), +(8, 1, 10), +(8, 2, 100), +(8, 3, 2000), +(10, 1, 10), +(11, 1, 300), +(14, 1, 60), +(14, 2, 100), +(14, 3, 7000); + +INSERT INTO t2 (time, dev, v) values +(9, 1, 20), +(9, 3, 300), +(9, 2, 30), +(9, 2, 40), +(10, 1, 10), +(10, 2, 100), +(10, 3, 2000), +(11, 1, 10), +(12, 1, 300), +(13, 1, 60), +(13, 2, 100), +(13, 3, 7000); + +SELECT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('t2') chunk; + +SET timescaledb.debug_require_batch_sorted_merge = 'force'; + +:PREFIX +SELECT dev, time FROM t2 WHERE dev = 1 ORDER BY time DESC; +SELECT dev, time FROM t2 WHERE dev = 1 ORDER BY time DESC; + +-- Can't use compressed indexscan order if orderby direction does not match: +-- we will need to sort on "last" metadata column instead of "first" in this case. +:PREFIX +SELECT dev, time FROM t2 WHERE dev = 1 ORDER BY time; +SELECT dev, time FROM t2 WHERE dev = 1 ORDER BY time; + +-- Predicates on non-segmentby columns +:PREFIX +SELECT dev, time FROM t2 WHERE time > 8 AND dev=1 ORDER BY dev, time DESC; +SELECT dev, time FROM t2 WHERE time > 8 AND dev=1 ORDER BY dev, time DESC; + +:PREFIX +SELECT dev, time, v FROM t2 WHERE v = 20 AND dev=1 ORDER BY dev, time DESC; +SELECT dev, time, v FROM t2 WHERE v = 20 AND dev=1 ORDER BY dev, time DESC; + +-- Rescan with lateral subquery +:PREFIX +SELECT dev, time +FROM (VALUES (1), (2), (3)) a(dv), + LATERAL (SELECT dev, time FROM t2 WHERE dev = a.dv ORDER BY time DESC) b; +SELECT dev, time +FROM (VALUES (1), (2), (3)) a(dv), + LATERAL (SELECT dev, time FROM t2 WHERE dev = a.dv ORDER BY time DESC) b; + +-- Test for correct results on table with overlapping batches and no segmentby +CREATE TABLE t2_noseg(time int NOT NULL, v int); +SELECT table_name FROM create_hypertable('t2_noseg', 'time', chunk_time_interval => 10000); +ALTER TABLE t2_noseg SET (timescaledb.compress, timescaledb.compress_orderby='time DESC'); + +INSERT INTO t2_noseg (time, v) values +(1, 20), +(1, 20), +(2, 20), +(3, 20), +(4, 10), +(4, 10), +(6, 10), +(6, 10), +(6, 10), +(8, 30), +(8, 30), +(8, 30); + +INSERT INTO t2_noseg (time, v) values +(3, 20), +(3, 20), +(3, 20), +(5, 10), +(5, 10), +(5, 10), +(6, 20), +(6, 20), +(6, 20), +(7, 10), +(7, 10), +(7, 10); + +INSERT INTO t2_noseg (time, v) values +(5, 10), +(5, 10), +(5, 10), +(7, 10), +(8, 30), +(8, 30), +(8, 30), +(10, 10), +(11, 30), +(14, 20), +(14, 20), +(14, 20); + +INSERT INTO t2_noseg (time, v) values +(9, 30), +(9, 30), +(9, 30), +(9, 30), +(10, 10), +(10, 10), +(10, 10), +(11, 30), +(12, 30), +(13, 30), +(13, 30), +(13, 30); + +SELECT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('t2_noseg') chunk; + +:PREFIX +SELECT time, v FROM t2_noseg ORDER BY time DESC; +SELECT time, v FROM t2_noseg ORDER BY time DESC; + +-- Can't use compressed indexscan order if orderby direction does not match: +-- we will need to sort on "last" metadata column instead of "first" in this case. +:PREFIX +SELECT time, v FROM t2_noseg ORDER BY time; +SELECT time, v FROM t2_noseg ORDER BY time; + +-- Predicates +:PREFIX +SELECT time, v FROM t2_noseg WHERE time > 8 ORDER BY time DESC; +SELECT time, v FROM t2_noseg WHERE time > 8 ORDER BY time DESC; + +:PREFIX +SELECT time, v FROM t2_noseg WHERE v > 10 ORDER BY time DESC; +SELECT time, v FROM t2_noseg WHERE v > 10 ORDER BY time DESC; + +-- Rescan with lateral subquery +:PREFIX +SELECT time, v +FROM (VALUES (10), (20), (30)) a(dv), + LATERAL (SELECT time, v FROM t2_noseg WHERE v = a.dv ORDER BY time DESC) b; +SELECT time, v +FROM (VALUES (10), (20), (30)) a(dv), + LATERAL (SELECT time, v FROM t2_noseg WHERE v = a.dv ORDER BY time DESC) b; + drop table test1 cascade; drop table test2 cascade; drop table test_segby cascade; drop table test_nosegby cascade; +drop table t2 cascade; +drop table t2_noseg cascade; RESET timescaledb.enable_direct_compress_insert; RESET timescaledb.debug_require_batch_sorted_merge; + +RESET enable_seqscan; +RESET enable_bitmapscan;