From 5586fdf0461e7de96e5080ddbb052fce40d72848 Mon Sep 17 00:00:00 2001 From: Melih Mutlu Date: Tue, 2 Jun 2026 14:41:00 +0300 Subject: [PATCH] Incremental manual refresh support refresh_continuous_aggregate() can now refresh incrementally in batches, matching the behavior the continuous aggregate policy already supports. Forced refresh is also incremental by default. Options available for incremental refresh with policies can be set for manual refresh via JSONB options: - `buckets_per_batch` (default 10, as policies) - `max_batches_per_execution` (default 0 = no limit) - `refresh_newest_first` Usage: Without options, it behaves similar to a refresh policy with default values: CALL refresh_continuous_aggregate('cagg', t1, t2); --- .unreleased/pr_9903 | 1 + tsl/src/bgw_policy/continuous_aggregate_api.c | 5 +- tsl/src/bgw_policy/job.c | 80 +- tsl/src/continuous_aggs/common.c | 18 + tsl/src/continuous_aggs/common.h | 8 + tsl/src/continuous_aggs/create.c | 20 +- tsl/src/continuous_aggs/refresh.c | 341 +++++++-- tsl/src/continuous_aggs/refresh.h | 16 +- tsl/test/expected/cagg-15.out | 6 +- tsl/test/expected/cagg-16.out | 6 +- tsl/test/expected/cagg-17.out | 6 +- tsl/test/expected/cagg-18.out | 6 +- tsl/test/expected/cagg_direct_compress.out | 23 +- tsl/test/expected/cagg_invalidation.out | 24 +- .../cagg_invalidation_variable_bucket.out | 9 +- tsl/test/expected/cagg_policy_run.out | 52 +- tsl/test/expected/cagg_query-15.out | 63 +- tsl/test/expected/cagg_query-16.out | 63 +- tsl/test/expected/cagg_query-17.out | 63 +- tsl/test/expected/cagg_query-18.out | 63 +- .../expected/cagg_query_using_merge-15.out | 63 +- .../expected/cagg_query_using_merge-16.out | 63 +- .../expected/cagg_query_using_merge-17.out | 63 +- .../expected/cagg_query_using_merge-18.out | 63 +- .../expected/cagg_refresh_incremental.out | 722 ++++++++++++++++++ .../expected/cagg_refresh_using_merge.out | 5 +- tsl/test/expected/cagg_watermark.out | 162 ++-- tsl/test/expected/telemetry_stats.out | 2 +- .../expected/cagg_cancel_kill_refresh.out | 8 +- .../expected/cagg_concurrent_register.out | 13 +- .../cagg_hierarchical_concurrent_refresh.out | 9 +- .../expected/cagg_incremental_concurrent.out | 222 +++++- .../cagg_refresh_cleanup_register.out | 36 +- .../specs/cagg_cancel_kill_refresh.spec | 4 +- .../specs/cagg_incremental_concurrent.spec | 29 +- .../specs/cagg_refresh_cleanup_register.spec | 8 +- tsl/test/shared/expected/constify_now-15.out | 8 +- tsl/test/shared/expected/constify_now-16.out | 8 +- tsl/test/shared/expected/constify_now-17.out | 8 +- tsl/test/shared/expected/constify_now-18.out | 8 +- tsl/test/sql/CMakeLists.txt | 1 + tsl/test/sql/cagg_direct_compress.sql | 23 +- tsl/test/sql/cagg_invalidation.sql | 19 +- .../sql/cagg_invalidation_variable_bucket.sql | 7 +- tsl/test/sql/cagg_refresh_incremental.sql | 491 ++++++++++++ tsl/test/sql/include/cagg_query_common.sql | 42 +- tsl/test/sql/telemetry_stats.sql | 2 +- 47 files changed, 2345 insertions(+), 617 deletions(-) create mode 100644 .unreleased/pr_9903 create mode 100644 tsl/test/expected/cagg_refresh_incremental.out create mode 100644 tsl/test/sql/cagg_refresh_incremental.sql diff --git a/.unreleased/pr_9903 b/.unreleased/pr_9903 new file mode 100644 index 00000000000..fdfdd9e1bf6 --- /dev/null +++ b/.unreleased/pr_9903 @@ -0,0 +1 @@ +Implements: #9903 Incremental refresh for refresh_continuous_aggregate() diff --git a/tsl/src/bgw_policy/continuous_aggregate_api.c b/tsl/src/bgw_policy/continuous_aggregate_api.c index 0e249157dc1..cc60a2ded5f 100644 --- a/tsl/src/bgw_policy/continuous_aggregate_api.c +++ b/tsl/src/bgw_policy/continuous_aggregate_api.c @@ -21,6 +21,7 @@ #include "bgw_policy/job_api.h" #include "bgw_policy/policies_v2.h" #include "bgw_policy/policy_utils.h" +#include "continuous_aggs/refresh.h" #include "dimension.h" #include "guc.h" #include "jsonb_utils.h" @@ -32,12 +33,8 @@ /* Default max runtime for a continuous aggregate jobs is unlimited for now */ #define DEFAULT_MAX_RUNTIME \ DatumGetIntervalP(DirectFunctionCall3(interval_in, CStringGetDatum("0"), InvalidOid, -1)) -/* Default buckets per batch is 1, which means that the job will refresh 1 bucket at a time */ -#define DEFAULT_BUCKETS_PER_BATCH 10 /* Default max batches per execution is 0, which means no limit */ #define DEFAULT_MAX_BATCHES_PER_EXECUTION 0 -/* Default refresh newest first is true, which means from newest data to the oldest */ -#define DEFAULT_REFRESH_NEWEST_FIRST true /* Default compress after refresh is false, which means compression does not run after refresh */ #define DEFAULT_COMPRESS_AFTER_REFRESH false diff --git a/tsl/src/bgw_policy/job.c b/tsl/src/bgw_policy/job.c index 9c2bd8950b8..4e9eb9a3dfd 100644 --- a/tsl/src/bgw_policy/job.c +++ b/tsl/src/bgw_policy/job.c @@ -422,74 +422,18 @@ policy_refresh_cagg_execute(int32 job_id, Jsonb *config) PGC_S_SESSION); } - ContinuousAggRefreshContext context = { .callctx = CAGG_REFRESH_POLICY, .job_id = job_id }; - - /* Try to split window range into a list of ranges */ - List *refresh_window_list = continuous_agg_split_refresh_window(policy_data.cagg, - &policy_data.refresh_window, - policy_data.buckets_per_batch); - if (refresh_window_list == NIL) - { - refresh_window_list = lappend(refresh_window_list, &policy_data.refresh_window); - } - else - { - context.callctx = CAGG_REFRESH_POLICY_BATCHED; - } - - context.number_of_batches = list_length(refresh_window_list); - - /* - * The list is always built oldest-first. When refresh_newest_first is true we - * iterate from the last element down to the first using index-based access so - * that no reversal copy of the list is needed. - */ - int32 processing_batch = 0; - int32 nbatches = list_length(refresh_window_list); - int32 batch_start = policy_data.refresh_newest_first ? nbatches - 1 : 0; - int32 batch_end = policy_data.refresh_newest_first ? -1 : nbatches; - int32 batch_step = policy_data.refresh_newest_first ? -1 : 1; - for (int32 batch_idx = batch_start; batch_idx != batch_end; batch_idx += batch_step) - { - InternalTimeRange *refresh_window = - (InternalTimeRange *) list_nth(refresh_window_list, batch_idx); - elog(DEBUG1, - "refreshing continuous aggregate \"%s\" from %s to %s", - NameStr(policy_data.cagg->data.user_view_name), - ts_internal_to_time_string(refresh_window->start, refresh_window->type), - ts_internal_to_time_string(refresh_window->end, refresh_window->type)); - - context.processing_batch = ++processing_batch; - - /* extend_last_bucket must only apply to the boundary batch — the one - * whose window abuts the adjacent policy. For newest-first ordering - * that is batch 1; for oldest-first it is the final batch. - * In non-batched mode (single batch) the one batch is always the boundary. */ - bool apply_extend = - extend_last_bucket && - (policy_data.refresh_newest_first ? processing_batch == 1 : - processing_batch == context.number_of_batches); - - continuous_agg_refresh_internal(policy_data.cagg, - refresh_window, - context, - refresh_window->start_isnull, - refresh_window->end_isnull, - (context.callctx != CAGG_REFRESH_POLICY_BATCHED), - false, /* force */ - apply_extend); - DEBUG_ERROR_INJECTION(psprintf("cagg_policy_batch_%d_after_refresh", processing_batch)); - if (processing_batch >= policy_data.max_batches_per_execution && - processing_batch < context.number_of_batches && - policy_data.max_batches_per_execution > 0) - { - elog(LOG, - "reached maximum number of batches per execution (%d), batches not processed (%d)", - policy_data.max_batches_per_execution, - context.number_of_batches - processing_batch); - break; - } - } + ContinuousAggRefreshContext context = { + .callctx = CAGG_REFRESH_POLICY, + .job_id = job_id, + .buckets_per_batch = policy_data.buckets_per_batch, + .max_batches_per_execution = policy_data.max_batches_per_execution, + .refresh_newest_first = policy_data.refresh_newest_first, + }; + + continuous_agg_refresh_batched(policy_data.cagg, + &policy_data.refresh_window, + context, + extend_last_bucket); if (!policy_data.include_tiered_data_isnull) { diff --git a/tsl/src/continuous_aggs/common.c b/tsl/src/continuous_aggs/common.c index 1f73f6a76fd..6095c3fb3ba 100644 --- a/tsl/src/continuous_aggs/common.c +++ b/tsl/src/continuous_aggs/common.c @@ -2102,3 +2102,21 @@ cagg_find_groupingcols(ContinuousAgg *agg, Hypertable *mat_ht) } return retlist; } + +void +emit_up_to_date_notice(const ContinuousAgg *cagg, const ContinuousAggRefreshContext context) +{ + switch (context.callctx) + { + case CAGG_REFRESH_WINDOW: + case CAGG_REFRESH_CREATION: + case CAGG_REFRESH_WINDOW_BATCHED: + elog(NOTICE, + "continuous aggregate \"%s\" is already up-to-date", + NameStr(cagg->data.user_view_name)); + break; + case CAGG_REFRESH_POLICY: + case CAGG_REFRESH_POLICY_BATCHED: + break; + } +} diff --git a/tsl/src/continuous_aggs/common.h b/tsl/src/continuous_aggs/common.h index b1f7cac7e82..2710dcccc4e 100644 --- a/tsl/src/continuous_aggs/common.h +++ b/tsl/src/continuous_aggs/common.h @@ -79,6 +79,7 @@ typedef enum ContinuousAggRefreshCallContext { CAGG_REFRESH_CREATION, CAGG_REFRESH_WINDOW, + CAGG_REFRESH_WINDOW_BATCHED, CAGG_REFRESH_POLICY, CAGG_REFRESH_POLICY_BATCHED } ContinuousAggRefreshCallContext; @@ -89,6 +90,11 @@ typedef struct ContinuousAggRefreshContext int32 job_id; int32 processing_batch; int32 number_of_batches; + /* Batch configuration */ + int32 buckets_per_batch; /* 0 = disabled */ + int32 max_batches_per_execution; /* 0 = no limit */ + bool refresh_newest_first; + bool force; /* re-materialize the whole window, ignoring invalidations */ } ContinuousAggRefreshContext; #define IS_TIME_BUCKET_INFO_TIME_BASED(bucket_function) \ @@ -173,3 +179,5 @@ extern bool caggtimebucket_validate_common(ContinuousAggBucketFunction *bf, List List *targetList, List *rtable, int ht_partcolno, StringInfo msg, bool is_cagg_create, const bool for_rewrites); +extern void emit_up_to_date_notice(const ContinuousAgg *cagg, + const ContinuousAggRefreshContext context); diff --git a/tsl/src/continuous_aggs/create.c b/tsl/src/continuous_aggs/create.c index 5a1b63ca4b6..a5f0bbe1e48 100644 --- a/tsl/src/continuous_aggs/create.c +++ b/tsl/src/continuous_aggs/create.c @@ -857,8 +857,11 @@ tsl_process_continuous_agg_viewstmt(Node *node, const char *query_string, void * if (!stmt->into->skipData) { + bool refreshed = false; InternalTimeRange refresh_window = { .type = InvalidOid, + .start_isnull = true, + .end_isnull = true, }; /* @@ -887,14 +890,15 @@ tsl_process_continuous_agg_viewstmt(Node *node, const char *query_string, void * refresh_window.end = ts_time_get_noend_or_max(refresh_window.type); ContinuousAggRefreshContext context = { .callctx = CAGG_REFRESH_CREATION }; - continuous_agg_refresh_internal(cagg, - &refresh_window, - context, - true, /* start_isnull */ - true, /* end_isnull */ - true, /* bucketing_refresh_window */ - false, /* force */ - false /*extend_last_bucket*/); + refreshed = continuous_agg_refresh_internal(cagg, + &refresh_window, + context, + true, /* bucketing_refresh_window */ + false /*extend_last_bucket*/); + if (!refreshed) + { + emit_up_to_date_notice(cagg, context); + } } return DDL_DONE; diff --git a/tsl/src/continuous_aggs/refresh.c b/tsl/src/continuous_aggs/refresh.c index c12e1b2db96..f281bdb7226 100644 --- a/tsl/src/continuous_aggs/refresh.c +++ b/tsl/src/continuous_aggs/refresh.c @@ -20,6 +20,7 @@ #include #include +#include "bgw_policy/policies_v2.h" #include "debug_point.h" #include "dimension.h" #include "dimension_slice.h" @@ -79,12 +80,10 @@ static void continuous_agg_refresh_with_window(const ContinuousAgg *cagg, const InvalidationStore *invalidations, const ContinuousAggRefreshContext context, bool bucketing_refresh_window); -static void emit_up_to_date_notice(const ContinuousAgg *cagg, - const ContinuousAggRefreshContext context); static bool process_cagg_invalidations_and_refresh(const ContinuousAgg *cagg, const InternalTimeRange *refresh_window, const ContinuousAggRefreshContext context, - bool bucketing_refresh_window, bool force); + bool bucketing_refresh_window); static Hypertable * cagg_get_hypertable_or_fail(int32 hypertable_id) { @@ -624,6 +623,111 @@ continuous_agg_refresh_with_window(const ContinuousAgg *cagg, } #define REFRESH_FUNCTION_NAME "refresh_continuous_aggregate()" + +/* + * Refresh a continuous aggregate over a window, splitting it into batches when + * incremental refresh is enabled. + * + * This is the shared entry point used by both the manual refresh + * (refresh_continuous_aggregate) and the continuous aggregate refresh policy. + * + * For a normal refresh, batching is driven by the invalidation logs, so it only + * produces batches for the regions that actually need to be refreshed. A forced + * refresh ignores the invalidation logs and instead batches every bucket-aligned + * chunk of the window that contains data, so the whole window is re-materialized. + * Set buckets_per_batch to 0 for a single atomic pass. + */ +void +continuous_agg_refresh_batched(ContinuousAgg *cagg, InternalTimeRange *refresh_window, + ContinuousAggRefreshContext context, bool extend_last_bucket) +{ + List *refresh_window_list = continuous_agg_split_refresh_window(cagg, + refresh_window, + context.buckets_per_batch, + context.force); + + bool batched = (refresh_window_list != NIL); + if (!batched) + { + /* No batching: refresh the whole window as a single batch */ + refresh_window_list = lappend(refresh_window_list, refresh_window); + } + else + { + /* Batches are already bucket-aligned by the split function */ + switch (context.callctx) + { + case CAGG_REFRESH_POLICY: + context.callctx = CAGG_REFRESH_POLICY_BATCHED; + break; + case CAGG_REFRESH_WINDOW: + context.callctx = CAGG_REFRESH_WINDOW_BATCHED; + break; + default: + break; + } + } + + context.number_of_batches = list_length(refresh_window_list); + + /* + * The list is always built oldest-first. When refresh_newest_first is true we + * iterate from the last element down to the first using index-based access so + * that no reversal copy of the list is needed. + */ + int32 processing_batch = 0; + int32 nbatches = context.number_of_batches; + int32 batch_start = context.refresh_newest_first ? nbatches - 1 : 0; + int32 batch_end = context.refresh_newest_first ? -1 : nbatches; + int32 batch_step = context.refresh_newest_first ? -1 : 1; + bool any_refreshed = false; + for (int32 batch_idx = batch_start; batch_idx != batch_end; batch_idx += batch_step) + { + InternalTimeRange *batch_window = + (InternalTimeRange *) list_nth(refresh_window_list, batch_idx); + + elog(DEBUG1, + "refreshing continuous aggregate \"%s\" from %s to %s", + NameStr(cagg->data.user_view_name), + ts_internal_to_time_string(batch_window->start, batch_window->type), + ts_internal_to_time_string(batch_window->end, batch_window->type)); + + context.processing_batch = ++processing_batch; + + /* extend_last_bucket must only apply to the boundary batch -- the one + * whose window abuts the adjacent policy. For newest-first ordering + * that is batch 1; for oldest-first it is the final batch. + * In non-batched mode (single batch) the one batch is always the boundary. */ + bool apply_extend = + extend_last_bucket && + (context.refresh_newest_first ? processing_batch == 1 : + processing_batch == context.number_of_batches); + + any_refreshed |= continuous_agg_refresh_internal(cagg, + batch_window, + context, + !batched, /* bucketing_refresh_window */ + apply_extend); + DEBUG_ERROR_INJECTION(psprintf("cagg_policy_batch_%d_after_refresh", processing_batch)); + + if (context.max_batches_per_execution > 0 && + processing_batch >= context.max_batches_per_execution && + processing_batch < context.number_of_batches) + { + elog(LOG, + "reached maximum number of batches per execution (%d), batches not processed (%d)", + context.max_batches_per_execution, + context.number_of_batches - processing_batch); + break; + } + } + + if (!any_refreshed) + { + emit_up_to_date_notice(cagg, context); + } +} + /* * Refresh a continuous aggregate across the given window. */ @@ -642,6 +746,17 @@ continuous_agg_refresh(PG_FUNCTION_ARGS) cagg = cagg_get_by_relid_or_fail(cagg_relid); refresh_window.type = cagg->partition_type; + /* + * Check ownership up front, before any work that touches the source + * hypertable (e.g. computing the batches for an incremental refresh). + */ + if (!object_ownercheck(RelationRelationId, cagg_relid, GetUserId())) + { + aclcheck_error(ACLCHECK_NOT_OWNER, + get_relkind_objtype(get_rel_relkind(cagg_relid)), + get_rel_name(cagg_relid)); + } + if (!PG_ARGISNULL(1)) { refresh_window.start = ts_time_value_from_arg(PG_GETARG_DATUM(1), @@ -653,6 +768,7 @@ continuous_agg_refresh(PG_FUNCTION_ARGS) { /* get min time for a cagg depending of the primary partition type */ refresh_window.start = cagg_get_time_min(cagg); + refresh_window.start_isnull = true; } if (!PG_ARGISNULL(2)) @@ -665,43 +781,83 @@ continuous_agg_refresh(PG_FUNCTION_ARGS) else { refresh_window.end = ts_time_get_noend_or_max(refresh_window.type); + refresh_window.end_isnull = true; } - ContinuousAggRefreshContext context = { .callctx = CAGG_REFRESH_WINDOW }; - continuous_agg_refresh_internal(cagg, - &refresh_window, - context, - PG_ARGISNULL(1), - PG_ARGISNULL(2), - true, - force, - false /*extend_last_bucket*/); + /* + * Manual refreshes batch by default (DEFAULT_BUCKETS_PER_BATCH), matching the + * continuous aggregate policy. Callers can override through the options JSONB; + * set buckets_per_batch to 0 to force a single-pass atomic refresh. + */ + int32 buckets_per_batch = DEFAULT_BUCKETS_PER_BATCH; + int32 max_batches_per_execution = 0; + bool refresh_newest_first = DEFAULT_REFRESH_NEWEST_FIRST; - PG_RETURN_VOID(); -} + if (!PG_ARGISNULL(4)) + { + Jsonb *options = PG_GETARG_JSONB_P(4); + bool found; -static void -emit_up_to_date_notice(const ContinuousAgg *cagg, const ContinuousAggRefreshContext context) -{ - switch (context.callctx) + int32 v = ts_jsonb_get_int32_field(options, POL_REFRESH_CONF_KEY_BUCKETS_PER_BATCH, &found); + if (found) + { + buckets_per_batch = v; + } + + v = ts_jsonb_get_int32_field(options, + POL_REFRESH_CONF_KEY_MAX_BATCHES_PER_EXECUTION, + &found); + if (found) + { + max_batches_per_execution = v; + } + + bool b = + ts_jsonb_get_bool_field(options, POL_REFRESH_CONF_KEY_REFRESH_NEWEST_FIRST, &found); + if (found) + { + refresh_newest_first = b; + } + } + + if (buckets_per_batch < 0) { - case CAGG_REFRESH_WINDOW: - case CAGG_REFRESH_CREATION: - elog(NOTICE, - "continuous aggregate \"%s\" is already up-to-date", - NameStr(cagg->data.user_view_name)); - break; - case CAGG_REFRESH_POLICY: - case CAGG_REFRESH_POLICY_BATCHED: - break; + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid buckets per batch"), + errdetail("buckets_per_batch: %d", buckets_per_batch), + errhint("The buckets per batch should be greater than or equal to zero."))); + } + + if (max_batches_per_execution < 0) + { + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid max batches per execution"), + errdetail("max_batches_per_execution: %d", max_batches_per_execution), + errhint( + "The max batches per execution should be greater than or equal to zero."))); } + + ContinuousAggRefreshContext context = { + .callctx = CAGG_REFRESH_WINDOW, + .buckets_per_batch = buckets_per_batch, + .max_batches_per_execution = max_batches_per_execution, + .refresh_newest_first = refresh_newest_first, + .force = force, + }; + + continuous_agg_refresh_batched(cagg, &refresh_window, context, false /*extend_last_bucket*/); + DEBUG_WAITPOINT("after_cagg_refresh_window"); + + PG_RETURN_VOID(); } static bool process_cagg_invalidations_and_refresh(const ContinuousAgg *cagg, const InternalTimeRange *refresh_window, const ContinuousAggRefreshContext context, - bool bucketing_refresh_window, bool force) + bool bucketing_refresh_window) { /* Lock the continuous aggregate's catalog table entry to protect against concurrent refreshes * on the same cagg processing the cagg invalidation logs for that CAgg. @@ -721,7 +877,7 @@ process_cagg_invalidations_and_refresh(const ContinuousAgg *cagg, DEBUG_WAITPOINT("after_process_cagg_invalidations_for_refresh_lock"); InvalidationStore *invalidations = - collect_and_delete_cagg_invalidations_in_window(cagg, refresh_window, force); + collect_and_delete_cagg_invalidations_in_window(cagg, refresh_window, context.force); if (invalidations != NULL) { @@ -840,12 +996,11 @@ rollback_and_error(const ContinuousAgg *cagg, CaggRefreshSpiContext *cagg_spi_ct ThrowErrorData(edata); } -void +bool continuous_agg_refresh_internal(const ContinuousAgg *cagg_arg, const InternalTimeRange *refresh_window_arg, - const ContinuousAggRefreshContext context, const bool start_isnull, - const bool end_isnull, bool bucketing_refresh_window, bool force, - bool extend_last_bucket) + const ContinuousAggRefreshContext context, + bool bucketing_refresh_window, bool extend_last_bucket) { const ContinuousAgg *volatile cagg = cagg_arg; int32 mat_id = cagg->data.mat_hypertable_id; @@ -862,7 +1017,8 @@ continuous_agg_refresh_internal(const ContinuousAgg *cagg_arg, continuous_agg_refresh_spi_setup_and_connect(&cagg_spi_ctx); /* No bucketing when open ended */ - if (bucketing_refresh_window && !(start_isnull && end_isnull)) + if (bucketing_refresh_window && + !(refresh_window_arg->start_isnull && refresh_window_arg->end_isnull)) { refresh_window = compute_inscribed_bucketed_refresh_window(refresh_window_arg, cagg->bucket_function); @@ -875,7 +1031,7 @@ continuous_agg_refresh_internal(const ContinuousAgg *cagg_arg, * bucket We don't need to do this when the CAgg is created WITH DATA, or manually * refreshed */ - if (extend_last_bucket && !(start_isnull && end_isnull)) + if (extend_last_bucket && !(refresh_window_arg->start_isnull && refresh_window_arg->end_isnull)) { refresh_window.end = cagg_next_bucket_start(refresh_window.end, refresh_window.type, cagg->bucket_function); @@ -1028,8 +1184,7 @@ continuous_agg_refresh_internal(const ContinuousAgg *cagg_arg, refreshed = process_cagg_invalidations_and_refresh(cagg, &refresh_window, context, - bucketing_refresh_window, - force); + bucketing_refresh_window); DEBUG_WAITPOINT("after_process_cagg_materializations"); } @@ -1050,11 +1205,7 @@ continuous_agg_refresh_internal(const ContinuousAgg *cagg_arg, if (edata) { rollback_and_error(cagg, &cagg_spi_ctx, (ErrorData *) edata); - return; - } - if (!refreshed) - { - emit_up_to_date_notice(cagg, context); + return false; } cleanup_before_cagg_refresh_exit(cagg, &cagg_spi_ctx); @@ -1065,6 +1216,8 @@ continuous_agg_refresh_internal(const ContinuousAgg *cagg_arg, { elog(ERROR, "SPI_finish failed: %s", SPI_result_code_string(rc)); } + + return refreshed; } static void @@ -1085,7 +1238,7 @@ debug_refresh_window(const ContinuousAgg *cagg, const InternalTimeRange *refresh List * continuous_agg_split_refresh_window(ContinuousAgg *cagg, InternalTimeRange *original_refresh_window, - int32 buckets_per_batch) + int32 buckets_per_batch, bool force) { /* Do not produce batches when the number of buckets per batch is zero (disabled) */ if (buckets_per_batch == 0) @@ -1224,42 +1377,61 @@ continuous_agg_split_refresh_window(ContinuousAgg *cagg, InternalTimeRange *orig " ORDER BY 1 ASC"; List *refresh_window_list = NIL; - int res; MemoryContext oldcontext = CurrentMemoryContext; - if (SPI_connect() != SPI_OK_CONNECT) - { - elog(ERROR, "could not connect to SPI"); - } + int inval_count; + TupleDesc inval_tupdesc = NULL; + int save_nestlevel = 0; + bool spi_connected = false; - int save_nestlevel = NewGUCNestLevel(); - RestrictSearchPath(); + /* + * A forced refresh must re-materialize the whole window regardless of the + * invalidation logs. We model that as a single synthetic invalidation + * spanning the entire refresh window and skip the invalidation-log query + * altogether. + */ + if (!force) + { + if (SPI_connect() != SPI_OK_CONNECT) + { + elog(ERROR, "could not connect to SPI"); + } + spi_connected = true; + + save_nestlevel = NewGUCNestLevel(); + RestrictSearchPath(); + + Oid inval_types[] = { INT4OID, INT4OID, INT8OID, INT8OID, INT8OID }; + Datum inval_values[] = { Int32GetDatum(cagg->data.mat_hypertable_id), + Int32GetDatum(ht->fd.id), + Int64GetDatum(CAGG_INVALIDATION_WRONG_GREATEST_VALUE), + Int64GetDatum(refresh_window.start), + Int64GetDatum(refresh_window.end) }; + char inval_nulls[] = { false, false, false, false, false }; + + int res = SPI_execute_with_args(inval_query, + 5, + inval_types, + inval_values, + inval_nulls, + true /* read_only */, + 0); + if (res < 0) + { + elog(ERROR, "%s: could not fetch invalidation ranges for cagg refresh", __func__); + } - Oid inval_types[] = { INT4OID, INT4OID, INT8OID, INT8OID, INT8OID }; - Datum inval_values[] = { Int32GetDatum(cagg->data.mat_hypertable_id), - Int32GetDatum(ht->fd.id), - Int64GetDatum(CAGG_INVALIDATION_WRONG_GREATEST_VALUE), - Int64GetDatum(refresh_window.start), - Int64GetDatum(refresh_window.end) }; - char inval_nulls[] = { false, false, false, false, false }; - - res = SPI_execute_with_args(inval_query, - 5, - inval_types, - inval_values, - inval_nulls, - true /* read_only */, - 0); - if (res < 0) + Assert(SPI_processed <= INT_MAX); + inval_count = (int) SPI_processed; + Assert(SPI_tuptable != NULL); + inval_tupdesc = SPI_tuptable->tupdesc; + } + else { - elog(ERROR, "%s: could not fetch invalidation ranges for cagg refresh", __func__); + /* Single synthetic invalidation covering the whole refresh window. */ + inval_count = 1; } - Assert(SPI_processed <= INT_MAX); - int inval_count = (int) SPI_processed; - Assert(SPI_tuptable != NULL); - TupleDesc inval_tupdesc = SPI_tuptable->tupdesc; - /* * Open a streaming catalog scan for dimension slices overlapping the refresh * window. @@ -1312,7 +1484,15 @@ continuous_agg_split_refresh_window(ContinuousAgg *cagg, InternalTimeRange *orig int inval_idx = 0; int64 inval_low = 0, inval_high = 0; - if (inval_idx < inval_count) + if (force) + { + /* Synthetic invalidation spanning the whole window. It stays active for + * the entire merge loop (inval_high == window end, so it is never + * advanced past), making every data-backed chunk eligible for a batch. */ + inval_low = refresh_window.start; + inval_high = refresh_window.end; + } + else if (inval_idx < inval_count) { bool isnull; inval_low = @@ -1413,13 +1593,16 @@ continuous_agg_split_refresh_window(ContinuousAgg *cagg, InternalTimeRange *orig ts_scan_iterator_close(&dim_it); - /* Done with SPI */ - /* Restore search_path */ - AtEOXact_GUC(false, save_nestlevel); - res = SPI_finish(); - if (res != SPI_OK_FINISH) + /* Done with SPI (only used for the invalidation-log query in the non-forced + * path; a forced refresh never connects). Restore search_path. */ + if (spi_connected) { - elog(ERROR, "SPI_finish failed: %s", SPI_result_code_string(res)); + AtEOXact_GUC(false, save_nestlevel); + int res = SPI_finish(); + if (res != SPI_OK_FINISH) + { + elog(ERROR, "SPI_finish failed: %s", SPI_result_code_string(res)); + } } /* diff --git a/tsl/src/continuous_aggs/refresh.h b/tsl/src/continuous_aggs/refresh.h index a10e7f0a17b..8540ffbdf82 100644 --- a/tsl/src/continuous_aggs/refresh.h +++ b/tsl/src/continuous_aggs/refresh.h @@ -12,16 +12,22 @@ #include "materialize.h" #include "ts_catalog/continuous_agg.h" +/* Default buckets per batch for incremental refresh */ +#define DEFAULT_BUCKETS_PER_BATCH 10 +/* Default refresh newest first */ +#define DEFAULT_REFRESH_NEWEST_FIRST true + extern Datum continuous_agg_refresh(PG_FUNCTION_ARGS); -extern void continuous_agg_refresh_internal(const ContinuousAgg *cagg_arg, +extern void continuous_agg_refresh_batched(ContinuousAgg *cagg, InternalTimeRange *refresh_window, + ContinuousAggRefreshContext context, + bool extend_last_bucket); +extern bool continuous_agg_refresh_internal(const ContinuousAgg *cagg_arg, const InternalTimeRange *refresh_window, const ContinuousAggRefreshContext context, - const bool start_isnull, const bool end_isnull, - bool bucketing_refresh_window, bool force, - bool extend_last_bucket); + bool bucketing_refresh_window, bool extend_last_bucket); extern List *continuous_agg_split_refresh_window(ContinuousAgg *cagg, InternalTimeRange *original_refresh_window, - int32 buckets_per_batch); + int32 buckets_per_batch, bool force); InternalTimeRange compute_circumscribed_bucketed_refresh_window(const InternalTimeRange *const refresh_window, const ContinuousAggBucketFunction *bucket_function); diff --git a/tsl/test/expected/cagg-15.out b/tsl/test/expected/cagg-15.out index 9abe697c706..9691854b5d5 100644 --- a/tsl/test/expected/cagg-15.out +++ b/tsl/test/expected/cagg-15.out @@ -1445,7 +1445,6 @@ insert into raw_data select '2000-05-01 00:00+0','Q3', 0, 0; \set ON_ERROR_STOP 0 CALL refresh_continuous_aggregate('search_query_count_3', NULL, '2000-06-01 00:00+0'::timestamptz); CALL refresh_continuous_aggregate('search_query_count_3', '2000-05-01 00:00+0'::timestamptz, '2000-06-01 00:00+0'::timestamptz); -NOTICE: continuous aggregate "search_query_count_3" is already up-to-date \set ON_ERROR_STOP 1 --insert row insert into raw_data select '2001-05-10 00:00+0','Q3', 100, 100; @@ -1471,8 +1470,9 @@ WHERE materialization_id = :'MAT_HTID' ORDER BY 1, 2,3; materialization_id | lowest_modified_value | greatest_modified_value --------------------+-----------------------+------------------------- 41 | -9223372036854775808 | -210866803200000001 - 41 | 959817600000000 | 988675199999999 - 41 | 991353600000000 | 9223372036854775807 + 41 | 947376060000000 | 956447999999999 + 41 | 959817600000000 | 988847999999999 + 41 | 990144000000000 | 9223372036854775807 SELECT * from search_query_count_3 WHERE bucket > '2001-01-01' diff --git a/tsl/test/expected/cagg-16.out b/tsl/test/expected/cagg-16.out index 0f6cde69817..7130e53a3e9 100644 --- a/tsl/test/expected/cagg-16.out +++ b/tsl/test/expected/cagg-16.out @@ -1445,7 +1445,6 @@ insert into raw_data select '2000-05-01 00:00+0','Q3', 0, 0; \set ON_ERROR_STOP 0 CALL refresh_continuous_aggregate('search_query_count_3', NULL, '2000-06-01 00:00+0'::timestamptz); CALL refresh_continuous_aggregate('search_query_count_3', '2000-05-01 00:00+0'::timestamptz, '2000-06-01 00:00+0'::timestamptz); -NOTICE: continuous aggregate "search_query_count_3" is already up-to-date \set ON_ERROR_STOP 1 --insert row insert into raw_data select '2001-05-10 00:00+0','Q3', 100, 100; @@ -1471,8 +1470,9 @@ WHERE materialization_id = :'MAT_HTID' ORDER BY 1, 2,3; materialization_id | lowest_modified_value | greatest_modified_value --------------------+-----------------------+------------------------- 41 | -9223372036854775808 | -210866803200000001 - 41 | 959817600000000 | 988675199999999 - 41 | 991353600000000 | 9223372036854775807 + 41 | 947376060000000 | 956447999999999 + 41 | 959817600000000 | 988847999999999 + 41 | 990144000000000 | 9223372036854775807 SELECT * from search_query_count_3 WHERE bucket > '2001-01-01' diff --git a/tsl/test/expected/cagg-17.out b/tsl/test/expected/cagg-17.out index 0f6cde69817..7130e53a3e9 100644 --- a/tsl/test/expected/cagg-17.out +++ b/tsl/test/expected/cagg-17.out @@ -1445,7 +1445,6 @@ insert into raw_data select '2000-05-01 00:00+0','Q3', 0, 0; \set ON_ERROR_STOP 0 CALL refresh_continuous_aggregate('search_query_count_3', NULL, '2000-06-01 00:00+0'::timestamptz); CALL refresh_continuous_aggregate('search_query_count_3', '2000-05-01 00:00+0'::timestamptz, '2000-06-01 00:00+0'::timestamptz); -NOTICE: continuous aggregate "search_query_count_3" is already up-to-date \set ON_ERROR_STOP 1 --insert row insert into raw_data select '2001-05-10 00:00+0','Q3', 100, 100; @@ -1471,8 +1470,9 @@ WHERE materialization_id = :'MAT_HTID' ORDER BY 1, 2,3; materialization_id | lowest_modified_value | greatest_modified_value --------------------+-----------------------+------------------------- 41 | -9223372036854775808 | -210866803200000001 - 41 | 959817600000000 | 988675199999999 - 41 | 991353600000000 | 9223372036854775807 + 41 | 947376060000000 | 956447999999999 + 41 | 959817600000000 | 988847999999999 + 41 | 990144000000000 | 9223372036854775807 SELECT * from search_query_count_3 WHERE bucket > '2001-01-01' diff --git a/tsl/test/expected/cagg-18.out b/tsl/test/expected/cagg-18.out index 0f6cde69817..7130e53a3e9 100644 --- a/tsl/test/expected/cagg-18.out +++ b/tsl/test/expected/cagg-18.out @@ -1445,7 +1445,6 @@ insert into raw_data select '2000-05-01 00:00+0','Q3', 0, 0; \set ON_ERROR_STOP 0 CALL refresh_continuous_aggregate('search_query_count_3', NULL, '2000-06-01 00:00+0'::timestamptz); CALL refresh_continuous_aggregate('search_query_count_3', '2000-05-01 00:00+0'::timestamptz, '2000-06-01 00:00+0'::timestamptz); -NOTICE: continuous aggregate "search_query_count_3" is already up-to-date \set ON_ERROR_STOP 1 --insert row insert into raw_data select '2001-05-10 00:00+0','Q3', 100, 100; @@ -1471,8 +1470,9 @@ WHERE materialization_id = :'MAT_HTID' ORDER BY 1, 2,3; materialization_id | lowest_modified_value | greatest_modified_value --------------------+-----------------------+------------------------- 41 | -9223372036854775808 | -210866803200000001 - 41 | 959817600000000 | 988675199999999 - 41 | 991353600000000 | 9223372036854775807 + 41 | 947376060000000 | 956447999999999 + 41 | 959817600000000 | 988847999999999 + 41 | 990144000000000 | 9223372036854775807 SELECT * from search_query_count_3 WHERE bucket > '2001-01-01' diff --git a/tsl/test/expected/cagg_direct_compress.out b/tsl/test/expected/cagg_direct_compress.out index d037efd8685..4e370218cc3 100644 --- a/tsl/test/expected/cagg_direct_compress.out +++ b/tsl/test/expected/cagg_direct_compress.out @@ -26,7 +26,8 @@ SELECT FROM conditions GROUP BY 1, 2 WITH NO DATA; -CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL); +-- Setting buckets_per_batch to a high value to bypass "disabling direct compress because of too small batch size" situation +CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_hourly') chunk; chunk_status_text ------------------- @@ -38,7 +39,7 @@ ALTER MATERIALIZED VIEW conditions_hourly SET (timescaledb.compress); NOTICE: defaulting compress_orderby to bucket,device_id -- Enable direct compress on cagg refresh SET timescaledb.enable_direct_compress_on_cagg_refresh TO on; -CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL); +CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_hourly') chunk; chunk_status_text ------------------- @@ -48,7 +49,7 @@ SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks INSERT INTO conditions SELECT t, d::text, 1, 1 FROM generate_series('2025-12-15 00:00:00+00'::timestamptz - interval '1 year', '2025-12-15 00:00:00+00'::timestamptz, interval '1 hour') AS t, generate_series(1, 10) AS d; SET timescaledb.enable_direct_compress_on_cagg_refresh TO off; -CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL); +CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_hourly') chunk; chunk_status_text ---------------------- @@ -71,7 +72,7 @@ SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks INSERT INTO conditions SELECT t, d::text, 1, 1 FROM generate_series('2025-12-15 00:00:00+00'::timestamptz - interval '1 year', '2025-12-15 00:00:00+00'::timestamptz, interval '1 hour') AS t, generate_series(1, 10) AS d; SET timescaledb.enable_direct_compress_on_cagg_refresh TO on; -CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL); +CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_hourly') chunk; chunk_status_text ------------------- @@ -98,14 +99,14 @@ INSERT INTO conditions SELECT t, d::text, 1, 1 FROM generate_series('2025-12-15 00:00:00+00'::timestamptz - interval '1 year', '2025-12-15 00:00:00+00'::timestamptz, interval '1 hour') AS t, generate_series(1, 10) AS d; SET timescaledb.enable_direct_compress_on_cagg_refresh TO on; -- Refresh the base CAgg -CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL); +CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_hourly') chunk; chunk_status_text ------------------- {COMPRESSED} -- Refresh the hierarchical CAgg -CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL); +CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_daily') chunk; chunk_status_text ------------------- @@ -115,24 +116,24 @@ SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks INSERT INTO conditions SELECT t, d::text, 1, 1 FROM generate_series('2025-12-15 00:00:00+00'::timestamptz - interval '1 year', '2025-12-15 00:00:00+00'::timestamptz, interval '1 hour') AS t, generate_series(1, 10) AS d; -- Refresh the base CAgg -CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL); +CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_hourly') chunk; chunk_status_text ------------------- {COMPRESSED} -- Refreshing again the base CAgg is a no-op since everything is up to date -CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL); +CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); NOTICE: continuous aggregate "conditions_hourly" is already up-to-date -- Refresh the hierarchical CAgg with invalidations procuded by the base CAgg -CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL); +CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_daily') chunk; chunk_status_text ------------------- {COMPRESSED} -- Refreshing again the hierarchical CAgg is a no-op since everything is up to date -CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL); +CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); NOTICE: continuous aggregate "conditions_daily" is already up-to-date -- Tests with custom segmentby and orderby CREATE MATERIALIZED VIEW conditions_weekly @@ -148,7 +149,7 @@ FROM conditions GROUP BY 1, 2, 3 WITH NO DATA; ALTER MATERIALIZED VIEW conditions_weekly SET (timescaledb.compress_segmentby = 'device_id, location_id', timescaledb.compress_orderby = 'max, min, bucket DESC'); -CALL refresh_continuous_aggregate('conditions_weekly', NULL, NULL); +CALL refresh_continuous_aggregate('conditions_weekly', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_weekly') chunk; chunk_status_text ------------------- diff --git a/tsl/test/expected/cagg_invalidation.out b/tsl/test/expected/cagg_invalidation.out index d7930c60e13..b41fb41f1e6 100644 --- a/tsl/test/expected/cagg_invalidation.out +++ b/tsl/test/expected/cagg_invalidation.out @@ -1687,6 +1687,13 @@ SET timezone = 'UTC'; --Do the first refresh and check materialization invalidation log call refresh_continuous_aggregate ('test_cagg','2023-12-29 15:00:00', '2026-01-28 15:00:00'); call refresh_continuous_aggregate ('test_cagg_1d_offset','2023-12-29 18:00:00', '2024-01-01 18:00:00'); +-- test_cagg uses 1-month buckets with buckets_per_batch=10. The inscribed window +-- [2024-01-01, 2026-01-01) spans 24 months, producing two batches: +-- [2024-01-01, 2024-11-01) and [2024-11-01, 2025-09-01). +-- Data only exists through Dec 2024, so the split function finds no chunks past +-- Dec 2024 and stops at Sep 2025 (end of batch 2). The range [2025-09-01, 2026-01-01) +-- has no data and is left unprocessed, so the upper residual in the mat_inval_log +-- is 2025-09-01 rather than 2026-01-01 as it would be with single-pass refresh. SELECT materialization_id, _timescaledb_functions.to_timestamp(lowest_modified_value) as low, _timescaledb_functions.to_timestamp(greatest_modified_value) as high @@ -1698,7 +1705,7 @@ ORDER BY low; materialization_id | low | high --------------------+------------------------+------------------------------- 29 | -infinity | 2023-12-31 23:59:59.999999+00 - 29 | 2026-01-01 00:00:00+00 | infinity + 29 | 2025-09-01 00:00:00+00 | infinity SELECT CASE WHEN lowest_modified_value <= _timescaledb_functions.get_internal_time_min('timestamptz'::regtype) @@ -1720,9 +1727,10 @@ ORDER BY lowest_modified_value, greatest_modified_value; -infinity | 2023-12-29 17:59:59.999999+00 2024-01-01 18:00:00+00 | infinity ---now do the same refresh again, it should say the cagg is already up to date +-- Refresh the same range again, [2025-09-01 ─ 2026-01-01) was left. +-- This time refresh will go into the single batch path, so it will process the whole range. CALL refresh_continuous_aggregate ('test_cagg','2023-12-29 15:00:00', '2026-01-28 15:00:00'); -NOTICE: continuous aggregate "test_cagg" is already up-to-date +--Do the same refresh once again, it should say the cagg is already up to date CALL refresh_continuous_aggregate ('test_cagg','2023-12-29 15:00:00', '2026-01-28 15:00:00'); NOTICE: continuous aggregate "test_cagg" is already up-to-date --Insert data to test that invalidation is moved correctly from hypertable invalidation log @@ -1792,7 +1800,8 @@ WHERE hypertable_id IN ( 2026-01-01 00:00:00+00 INSERT INTO test_data values ('2026-01-05 00:00:00', 1); -CALL refresh_continuous_aggregate ('test_cagg_1d_offset','2023-12-29 15:00:00', NULL); +-- Setting buckets_per_batch to 0 as the intention is to test the capping when windew end is set to NULL, no need for batching. +CALL refresh_continuous_aggregate ('test_cagg_1d_offset','2023-12-29 15:00:00', NULL, options => '{"buckets_per_batch": 0}'::jsonb); --should be at the 18th hour SELECT _timescaledb_functions.to_timestamp(watermark) as invalidation_threshold FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold @@ -1860,10 +1869,11 @@ WHERE hypertable_id = ( -- Now refresh cagg_4hrs with NULL,NULL. -- cagg_4hrs computes its own threshold = 04:00, but stored threshold = 06:00 > 04:00, -- so the stored (misaligned) value is used but capped to the start of the current bucket of cagg_4hrs, --- which is 2020-01-01 04:00 UTC. +-- which is 2020-01-01 04:00 UTC. Disabling incremental refresh to preseve the isolate the test intention. SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_4hrs', NULL, NULL); -LOG: statement: CALL refresh_continuous_aggregate('cagg_4hrs', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4hrs', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +LOG: statement: CALL refresh_continuous_aggregate('cagg_4hrs', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +DEBUG: refreshing continuous aggregate "cagg_4hrs" from 4714-11-24 00:00:00+00 BC to infinity DEBUG: hypertable 31 existing watermark >= new invalidation threshold 1577858400000000 1577851200000000 DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4hrs" in window [ 4714-11-24 00:00:00+00 BC, 2020-01-01 04:00:00+00 ] LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_32" diff --git a/tsl/test/expected/cagg_invalidation_variable_bucket.out b/tsl/test/expected/cagg_invalidation_variable_bucket.out index 76d4d8e1070..32e12a35f2b 100644 --- a/tsl/test/expected/cagg_invalidation_variable_bucket.out +++ b/tsl/test/expected/cagg_invalidation_variable_bucket.out @@ -273,7 +273,8 @@ SELECT ts, 1.0 FROM generate_series('2025-03-30 00:00:00'::timestamptz, '2025-03-31 23:59:59.999999'::timestamptz, '1 hour'::interval) ts; -CALL refresh_continuous_aggregate('cagg_dst_daily', '2025-03-01 00:00:00', '2025-05-01 00:00:00'); +-- Disabling incremental refresh as it's not the focus of the test and it's easier to read the test output without it. +CALL refresh_continuous_aggregate('cagg_dst_daily', '2025-03-01 00:00:00', '2025-05-01 00:00:00', options => '{"buckets_per_batch": 0}'::jsonb); -- March 30 should have 23 hours SELECT bucket, cnt FROM cagg_dst_daily ORDER BY bucket; @@ -292,7 +293,7 @@ FROM generate_series('2025-10-26 00:00:00'::timestamptz, '2025-10-27 23:59:59.999999'::timestamptz, '1 hour'::interval) ts; -- Wide window to cover all DST-shifted buckets -CALL refresh_continuous_aggregate('cagg_dst_daily', '2025-10-01 00:00:00', '2026-12-01 00:00:00'); +CALL refresh_continuous_aggregate('cagg_dst_daily', '2025-10-01 00:00:00', '2026-12-01 00:00:00', options => '{"buckets_per_batch": 0}'::jsonb); -- October bucket should have extra hour (25-hour day on Oct 26) SELECT bucket, cnt FROM cagg_dst_daily WHERE bucket >= '2025-10-01 00:00:00' AND bucket < '2026-01-01 00:00:00' @@ -305,7 +306,7 @@ ORDER BY bucket; -- Insert near the fall-back boundary INSERT INTO dst_data VALUES ('2025-10-26 01:00:00', 888.0); -- 2:00 AM Europe/Berlin (after fall-back) INSERT INTO dst_data VALUES ('2025-10-26 00:30:00', 777.0); -- 2:30 AM Europe/Berlin (before fall-back) -CALL refresh_continuous_aggregate('cagg_dst_daily', '2025-09-01 00:00:00', '2026-02-01 00:00:00'); +CALL refresh_continuous_aggregate('cagg_dst_daily', '2025-09-01 00:00:00', '2026-02-01 00:00:00', options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_inval_log WHERE cagg_name = 'cagg_dst_daily'; cagg_name | inval_start | inval_end ----------------+------------------------+------------------------------- @@ -675,7 +676,7 @@ SELECT bucket, cnt FROM cagg_origin_tz ORDER BY bucket; SELECT * FROM cagg_inval_log WHERE cagg_name = 'cagg_origin_tz'; cagg_name | inval_start | inval_end ----------------+------------------------+------------------------------- - cagg_origin_tz | -infinity | 2025-02-01 04:59:59.999999+01 + cagg_origin_tz | -infinity | 2025-03-12 04:59:59.999999+01 cagg_origin_tz | 2025-03-31 05:00:00+02 | infinity SET timezone TO 'UTC'; diff --git a/tsl/test/expected/cagg_policy_run.out b/tsl/test/expected/cagg_policy_run.out index 7ecfdb487bd..229e7f576e3 100644 --- a/tsl/test/expected/cagg_policy_run.out +++ b/tsl/test/expected/cagg_policy_run.out @@ -198,19 +198,19 @@ ORDER BY range_start; SELECT * FROM measurements_chunks; chunk_name | range_start | is_compressed -------------------+-------------+--------------- - _hyper_9_33_chunk | 01-01-2025 | f - _hyper_9_34_chunk | 01-02-2025 | f - _hyper_9_35_chunk | 01-03-2025 | f - _hyper_9_32_chunk | 01-04-2025 | f - _hyper_9_29_chunk | 01-05-2025 | f - _hyper_9_37_chunk | 01-06-2025 | f - _hyper_9_31_chunk | 01-07-2025 | f - _hyper_9_38_chunk | 01-08-2025 | f - _hyper_9_40_chunk | 01-09-2025 | f - _hyper_9_39_chunk | 01-10-2025 | f - _hyper_9_30_chunk | 01-11-2025 | f - _hyper_9_36_chunk | 01-12-2025 | f - _hyper_9_41_chunk | 01-13-2025 | f + _hyper_9_35_chunk | 01-01-2025 | f + _hyper_9_36_chunk | 01-02-2025 | f + _hyper_9_37_chunk | 01-03-2025 | f + _hyper_9_34_chunk | 01-04-2025 | f + _hyper_9_32_chunk | 01-05-2025 | f + _hyper_9_38_chunk | 01-06-2025 | f + _hyper_9_33_chunk | 01-07-2025 | f + _hyper_9_39_chunk | 01-08-2025 | f + _hyper_9_41_chunk | 01-09-2025 | f + _hyper_9_40_chunk | 01-10-2025 | f + _hyper_9_31_chunk | 01-11-2025 | f + _hyper_9_29_chunk | 01-12-2025 | f + _hyper_9_30_chunk | 01-13-2025 | f -- Mock now() at 2025-01-15. With start_offset = 8 days and end_offset = 4 days -- the refresh window is [2025-01-07, 2025-01-11). Chunks 01-07..01-10 should @@ -233,19 +233,19 @@ CALL run_job(:job_id); SELECT * FROM measurements_chunks; chunk_name | range_start | is_compressed -------------------+-------------+--------------- - _hyper_9_33_chunk | 01-01-2025 | f - _hyper_9_34_chunk | 01-02-2025 | f - _hyper_9_35_chunk | 01-03-2025 | f - _hyper_9_32_chunk | 01-04-2025 | f - _hyper_9_29_chunk | 01-05-2025 | f - _hyper_9_37_chunk | 01-06-2025 | f - _hyper_9_31_chunk | 01-07-2025 | t - _hyper_9_38_chunk | 01-08-2025 | t - _hyper_9_40_chunk | 01-09-2025 | t - _hyper_9_39_chunk | 01-10-2025 | t - _hyper_9_30_chunk | 01-11-2025 | f - _hyper_9_36_chunk | 01-12-2025 | f - _hyper_9_41_chunk | 01-13-2025 | f + _hyper_9_35_chunk | 01-01-2025 | f + _hyper_9_36_chunk | 01-02-2025 | f + _hyper_9_37_chunk | 01-03-2025 | f + _hyper_9_34_chunk | 01-04-2025 | f + _hyper_9_32_chunk | 01-05-2025 | f + _hyper_9_38_chunk | 01-06-2025 | f + _hyper_9_33_chunk | 01-07-2025 | t + _hyper_9_39_chunk | 01-08-2025 | t + _hyper_9_41_chunk | 01-09-2025 | t + _hyper_9_40_chunk | 01-10-2025 | t + _hyper_9_31_chunk | 01-11-2025 | f + _hyper_9_29_chunk | 01-12-2025 | f + _hyper_9_30_chunk | 01-13-2025 | f SELECT delete_job(:job_id); delete_job diff --git a/tsl/test/expected/cagg_query-15.out b/tsl/test/expected/cagg_query-15.out index c07e1ea487d..8c6fb5657d2 100644 --- a/tsl/test/expected/cagg_query-15.out +++ b/tsl/test/expected/cagg_query-15.out @@ -62,7 +62,7 @@ FROM ( select max(timec)as timeval from conditions ) as q; ------------------------------ Sat Nov 03 17:00:00 2018 PDT -CALL refresh_continuous_aggregate('mat_m1', NULL, NULL); +CALL refresh_continuous_aggregate('mat_m1', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); --test first/last create materialized view mat_m2(location, timec, firsth, lasth, maxtemp, mintemp) WITH (timescaledb.continuous, timescaledb.materialized_only=false) @@ -77,7 +77,7 @@ FROM ( select max(timec)as timeval from conditions ) as q; ------------------------------ Sat Nov 03 17:00:00 2018 PDT -CALL refresh_continuous_aggregate('mat_m2', NULL, NULL); +CALL refresh_continuous_aggregate('mat_m2', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); --normal view -- create or replace view regview( location, timec, minl, sumt , sumh) as @@ -1087,7 +1087,7 @@ SET ROLE :ROLE_SUPERUSER; BEGIN; UPDATE _timescaledb_catalog.continuous_aggs_bucket_function SET bucket_func = 'func_does_not_exist()'; -- should error because function does not exist -CALL refresh_continuous_aggregate('cagg_bigint_offset2', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_bigint_offset2', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); psql:include/cagg_query_common.sql:566: ERROR: function "func_does_not_exist()" does not exist ROLLBACK; \set ON_ERROR_STOP 1 @@ -1304,9 +1304,9 @@ SELECT * FROM cagg_4_hours_origin; -- Update the last bucket and re-materialize INSERT INTO temperature values('2020-01-01 23:55:00 PST', 10); -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_4_hours; time_bucket | max ------------------------------+----- @@ -1439,18 +1439,21 @@ SELECT * FROM cagg_4_hours_origin; -- Update materialized data SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -psql:include/cagg_query_common.sql:683: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:683: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:683: DEBUG: refreshing continuous aggregate "cagg_4_hours" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:683: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Thu Jan 02 00:00:00 2020 PST, Thu Jan 02 12:00:00 2020 PST ] psql:include/cagg_query_common.sql:683: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_33" psql:include/cagg_query_common.sql:683: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -psql:include/cagg_query_common.sql:684: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:684: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:684: DEBUG: refreshing continuous aggregate "cagg_4_hours_offset" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:684: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Wed Jan 01 20:30:00 2020 PST, Thu Jan 02 12:30:00 2020 PST ] psql:include/cagg_query_common.sql:684: LOG: deleted 1 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:684: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); -psql:include/cagg_query_common.sql:685: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:685: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:685: DEBUG: refreshing continuous aggregate "cagg_4_hours_origin" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:685: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_origin" in window [ Wed Jan 01 21:00:00 2020 PST, Thu Jan 02 13:00:00 2020 PST ] psql:include/cagg_query_common.sql:685: LOG: deleted 1 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_35" psql:include/cagg_query_common.sql:685: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_35" @@ -1633,9 +1636,9 @@ SELECT time_bucket('4 hour', time, '2000-01-01 01:00:00 PST'::timestamptz), max( -- Test invalidations TRUNCATE temperature; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); INSERT INTO temperature SELECT time, 5 FROM generate_series('2000-01-01 01:00:00 PST'::timestamptz, @@ -1648,8 +1651,9 @@ INSERT INTO temperature values('2020-01-02 01:05:00+01', 2222); INSERT INTO temperature values('2020-01-02 01:35:00+01', 5555); INSERT INTO temperature values('2020-01-02 05:05:00+01', 8888); SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -psql:include/cagg_query_common.sql:725: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:725: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:725: DEBUG: refreshing continuous aggregate "cagg_4_hours" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:725: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577952000000000 psql:include/cagg_query_common.sql:725: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Sat Jan 01 00:00:00 2000 PST, Sun Jan 02 00:00:00 2000 PST ] psql:include/cagg_query_common.sql:725: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_33" @@ -1659,8 +1663,9 @@ psql:include/cagg_query_common.sql:725: DEBUG: continuous aggregate refresh (in psql:include/cagg_query_common.sql:725: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_33" psql:include/cagg_query_common.sql:725: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" psql:include/cagg_query_common.sql:725: DEBUG: hypertable 33 existing watermark >= new watermark 1577995200000000 1577952000000000 -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -psql:include/cagg_query_common.sql:726: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:726: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:726: DEBUG: refreshing continuous aggregate "cagg_4_hours_offset" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:726: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577953800000000 psql:include/cagg_query_common.sql:726: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Sat Jan 01 00:30:00 2000 PST, Sun Jan 02 00:30:00 2000 PST ] psql:include/cagg_query_common.sql:726: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_34" @@ -1670,8 +1675,9 @@ psql:include/cagg_query_common.sql:726: DEBUG: continuous aggregate refresh (in psql:include/cagg_query_common.sql:726: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:726: LOG: inserted 7 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:726: DEBUG: hypertable 34 existing watermark >= new watermark 1577997000000000 1577953800000000 -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); -psql:include/cagg_query_common.sql:727: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:727: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:727: DEBUG: refreshing continuous aggregate "cagg_4_hours_origin" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:727: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577955600000000 psql:include/cagg_query_common.sql:727: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_origin" in window [ Sat Jan 01 01:00:00 2000 PST, Sun Jan 02 01:00:00 2000 PST ] psql:include/cagg_query_common.sql:727: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_35" @@ -1917,7 +1923,7 @@ ORDER BY 1, 2, 3; 33 | 1577995200000000 | 9223372036854775807 33 | 1577995200000000 | 9223372036854775807 -CALL refresh_continuous_aggregate('cagg_4_hours', '2000-01-01 00:00:00'::timestamptz, '2020-12-31 23:59:59'::timestamptz); +CALL refresh_continuous_aggregate('cagg_4_hours', '2000-01-01 00:00:00'::timestamptz, '2020-12-31 23:59:59'::timestamptz, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log WHERE materialization_id IN (SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'cagg_4_hours') @@ -2045,7 +2051,7 @@ SELECT * FROM cagg_int; 50 | 5 100 | 555 -CALL refresh_continuous_aggregate('cagg_int', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_int', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_int; time_bucket | value -------------+------- @@ -2094,7 +2100,7 @@ SELECT * FROM cagg_int_offset; -- the value 100 is part of the already serializ 45 | 30 95 | 555 -CALL refresh_continuous_aggregate('cagg_int_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_int_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_int_offset; time_bucket | value -------------+------- @@ -2149,8 +2155,9 @@ SELECT * FROM cagg_int_offset; -- Check that the refresh is properly aligned INSERT INTO table_int VALUES(114, 0); SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130); -psql:include/cagg_query_common.sql:845: LOG: statement: CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130); +CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:845: LOG: statement: CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:845: DEBUG: refreshing continuous aggregate "cagg_int_offset" from 100 to 130 psql:include/cagg_query_common.sql:845: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_int_offset" in window [ 105, 125 ] psql:include/cagg_query_common.sql:845: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_38" psql:include/cagg_query_common.sql:845: DEBUG: building index "_hyper_38_68_chunk__materialized_hypertable_38_time_bucket_idx" on table "_hyper_38_68_chunk" serially @@ -2369,8 +2376,8 @@ SELECT time_bucket('1 week', time, origin=>'2000-01-02 01:00:00 PST'::timestampt Sun Dec 30 01:00:00 2029 PST | 55555 -- Test refresh -CALL refresh_continuous_aggregate('cagg_1_hour_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_1_week_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_1_hour_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_1_week_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); -- Everything should be now materailized ALTER MATERIALIZED VIEW cagg_1_hour_offset SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW cagg_1_week_offset SET (timescaledb.materialized_only=false); diff --git a/tsl/test/expected/cagg_query-16.out b/tsl/test/expected/cagg_query-16.out index 29cfcf539fa..7db97c7fd4c 100644 --- a/tsl/test/expected/cagg_query-16.out +++ b/tsl/test/expected/cagg_query-16.out @@ -62,7 +62,7 @@ FROM ( select max(timec)as timeval from conditions ) as q; ------------------------------ Sat Nov 03 17:00:00 2018 PDT -CALL refresh_continuous_aggregate('mat_m1', NULL, NULL); +CALL refresh_continuous_aggregate('mat_m1', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); --test first/last create materialized view mat_m2(location, timec, firsth, lasth, maxtemp, mintemp) WITH (timescaledb.continuous, timescaledb.materialized_only=false) @@ -77,7 +77,7 @@ FROM ( select max(timec)as timeval from conditions ) as q; ------------------------------ Sat Nov 03 17:00:00 2018 PDT -CALL refresh_continuous_aggregate('mat_m2', NULL, NULL); +CALL refresh_continuous_aggregate('mat_m2', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); --normal view -- create or replace view regview( location, timec, minl, sumt , sumh) as @@ -1081,7 +1081,7 @@ SET ROLE :ROLE_SUPERUSER; BEGIN; UPDATE _timescaledb_catalog.continuous_aggs_bucket_function SET bucket_func = 'func_does_not_exist()'; -- should error because function does not exist -CALL refresh_continuous_aggregate('cagg_bigint_offset2', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_bigint_offset2', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); psql:include/cagg_query_common.sql:566: ERROR: function "func_does_not_exist()" does not exist ROLLBACK; \set ON_ERROR_STOP 1 @@ -1298,9 +1298,9 @@ SELECT * FROM cagg_4_hours_origin; -- Update the last bucket and re-materialize INSERT INTO temperature values('2020-01-01 23:55:00 PST', 10); -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_4_hours; time_bucket | max ------------------------------+----- @@ -1433,18 +1433,21 @@ SELECT * FROM cagg_4_hours_origin; -- Update materialized data SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -psql:include/cagg_query_common.sql:683: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:683: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:683: DEBUG: refreshing continuous aggregate "cagg_4_hours" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:683: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Thu Jan 02 00:00:00 2020 PST, Thu Jan 02 12:00:00 2020 PST ] psql:include/cagg_query_common.sql:683: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_33" psql:include/cagg_query_common.sql:683: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -psql:include/cagg_query_common.sql:684: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:684: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:684: DEBUG: refreshing continuous aggregate "cagg_4_hours_offset" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:684: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Wed Jan 01 20:30:00 2020 PST, Thu Jan 02 12:30:00 2020 PST ] psql:include/cagg_query_common.sql:684: LOG: deleted 1 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:684: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); -psql:include/cagg_query_common.sql:685: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:685: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:685: DEBUG: refreshing continuous aggregate "cagg_4_hours_origin" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:685: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_origin" in window [ Wed Jan 01 21:00:00 2020 PST, Thu Jan 02 13:00:00 2020 PST ] psql:include/cagg_query_common.sql:685: LOG: deleted 1 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_35" psql:include/cagg_query_common.sql:685: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_35" @@ -1627,9 +1630,9 @@ SELECT time_bucket('4 hour', time, '2000-01-01 01:00:00 PST'::timestamptz), max( -- Test invalidations TRUNCATE temperature; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); INSERT INTO temperature SELECT time, 5 FROM generate_series('2000-01-01 01:00:00 PST'::timestamptz, @@ -1642,8 +1645,9 @@ INSERT INTO temperature values('2020-01-02 01:05:00+01', 2222); INSERT INTO temperature values('2020-01-02 01:35:00+01', 5555); INSERT INTO temperature values('2020-01-02 05:05:00+01', 8888); SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -psql:include/cagg_query_common.sql:725: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:725: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:725: DEBUG: refreshing continuous aggregate "cagg_4_hours" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:725: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577952000000000 psql:include/cagg_query_common.sql:725: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Sat Jan 01 00:00:00 2000 PST, Sun Jan 02 00:00:00 2000 PST ] psql:include/cagg_query_common.sql:725: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_33" @@ -1653,8 +1657,9 @@ psql:include/cagg_query_common.sql:725: DEBUG: continuous aggregate refresh (in psql:include/cagg_query_common.sql:725: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_33" psql:include/cagg_query_common.sql:725: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" psql:include/cagg_query_common.sql:725: DEBUG: hypertable 33 existing watermark >= new watermark 1577995200000000 1577952000000000 -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -psql:include/cagg_query_common.sql:726: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:726: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:726: DEBUG: refreshing continuous aggregate "cagg_4_hours_offset" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:726: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577953800000000 psql:include/cagg_query_common.sql:726: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Sat Jan 01 00:30:00 2000 PST, Sun Jan 02 00:30:00 2000 PST ] psql:include/cagg_query_common.sql:726: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_34" @@ -1664,8 +1669,9 @@ psql:include/cagg_query_common.sql:726: DEBUG: continuous aggregate refresh (in psql:include/cagg_query_common.sql:726: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:726: LOG: inserted 7 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:726: DEBUG: hypertable 34 existing watermark >= new watermark 1577997000000000 1577953800000000 -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); -psql:include/cagg_query_common.sql:727: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:727: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:727: DEBUG: refreshing continuous aggregate "cagg_4_hours_origin" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:727: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577955600000000 psql:include/cagg_query_common.sql:727: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_origin" in window [ Sat Jan 01 01:00:00 2000 PST, Sun Jan 02 01:00:00 2000 PST ] psql:include/cagg_query_common.sql:727: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_35" @@ -1911,7 +1917,7 @@ ORDER BY 1, 2, 3; 33 | 1577995200000000 | 9223372036854775807 33 | 1577995200000000 | 9223372036854775807 -CALL refresh_continuous_aggregate('cagg_4_hours', '2000-01-01 00:00:00'::timestamptz, '2020-12-31 23:59:59'::timestamptz); +CALL refresh_continuous_aggregate('cagg_4_hours', '2000-01-01 00:00:00'::timestamptz, '2020-12-31 23:59:59'::timestamptz, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log WHERE materialization_id IN (SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'cagg_4_hours') @@ -2039,7 +2045,7 @@ SELECT * FROM cagg_int; 50 | 5 100 | 555 -CALL refresh_continuous_aggregate('cagg_int', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_int', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_int; time_bucket | value -------------+------- @@ -2088,7 +2094,7 @@ SELECT * FROM cagg_int_offset; -- the value 100 is part of the already serializ 45 | 30 95 | 555 -CALL refresh_continuous_aggregate('cagg_int_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_int_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_int_offset; time_bucket | value -------------+------- @@ -2143,8 +2149,9 @@ SELECT * FROM cagg_int_offset; -- Check that the refresh is properly aligned INSERT INTO table_int VALUES(114, 0); SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130); -psql:include/cagg_query_common.sql:845: LOG: statement: CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130); +CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:845: LOG: statement: CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:845: DEBUG: refreshing continuous aggregate "cagg_int_offset" from 100 to 130 psql:include/cagg_query_common.sql:845: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_int_offset" in window [ 105, 125 ] psql:include/cagg_query_common.sql:845: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_38" psql:include/cagg_query_common.sql:845: DEBUG: building index "_hyper_38_68_chunk__materialized_hypertable_38_time_bucket_idx" on table "_hyper_38_68_chunk" serially @@ -2363,8 +2370,8 @@ SELECT time_bucket('1 week', time, origin=>'2000-01-02 01:00:00 PST'::timestampt Sun Dec 30 01:00:00 2029 PST | 55555 -- Test refresh -CALL refresh_continuous_aggregate('cagg_1_hour_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_1_week_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_1_hour_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_1_week_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); -- Everything should be now materailized ALTER MATERIALIZED VIEW cagg_1_hour_offset SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW cagg_1_week_offset SET (timescaledb.materialized_only=false); diff --git a/tsl/test/expected/cagg_query-17.out b/tsl/test/expected/cagg_query-17.out index 29cfcf539fa..7db97c7fd4c 100644 --- a/tsl/test/expected/cagg_query-17.out +++ b/tsl/test/expected/cagg_query-17.out @@ -62,7 +62,7 @@ FROM ( select max(timec)as timeval from conditions ) as q; ------------------------------ Sat Nov 03 17:00:00 2018 PDT -CALL refresh_continuous_aggregate('mat_m1', NULL, NULL); +CALL refresh_continuous_aggregate('mat_m1', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); --test first/last create materialized view mat_m2(location, timec, firsth, lasth, maxtemp, mintemp) WITH (timescaledb.continuous, timescaledb.materialized_only=false) @@ -77,7 +77,7 @@ FROM ( select max(timec)as timeval from conditions ) as q; ------------------------------ Sat Nov 03 17:00:00 2018 PDT -CALL refresh_continuous_aggregate('mat_m2', NULL, NULL); +CALL refresh_continuous_aggregate('mat_m2', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); --normal view -- create or replace view regview( location, timec, minl, sumt , sumh) as @@ -1081,7 +1081,7 @@ SET ROLE :ROLE_SUPERUSER; BEGIN; UPDATE _timescaledb_catalog.continuous_aggs_bucket_function SET bucket_func = 'func_does_not_exist()'; -- should error because function does not exist -CALL refresh_continuous_aggregate('cagg_bigint_offset2', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_bigint_offset2', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); psql:include/cagg_query_common.sql:566: ERROR: function "func_does_not_exist()" does not exist ROLLBACK; \set ON_ERROR_STOP 1 @@ -1298,9 +1298,9 @@ SELECT * FROM cagg_4_hours_origin; -- Update the last bucket and re-materialize INSERT INTO temperature values('2020-01-01 23:55:00 PST', 10); -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_4_hours; time_bucket | max ------------------------------+----- @@ -1433,18 +1433,21 @@ SELECT * FROM cagg_4_hours_origin; -- Update materialized data SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -psql:include/cagg_query_common.sql:683: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:683: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:683: DEBUG: refreshing continuous aggregate "cagg_4_hours" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:683: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Thu Jan 02 00:00:00 2020 PST, Thu Jan 02 12:00:00 2020 PST ] psql:include/cagg_query_common.sql:683: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_33" psql:include/cagg_query_common.sql:683: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -psql:include/cagg_query_common.sql:684: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:684: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:684: DEBUG: refreshing continuous aggregate "cagg_4_hours_offset" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:684: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Wed Jan 01 20:30:00 2020 PST, Thu Jan 02 12:30:00 2020 PST ] psql:include/cagg_query_common.sql:684: LOG: deleted 1 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:684: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); -psql:include/cagg_query_common.sql:685: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:685: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:685: DEBUG: refreshing continuous aggregate "cagg_4_hours_origin" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:685: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_origin" in window [ Wed Jan 01 21:00:00 2020 PST, Thu Jan 02 13:00:00 2020 PST ] psql:include/cagg_query_common.sql:685: LOG: deleted 1 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_35" psql:include/cagg_query_common.sql:685: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_35" @@ -1627,9 +1630,9 @@ SELECT time_bucket('4 hour', time, '2000-01-01 01:00:00 PST'::timestamptz), max( -- Test invalidations TRUNCATE temperature; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); INSERT INTO temperature SELECT time, 5 FROM generate_series('2000-01-01 01:00:00 PST'::timestamptz, @@ -1642,8 +1645,9 @@ INSERT INTO temperature values('2020-01-02 01:05:00+01', 2222); INSERT INTO temperature values('2020-01-02 01:35:00+01', 5555); INSERT INTO temperature values('2020-01-02 05:05:00+01', 8888); SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -psql:include/cagg_query_common.sql:725: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:725: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:725: DEBUG: refreshing continuous aggregate "cagg_4_hours" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:725: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577952000000000 psql:include/cagg_query_common.sql:725: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Sat Jan 01 00:00:00 2000 PST, Sun Jan 02 00:00:00 2000 PST ] psql:include/cagg_query_common.sql:725: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_33" @@ -1653,8 +1657,9 @@ psql:include/cagg_query_common.sql:725: DEBUG: continuous aggregate refresh (in psql:include/cagg_query_common.sql:725: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_33" psql:include/cagg_query_common.sql:725: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" psql:include/cagg_query_common.sql:725: DEBUG: hypertable 33 existing watermark >= new watermark 1577995200000000 1577952000000000 -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -psql:include/cagg_query_common.sql:726: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:726: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:726: DEBUG: refreshing continuous aggregate "cagg_4_hours_offset" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:726: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577953800000000 psql:include/cagg_query_common.sql:726: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Sat Jan 01 00:30:00 2000 PST, Sun Jan 02 00:30:00 2000 PST ] psql:include/cagg_query_common.sql:726: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_34" @@ -1664,8 +1669,9 @@ psql:include/cagg_query_common.sql:726: DEBUG: continuous aggregate refresh (in psql:include/cagg_query_common.sql:726: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:726: LOG: inserted 7 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:726: DEBUG: hypertable 34 existing watermark >= new watermark 1577997000000000 1577953800000000 -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); -psql:include/cagg_query_common.sql:727: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:727: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:727: DEBUG: refreshing continuous aggregate "cagg_4_hours_origin" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:727: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577955600000000 psql:include/cagg_query_common.sql:727: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_origin" in window [ Sat Jan 01 01:00:00 2000 PST, Sun Jan 02 01:00:00 2000 PST ] psql:include/cagg_query_common.sql:727: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_35" @@ -1911,7 +1917,7 @@ ORDER BY 1, 2, 3; 33 | 1577995200000000 | 9223372036854775807 33 | 1577995200000000 | 9223372036854775807 -CALL refresh_continuous_aggregate('cagg_4_hours', '2000-01-01 00:00:00'::timestamptz, '2020-12-31 23:59:59'::timestamptz); +CALL refresh_continuous_aggregate('cagg_4_hours', '2000-01-01 00:00:00'::timestamptz, '2020-12-31 23:59:59'::timestamptz, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log WHERE materialization_id IN (SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'cagg_4_hours') @@ -2039,7 +2045,7 @@ SELECT * FROM cagg_int; 50 | 5 100 | 555 -CALL refresh_continuous_aggregate('cagg_int', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_int', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_int; time_bucket | value -------------+------- @@ -2088,7 +2094,7 @@ SELECT * FROM cagg_int_offset; -- the value 100 is part of the already serializ 45 | 30 95 | 555 -CALL refresh_continuous_aggregate('cagg_int_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_int_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_int_offset; time_bucket | value -------------+------- @@ -2143,8 +2149,9 @@ SELECT * FROM cagg_int_offset; -- Check that the refresh is properly aligned INSERT INTO table_int VALUES(114, 0); SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130); -psql:include/cagg_query_common.sql:845: LOG: statement: CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130); +CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:845: LOG: statement: CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:845: DEBUG: refreshing continuous aggregate "cagg_int_offset" from 100 to 130 psql:include/cagg_query_common.sql:845: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_int_offset" in window [ 105, 125 ] psql:include/cagg_query_common.sql:845: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_38" psql:include/cagg_query_common.sql:845: DEBUG: building index "_hyper_38_68_chunk__materialized_hypertable_38_time_bucket_idx" on table "_hyper_38_68_chunk" serially @@ -2363,8 +2370,8 @@ SELECT time_bucket('1 week', time, origin=>'2000-01-02 01:00:00 PST'::timestampt Sun Dec 30 01:00:00 2029 PST | 55555 -- Test refresh -CALL refresh_continuous_aggregate('cagg_1_hour_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_1_week_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_1_hour_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_1_week_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); -- Everything should be now materailized ALTER MATERIALIZED VIEW cagg_1_hour_offset SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW cagg_1_week_offset SET (timescaledb.materialized_only=false); diff --git a/tsl/test/expected/cagg_query-18.out b/tsl/test/expected/cagg_query-18.out index 29cfcf539fa..7db97c7fd4c 100644 --- a/tsl/test/expected/cagg_query-18.out +++ b/tsl/test/expected/cagg_query-18.out @@ -62,7 +62,7 @@ FROM ( select max(timec)as timeval from conditions ) as q; ------------------------------ Sat Nov 03 17:00:00 2018 PDT -CALL refresh_continuous_aggregate('mat_m1', NULL, NULL); +CALL refresh_continuous_aggregate('mat_m1', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); --test first/last create materialized view mat_m2(location, timec, firsth, lasth, maxtemp, mintemp) WITH (timescaledb.continuous, timescaledb.materialized_only=false) @@ -77,7 +77,7 @@ FROM ( select max(timec)as timeval from conditions ) as q; ------------------------------ Sat Nov 03 17:00:00 2018 PDT -CALL refresh_continuous_aggregate('mat_m2', NULL, NULL); +CALL refresh_continuous_aggregate('mat_m2', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); --normal view -- create or replace view regview( location, timec, minl, sumt , sumh) as @@ -1081,7 +1081,7 @@ SET ROLE :ROLE_SUPERUSER; BEGIN; UPDATE _timescaledb_catalog.continuous_aggs_bucket_function SET bucket_func = 'func_does_not_exist()'; -- should error because function does not exist -CALL refresh_continuous_aggregate('cagg_bigint_offset2', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_bigint_offset2', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); psql:include/cagg_query_common.sql:566: ERROR: function "func_does_not_exist()" does not exist ROLLBACK; \set ON_ERROR_STOP 1 @@ -1298,9 +1298,9 @@ SELECT * FROM cagg_4_hours_origin; -- Update the last bucket and re-materialize INSERT INTO temperature values('2020-01-01 23:55:00 PST', 10); -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_4_hours; time_bucket | max ------------------------------+----- @@ -1433,18 +1433,21 @@ SELECT * FROM cagg_4_hours_origin; -- Update materialized data SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -psql:include/cagg_query_common.sql:683: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:683: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:683: DEBUG: refreshing continuous aggregate "cagg_4_hours" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:683: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Thu Jan 02 00:00:00 2020 PST, Thu Jan 02 12:00:00 2020 PST ] psql:include/cagg_query_common.sql:683: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_33" psql:include/cagg_query_common.sql:683: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -psql:include/cagg_query_common.sql:684: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:684: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:684: DEBUG: refreshing continuous aggregate "cagg_4_hours_offset" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:684: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Wed Jan 01 20:30:00 2020 PST, Thu Jan 02 12:30:00 2020 PST ] psql:include/cagg_query_common.sql:684: LOG: deleted 1 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:684: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); -psql:include/cagg_query_common.sql:685: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:685: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:685: DEBUG: refreshing continuous aggregate "cagg_4_hours_origin" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:685: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_origin" in window [ Wed Jan 01 21:00:00 2020 PST, Thu Jan 02 13:00:00 2020 PST ] psql:include/cagg_query_common.sql:685: LOG: deleted 1 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_35" psql:include/cagg_query_common.sql:685: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_35" @@ -1627,9 +1630,9 @@ SELECT time_bucket('4 hour', time, '2000-01-01 01:00:00 PST'::timestamptz), max( -- Test invalidations TRUNCATE temperature; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); INSERT INTO temperature SELECT time, 5 FROM generate_series('2000-01-01 01:00:00 PST'::timestamptz, @@ -1642,8 +1645,9 @@ INSERT INTO temperature values('2020-01-02 01:05:00+01', 2222); INSERT INTO temperature values('2020-01-02 01:35:00+01', 5555); INSERT INTO temperature values('2020-01-02 05:05:00+01', 8888); SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -psql:include/cagg_query_common.sql:725: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:725: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:725: DEBUG: refreshing continuous aggregate "cagg_4_hours" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:725: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577952000000000 psql:include/cagg_query_common.sql:725: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Sat Jan 01 00:00:00 2000 PST, Sun Jan 02 00:00:00 2000 PST ] psql:include/cagg_query_common.sql:725: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_33" @@ -1653,8 +1657,9 @@ psql:include/cagg_query_common.sql:725: DEBUG: continuous aggregate refresh (in psql:include/cagg_query_common.sql:725: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_33" psql:include/cagg_query_common.sql:725: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" psql:include/cagg_query_common.sql:725: DEBUG: hypertable 33 existing watermark >= new watermark 1577995200000000 1577952000000000 -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -psql:include/cagg_query_common.sql:726: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:726: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:726: DEBUG: refreshing continuous aggregate "cagg_4_hours_offset" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:726: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577953800000000 psql:include/cagg_query_common.sql:726: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Sat Jan 01 00:30:00 2000 PST, Sun Jan 02 00:30:00 2000 PST ] psql:include/cagg_query_common.sql:726: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_34" @@ -1664,8 +1669,9 @@ psql:include/cagg_query_common.sql:726: DEBUG: continuous aggregate refresh (in psql:include/cagg_query_common.sql:726: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:726: LOG: inserted 7 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:726: DEBUG: hypertable 34 existing watermark >= new watermark 1577997000000000 1577953800000000 -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); -psql:include/cagg_query_common.sql:727: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:727: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:727: DEBUG: refreshing continuous aggregate "cagg_4_hours_origin" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:727: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577955600000000 psql:include/cagg_query_common.sql:727: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_origin" in window [ Sat Jan 01 01:00:00 2000 PST, Sun Jan 02 01:00:00 2000 PST ] psql:include/cagg_query_common.sql:727: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_35" @@ -1911,7 +1917,7 @@ ORDER BY 1, 2, 3; 33 | 1577995200000000 | 9223372036854775807 33 | 1577995200000000 | 9223372036854775807 -CALL refresh_continuous_aggregate('cagg_4_hours', '2000-01-01 00:00:00'::timestamptz, '2020-12-31 23:59:59'::timestamptz); +CALL refresh_continuous_aggregate('cagg_4_hours', '2000-01-01 00:00:00'::timestamptz, '2020-12-31 23:59:59'::timestamptz, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log WHERE materialization_id IN (SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'cagg_4_hours') @@ -2039,7 +2045,7 @@ SELECT * FROM cagg_int; 50 | 5 100 | 555 -CALL refresh_continuous_aggregate('cagg_int', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_int', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_int; time_bucket | value -------------+------- @@ -2088,7 +2094,7 @@ SELECT * FROM cagg_int_offset; -- the value 100 is part of the already serializ 45 | 30 95 | 555 -CALL refresh_continuous_aggregate('cagg_int_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_int_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_int_offset; time_bucket | value -------------+------- @@ -2143,8 +2149,9 @@ SELECT * FROM cagg_int_offset; -- Check that the refresh is properly aligned INSERT INTO table_int VALUES(114, 0); SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130); -psql:include/cagg_query_common.sql:845: LOG: statement: CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130); +CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:845: LOG: statement: CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:845: DEBUG: refreshing continuous aggregate "cagg_int_offset" from 100 to 130 psql:include/cagg_query_common.sql:845: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_int_offset" in window [ 105, 125 ] psql:include/cagg_query_common.sql:845: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_38" psql:include/cagg_query_common.sql:845: DEBUG: building index "_hyper_38_68_chunk__materialized_hypertable_38_time_bucket_idx" on table "_hyper_38_68_chunk" serially @@ -2363,8 +2370,8 @@ SELECT time_bucket('1 week', time, origin=>'2000-01-02 01:00:00 PST'::timestampt Sun Dec 30 01:00:00 2029 PST | 55555 -- Test refresh -CALL refresh_continuous_aggregate('cagg_1_hour_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_1_week_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_1_hour_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_1_week_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); -- Everything should be now materailized ALTER MATERIALIZED VIEW cagg_1_hour_offset SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW cagg_1_week_offset SET (timescaledb.materialized_only=false); diff --git a/tsl/test/expected/cagg_query_using_merge-15.out b/tsl/test/expected/cagg_query_using_merge-15.out index 80d8b5aa0f5..7db63d3950e 100644 --- a/tsl/test/expected/cagg_query_using_merge-15.out +++ b/tsl/test/expected/cagg_query_using_merge-15.out @@ -64,7 +64,7 @@ FROM ( select max(timec)as timeval from conditions ) as q; ------------------------------ Sat Nov 03 17:00:00 2018 PDT -CALL refresh_continuous_aggregate('mat_m1', NULL, NULL); +CALL refresh_continuous_aggregate('mat_m1', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); --test first/last create materialized view mat_m2(location, timec, firsth, lasth, maxtemp, mintemp) WITH (timescaledb.continuous, timescaledb.materialized_only=false) @@ -79,7 +79,7 @@ FROM ( select max(timec)as timeval from conditions ) as q; ------------------------------ Sat Nov 03 17:00:00 2018 PDT -CALL refresh_continuous_aggregate('mat_m2', NULL, NULL); +CALL refresh_continuous_aggregate('mat_m2', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); --normal view -- create or replace view regview( location, timec, minl, sumt , sumh) as @@ -1089,7 +1089,7 @@ SET ROLE :ROLE_SUPERUSER; BEGIN; UPDATE _timescaledb_catalog.continuous_aggs_bucket_function SET bucket_func = 'func_does_not_exist()'; -- should error because function does not exist -CALL refresh_continuous_aggregate('cagg_bigint_offset2', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_bigint_offset2', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); psql:include/cagg_query_common.sql:566: ERROR: function "func_does_not_exist()" does not exist ROLLBACK; \set ON_ERROR_STOP 1 @@ -1306,9 +1306,9 @@ SELECT * FROM cagg_4_hours_origin; -- Update the last bucket and re-materialize INSERT INTO temperature values('2020-01-01 23:55:00 PST', 10); -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_4_hours; time_bucket | max ------------------------------+----- @@ -1441,17 +1441,20 @@ SELECT * FROM cagg_4_hours_origin; -- Update materialized data SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -psql:include/cagg_query_common.sql:683: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:683: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:683: DEBUG: refreshing continuous aggregate "cagg_4_hours" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:683: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Thu Jan 02 00:00:00 2020 PST, Thu Jan 02 12:00:00 2020 PST ] psql:include/cagg_query_common.sql:683: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -psql:include/cagg_query_common.sql:684: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:684: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:684: DEBUG: refreshing continuous aggregate "cagg_4_hours_offset" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:684: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Wed Jan 01 20:30:00 2020 PST, Thu Jan 02 12:30:00 2020 PST ] psql:include/cagg_query_common.sql:684: LOG: merged 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:684: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_34" -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); -psql:include/cagg_query_common.sql:685: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:685: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:685: DEBUG: refreshing continuous aggregate "cagg_4_hours_origin" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:685: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_origin" in window [ Wed Jan 01 21:00:00 2020 PST, Thu Jan 02 13:00:00 2020 PST ] psql:include/cagg_query_common.sql:685: LOG: merged 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_35" psql:include/cagg_query_common.sql:685: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_35" @@ -1634,9 +1637,9 @@ SELECT time_bucket('4 hour', time, '2000-01-01 01:00:00 PST'::timestamptz), max( -- Test invalidations TRUNCATE temperature; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); INSERT INTO temperature SELECT time, 5 FROM generate_series('2000-01-01 01:00:00 PST'::timestamptz, @@ -1649,8 +1652,9 @@ INSERT INTO temperature values('2020-01-02 01:05:00+01', 2222); INSERT INTO temperature values('2020-01-02 01:35:00+01', 5555); INSERT INTO temperature values('2020-01-02 05:05:00+01', 8888); SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -psql:include/cagg_query_common.sql:725: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:725: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:725: DEBUG: refreshing continuous aggregate "cagg_4_hours" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:725: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577952000000000 psql:include/cagg_query_common.sql:725: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Sat Jan 01 00:00:00 2000 PST, Sun Jan 02 00:00:00 2000 PST ] psql:include/cagg_query_common.sql:725: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" @@ -1658,8 +1662,9 @@ psql:include/cagg_query_common.sql:725: DEBUG: hypertable 33 existing watermark psql:include/cagg_query_common.sql:725: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Wed Jan 01 00:00:00 2020 PST, Thu Jan 02 00:00:00 2020 PST ] psql:include/cagg_query_common.sql:725: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" psql:include/cagg_query_common.sql:725: DEBUG: hypertable 33 existing watermark >= new watermark 1577995200000000 1577952000000000 -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -psql:include/cagg_query_common.sql:726: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:726: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:726: DEBUG: refreshing continuous aggregate "cagg_4_hours_offset" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:726: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577953800000000 psql:include/cagg_query_common.sql:726: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Sat Jan 01 00:30:00 2000 PST, Sun Jan 02 00:30:00 2000 PST ] psql:include/cagg_query_common.sql:726: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" @@ -1667,8 +1672,9 @@ psql:include/cagg_query_common.sql:726: DEBUG: hypertable 34 existing watermark psql:include/cagg_query_common.sql:726: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Tue Dec 31 20:30:00 2019 PST, Thu Jan 02 00:30:00 2020 PST ] psql:include/cagg_query_common.sql:726: LOG: inserted 7 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:726: DEBUG: hypertable 34 existing watermark >= new watermark 1577997000000000 1577953800000000 -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); -psql:include/cagg_query_common.sql:727: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:727: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:727: DEBUG: refreshing continuous aggregate "cagg_4_hours_origin" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:727: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577955600000000 psql:include/cagg_query_common.sql:727: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_origin" in window [ Sat Jan 01 01:00:00 2000 PST, Sun Jan 02 01:00:00 2000 PST ] psql:include/cagg_query_common.sql:727: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_35" @@ -1912,7 +1918,7 @@ ORDER BY 1, 2, 3; 33 | 1577995200000000 | 9223372036854775807 33 | 1577995200000000 | 9223372036854775807 -CALL refresh_continuous_aggregate('cagg_4_hours', '2000-01-01 00:00:00'::timestamptz, '2020-12-31 23:59:59'::timestamptz); +CALL refresh_continuous_aggregate('cagg_4_hours', '2000-01-01 00:00:00'::timestamptz, '2020-12-31 23:59:59'::timestamptz, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log WHERE materialization_id IN (SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'cagg_4_hours') @@ -2040,7 +2046,7 @@ SELECT * FROM cagg_int; 50 | 5 100 | 555 -CALL refresh_continuous_aggregate('cagg_int', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_int', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_int; time_bucket | value -------------+------- @@ -2089,7 +2095,7 @@ SELECT * FROM cagg_int_offset; -- the value 100 is part of the already serializ 45 | 30 95 | 555 -CALL refresh_continuous_aggregate('cagg_int_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_int_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_int_offset; time_bucket | value -------------+------- @@ -2144,8 +2150,9 @@ SELECT * FROM cagg_int_offset; -- Check that the refresh is properly aligned INSERT INTO table_int VALUES(114, 0); SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130); -psql:include/cagg_query_common.sql:845: LOG: statement: CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130); +CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:845: LOG: statement: CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:845: DEBUG: refreshing continuous aggregate "cagg_int_offset" from 100 to 130 psql:include/cagg_query_common.sql:845: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_int_offset" in window [ 105, 125 ] psql:include/cagg_query_common.sql:845: DEBUG: building index "_hyper_38_68_chunk__materialized_hypertable_38_time_bucket_idx" on table "_hyper_38_68_chunk" serially psql:include/cagg_query_common.sql:845: DEBUG: index "_hyper_38_68_chunk__materialized_hypertable_38_time_bucket_idx" can safely use deduplication @@ -2363,8 +2370,8 @@ SELECT time_bucket('1 week', time, origin=>'2000-01-02 01:00:00 PST'::timestampt Sun Dec 30 01:00:00 2029 PST | 55555 -- Test refresh -CALL refresh_continuous_aggregate('cagg_1_hour_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_1_week_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_1_hour_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_1_week_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); -- Everything should be now materailized ALTER MATERIALIZED VIEW cagg_1_hour_offset SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW cagg_1_week_offset SET (timescaledb.materialized_only=false); diff --git a/tsl/test/expected/cagg_query_using_merge-16.out b/tsl/test/expected/cagg_query_using_merge-16.out index 1fdf3c7bc2f..f2a29c63102 100644 --- a/tsl/test/expected/cagg_query_using_merge-16.out +++ b/tsl/test/expected/cagg_query_using_merge-16.out @@ -64,7 +64,7 @@ FROM ( select max(timec)as timeval from conditions ) as q; ------------------------------ Sat Nov 03 17:00:00 2018 PDT -CALL refresh_continuous_aggregate('mat_m1', NULL, NULL); +CALL refresh_continuous_aggregate('mat_m1', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); --test first/last create materialized view mat_m2(location, timec, firsth, lasth, maxtemp, mintemp) WITH (timescaledb.continuous, timescaledb.materialized_only=false) @@ -79,7 +79,7 @@ FROM ( select max(timec)as timeval from conditions ) as q; ------------------------------ Sat Nov 03 17:00:00 2018 PDT -CALL refresh_continuous_aggregate('mat_m2', NULL, NULL); +CALL refresh_continuous_aggregate('mat_m2', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); --normal view -- create or replace view regview( location, timec, minl, sumt , sumh) as @@ -1083,7 +1083,7 @@ SET ROLE :ROLE_SUPERUSER; BEGIN; UPDATE _timescaledb_catalog.continuous_aggs_bucket_function SET bucket_func = 'func_does_not_exist()'; -- should error because function does not exist -CALL refresh_continuous_aggregate('cagg_bigint_offset2', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_bigint_offset2', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); psql:include/cagg_query_common.sql:566: ERROR: function "func_does_not_exist()" does not exist ROLLBACK; \set ON_ERROR_STOP 1 @@ -1300,9 +1300,9 @@ SELECT * FROM cagg_4_hours_origin; -- Update the last bucket and re-materialize INSERT INTO temperature values('2020-01-01 23:55:00 PST', 10); -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_4_hours; time_bucket | max ------------------------------+----- @@ -1435,17 +1435,20 @@ SELECT * FROM cagg_4_hours_origin; -- Update materialized data SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -psql:include/cagg_query_common.sql:683: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:683: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:683: DEBUG: refreshing continuous aggregate "cagg_4_hours" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:683: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Thu Jan 02 00:00:00 2020 PST, Thu Jan 02 12:00:00 2020 PST ] psql:include/cagg_query_common.sql:683: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -psql:include/cagg_query_common.sql:684: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:684: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:684: DEBUG: refreshing continuous aggregate "cagg_4_hours_offset" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:684: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Wed Jan 01 20:30:00 2020 PST, Thu Jan 02 12:30:00 2020 PST ] psql:include/cagg_query_common.sql:684: LOG: merged 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:684: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_34" -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); -psql:include/cagg_query_common.sql:685: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:685: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:685: DEBUG: refreshing continuous aggregate "cagg_4_hours_origin" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:685: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_origin" in window [ Wed Jan 01 21:00:00 2020 PST, Thu Jan 02 13:00:00 2020 PST ] psql:include/cagg_query_common.sql:685: LOG: merged 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_35" psql:include/cagg_query_common.sql:685: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_35" @@ -1628,9 +1631,9 @@ SELECT time_bucket('4 hour', time, '2000-01-01 01:00:00 PST'::timestamptz), max( -- Test invalidations TRUNCATE temperature; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); INSERT INTO temperature SELECT time, 5 FROM generate_series('2000-01-01 01:00:00 PST'::timestamptz, @@ -1643,8 +1646,9 @@ INSERT INTO temperature values('2020-01-02 01:05:00+01', 2222); INSERT INTO temperature values('2020-01-02 01:35:00+01', 5555); INSERT INTO temperature values('2020-01-02 05:05:00+01', 8888); SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -psql:include/cagg_query_common.sql:725: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:725: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:725: DEBUG: refreshing continuous aggregate "cagg_4_hours" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:725: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577952000000000 psql:include/cagg_query_common.sql:725: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Sat Jan 01 00:00:00 2000 PST, Sun Jan 02 00:00:00 2000 PST ] psql:include/cagg_query_common.sql:725: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" @@ -1652,8 +1656,9 @@ psql:include/cagg_query_common.sql:725: DEBUG: hypertable 33 existing watermark psql:include/cagg_query_common.sql:725: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Wed Jan 01 00:00:00 2020 PST, Thu Jan 02 00:00:00 2020 PST ] psql:include/cagg_query_common.sql:725: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" psql:include/cagg_query_common.sql:725: DEBUG: hypertable 33 existing watermark >= new watermark 1577995200000000 1577952000000000 -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -psql:include/cagg_query_common.sql:726: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:726: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:726: DEBUG: refreshing continuous aggregate "cagg_4_hours_offset" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:726: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577953800000000 psql:include/cagg_query_common.sql:726: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Sat Jan 01 00:30:00 2000 PST, Sun Jan 02 00:30:00 2000 PST ] psql:include/cagg_query_common.sql:726: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" @@ -1661,8 +1666,9 @@ psql:include/cagg_query_common.sql:726: DEBUG: hypertable 34 existing watermark psql:include/cagg_query_common.sql:726: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Tue Dec 31 20:30:00 2019 PST, Thu Jan 02 00:30:00 2020 PST ] psql:include/cagg_query_common.sql:726: LOG: inserted 7 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:726: DEBUG: hypertable 34 existing watermark >= new watermark 1577997000000000 1577953800000000 -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); -psql:include/cagg_query_common.sql:727: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:727: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:727: DEBUG: refreshing continuous aggregate "cagg_4_hours_origin" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:727: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577955600000000 psql:include/cagg_query_common.sql:727: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_origin" in window [ Sat Jan 01 01:00:00 2000 PST, Sun Jan 02 01:00:00 2000 PST ] psql:include/cagg_query_common.sql:727: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_35" @@ -1906,7 +1912,7 @@ ORDER BY 1, 2, 3; 33 | 1577995200000000 | 9223372036854775807 33 | 1577995200000000 | 9223372036854775807 -CALL refresh_continuous_aggregate('cagg_4_hours', '2000-01-01 00:00:00'::timestamptz, '2020-12-31 23:59:59'::timestamptz); +CALL refresh_continuous_aggregate('cagg_4_hours', '2000-01-01 00:00:00'::timestamptz, '2020-12-31 23:59:59'::timestamptz, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log WHERE materialization_id IN (SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'cagg_4_hours') @@ -2034,7 +2040,7 @@ SELECT * FROM cagg_int; 50 | 5 100 | 555 -CALL refresh_continuous_aggregate('cagg_int', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_int', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_int; time_bucket | value -------------+------- @@ -2083,7 +2089,7 @@ SELECT * FROM cagg_int_offset; -- the value 100 is part of the already serializ 45 | 30 95 | 555 -CALL refresh_continuous_aggregate('cagg_int_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_int_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_int_offset; time_bucket | value -------------+------- @@ -2138,8 +2144,9 @@ SELECT * FROM cagg_int_offset; -- Check that the refresh is properly aligned INSERT INTO table_int VALUES(114, 0); SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130); -psql:include/cagg_query_common.sql:845: LOG: statement: CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130); +CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:845: LOG: statement: CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:845: DEBUG: refreshing continuous aggregate "cagg_int_offset" from 100 to 130 psql:include/cagg_query_common.sql:845: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_int_offset" in window [ 105, 125 ] psql:include/cagg_query_common.sql:845: DEBUG: building index "_hyper_38_68_chunk__materialized_hypertable_38_time_bucket_idx" on table "_hyper_38_68_chunk" serially psql:include/cagg_query_common.sql:845: DEBUG: index "_hyper_38_68_chunk__materialized_hypertable_38_time_bucket_idx" can safely use deduplication @@ -2357,8 +2364,8 @@ SELECT time_bucket('1 week', time, origin=>'2000-01-02 01:00:00 PST'::timestampt Sun Dec 30 01:00:00 2029 PST | 55555 -- Test refresh -CALL refresh_continuous_aggregate('cagg_1_hour_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_1_week_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_1_hour_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_1_week_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); -- Everything should be now materailized ALTER MATERIALIZED VIEW cagg_1_hour_offset SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW cagg_1_week_offset SET (timescaledb.materialized_only=false); diff --git a/tsl/test/expected/cagg_query_using_merge-17.out b/tsl/test/expected/cagg_query_using_merge-17.out index 1fdf3c7bc2f..f2a29c63102 100644 --- a/tsl/test/expected/cagg_query_using_merge-17.out +++ b/tsl/test/expected/cagg_query_using_merge-17.out @@ -64,7 +64,7 @@ FROM ( select max(timec)as timeval from conditions ) as q; ------------------------------ Sat Nov 03 17:00:00 2018 PDT -CALL refresh_continuous_aggregate('mat_m1', NULL, NULL); +CALL refresh_continuous_aggregate('mat_m1', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); --test first/last create materialized view mat_m2(location, timec, firsth, lasth, maxtemp, mintemp) WITH (timescaledb.continuous, timescaledb.materialized_only=false) @@ -79,7 +79,7 @@ FROM ( select max(timec)as timeval from conditions ) as q; ------------------------------ Sat Nov 03 17:00:00 2018 PDT -CALL refresh_continuous_aggregate('mat_m2', NULL, NULL); +CALL refresh_continuous_aggregate('mat_m2', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); --normal view -- create or replace view regview( location, timec, minl, sumt , sumh) as @@ -1083,7 +1083,7 @@ SET ROLE :ROLE_SUPERUSER; BEGIN; UPDATE _timescaledb_catalog.continuous_aggs_bucket_function SET bucket_func = 'func_does_not_exist()'; -- should error because function does not exist -CALL refresh_continuous_aggregate('cagg_bigint_offset2', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_bigint_offset2', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); psql:include/cagg_query_common.sql:566: ERROR: function "func_does_not_exist()" does not exist ROLLBACK; \set ON_ERROR_STOP 1 @@ -1300,9 +1300,9 @@ SELECT * FROM cagg_4_hours_origin; -- Update the last bucket and re-materialize INSERT INTO temperature values('2020-01-01 23:55:00 PST', 10); -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_4_hours; time_bucket | max ------------------------------+----- @@ -1435,17 +1435,20 @@ SELECT * FROM cagg_4_hours_origin; -- Update materialized data SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -psql:include/cagg_query_common.sql:683: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:683: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:683: DEBUG: refreshing continuous aggregate "cagg_4_hours" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:683: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Thu Jan 02 00:00:00 2020 PST, Thu Jan 02 12:00:00 2020 PST ] psql:include/cagg_query_common.sql:683: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -psql:include/cagg_query_common.sql:684: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:684: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:684: DEBUG: refreshing continuous aggregate "cagg_4_hours_offset" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:684: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Wed Jan 01 20:30:00 2020 PST, Thu Jan 02 12:30:00 2020 PST ] psql:include/cagg_query_common.sql:684: LOG: merged 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:684: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_34" -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); -psql:include/cagg_query_common.sql:685: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:685: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:685: DEBUG: refreshing continuous aggregate "cagg_4_hours_origin" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:685: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_origin" in window [ Wed Jan 01 21:00:00 2020 PST, Thu Jan 02 13:00:00 2020 PST ] psql:include/cagg_query_common.sql:685: LOG: merged 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_35" psql:include/cagg_query_common.sql:685: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_35" @@ -1628,9 +1631,9 @@ SELECT time_bucket('4 hour', time, '2000-01-01 01:00:00 PST'::timestamptz), max( -- Test invalidations TRUNCATE temperature; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); INSERT INTO temperature SELECT time, 5 FROM generate_series('2000-01-01 01:00:00 PST'::timestamptz, @@ -1643,8 +1646,9 @@ INSERT INTO temperature values('2020-01-02 01:05:00+01', 2222); INSERT INTO temperature values('2020-01-02 01:35:00+01', 5555); INSERT INTO temperature values('2020-01-02 05:05:00+01', 8888); SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -psql:include/cagg_query_common.sql:725: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:725: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:725: DEBUG: refreshing continuous aggregate "cagg_4_hours" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:725: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577952000000000 psql:include/cagg_query_common.sql:725: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Sat Jan 01 00:00:00 2000 PST, Sun Jan 02 00:00:00 2000 PST ] psql:include/cagg_query_common.sql:725: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" @@ -1652,8 +1656,9 @@ psql:include/cagg_query_common.sql:725: DEBUG: hypertable 33 existing watermark psql:include/cagg_query_common.sql:725: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Wed Jan 01 00:00:00 2020 PST, Thu Jan 02 00:00:00 2020 PST ] psql:include/cagg_query_common.sql:725: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" psql:include/cagg_query_common.sql:725: DEBUG: hypertable 33 existing watermark >= new watermark 1577995200000000 1577952000000000 -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -psql:include/cagg_query_common.sql:726: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:726: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:726: DEBUG: refreshing continuous aggregate "cagg_4_hours_offset" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:726: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577953800000000 psql:include/cagg_query_common.sql:726: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Sat Jan 01 00:30:00 2000 PST, Sun Jan 02 00:30:00 2000 PST ] psql:include/cagg_query_common.sql:726: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" @@ -1661,8 +1666,9 @@ psql:include/cagg_query_common.sql:726: DEBUG: hypertable 34 existing watermark psql:include/cagg_query_common.sql:726: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Tue Dec 31 20:30:00 2019 PST, Thu Jan 02 00:30:00 2020 PST ] psql:include/cagg_query_common.sql:726: LOG: inserted 7 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:726: DEBUG: hypertable 34 existing watermark >= new watermark 1577997000000000 1577953800000000 -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); -psql:include/cagg_query_common.sql:727: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:727: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:727: DEBUG: refreshing continuous aggregate "cagg_4_hours_origin" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:727: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577955600000000 psql:include/cagg_query_common.sql:727: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_origin" in window [ Sat Jan 01 01:00:00 2000 PST, Sun Jan 02 01:00:00 2000 PST ] psql:include/cagg_query_common.sql:727: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_35" @@ -1906,7 +1912,7 @@ ORDER BY 1, 2, 3; 33 | 1577995200000000 | 9223372036854775807 33 | 1577995200000000 | 9223372036854775807 -CALL refresh_continuous_aggregate('cagg_4_hours', '2000-01-01 00:00:00'::timestamptz, '2020-12-31 23:59:59'::timestamptz); +CALL refresh_continuous_aggregate('cagg_4_hours', '2000-01-01 00:00:00'::timestamptz, '2020-12-31 23:59:59'::timestamptz, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log WHERE materialization_id IN (SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'cagg_4_hours') @@ -2034,7 +2040,7 @@ SELECT * FROM cagg_int; 50 | 5 100 | 555 -CALL refresh_continuous_aggregate('cagg_int', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_int', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_int; time_bucket | value -------------+------- @@ -2083,7 +2089,7 @@ SELECT * FROM cagg_int_offset; -- the value 100 is part of the already serializ 45 | 30 95 | 555 -CALL refresh_continuous_aggregate('cagg_int_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_int_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_int_offset; time_bucket | value -------------+------- @@ -2138,8 +2144,9 @@ SELECT * FROM cagg_int_offset; -- Check that the refresh is properly aligned INSERT INTO table_int VALUES(114, 0); SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130); -psql:include/cagg_query_common.sql:845: LOG: statement: CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130); +CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:845: LOG: statement: CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:845: DEBUG: refreshing continuous aggregate "cagg_int_offset" from 100 to 130 psql:include/cagg_query_common.sql:845: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_int_offset" in window [ 105, 125 ] psql:include/cagg_query_common.sql:845: DEBUG: building index "_hyper_38_68_chunk__materialized_hypertable_38_time_bucket_idx" on table "_hyper_38_68_chunk" serially psql:include/cagg_query_common.sql:845: DEBUG: index "_hyper_38_68_chunk__materialized_hypertable_38_time_bucket_idx" can safely use deduplication @@ -2357,8 +2364,8 @@ SELECT time_bucket('1 week', time, origin=>'2000-01-02 01:00:00 PST'::timestampt Sun Dec 30 01:00:00 2029 PST | 55555 -- Test refresh -CALL refresh_continuous_aggregate('cagg_1_hour_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_1_week_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_1_hour_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_1_week_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); -- Everything should be now materailized ALTER MATERIALIZED VIEW cagg_1_hour_offset SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW cagg_1_week_offset SET (timescaledb.materialized_only=false); diff --git a/tsl/test/expected/cagg_query_using_merge-18.out b/tsl/test/expected/cagg_query_using_merge-18.out index 1fdf3c7bc2f..f2a29c63102 100644 --- a/tsl/test/expected/cagg_query_using_merge-18.out +++ b/tsl/test/expected/cagg_query_using_merge-18.out @@ -64,7 +64,7 @@ FROM ( select max(timec)as timeval from conditions ) as q; ------------------------------ Sat Nov 03 17:00:00 2018 PDT -CALL refresh_continuous_aggregate('mat_m1', NULL, NULL); +CALL refresh_continuous_aggregate('mat_m1', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); --test first/last create materialized view mat_m2(location, timec, firsth, lasth, maxtemp, mintemp) WITH (timescaledb.continuous, timescaledb.materialized_only=false) @@ -79,7 +79,7 @@ FROM ( select max(timec)as timeval from conditions ) as q; ------------------------------ Sat Nov 03 17:00:00 2018 PDT -CALL refresh_continuous_aggregate('mat_m2', NULL, NULL); +CALL refresh_continuous_aggregate('mat_m2', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); --normal view -- create or replace view regview( location, timec, minl, sumt , sumh) as @@ -1083,7 +1083,7 @@ SET ROLE :ROLE_SUPERUSER; BEGIN; UPDATE _timescaledb_catalog.continuous_aggs_bucket_function SET bucket_func = 'func_does_not_exist()'; -- should error because function does not exist -CALL refresh_continuous_aggregate('cagg_bigint_offset2', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_bigint_offset2', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); psql:include/cagg_query_common.sql:566: ERROR: function "func_does_not_exist()" does not exist ROLLBACK; \set ON_ERROR_STOP 1 @@ -1300,9 +1300,9 @@ SELECT * FROM cagg_4_hours_origin; -- Update the last bucket and re-materialize INSERT INTO temperature values('2020-01-01 23:55:00 PST', 10); -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_4_hours; time_bucket | max ------------------------------+----- @@ -1435,17 +1435,20 @@ SELECT * FROM cagg_4_hours_origin; -- Update materialized data SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -psql:include/cagg_query_common.sql:683: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:683: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:683: DEBUG: refreshing continuous aggregate "cagg_4_hours" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:683: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Thu Jan 02 00:00:00 2020 PST, Thu Jan 02 12:00:00 2020 PST ] psql:include/cagg_query_common.sql:683: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -psql:include/cagg_query_common.sql:684: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:684: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:684: DEBUG: refreshing continuous aggregate "cagg_4_hours_offset" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:684: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Wed Jan 01 20:30:00 2020 PST, Thu Jan 02 12:30:00 2020 PST ] psql:include/cagg_query_common.sql:684: LOG: merged 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:684: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_34" -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); -psql:include/cagg_query_common.sql:685: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:685: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:685: DEBUG: refreshing continuous aggregate "cagg_4_hours_origin" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:685: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_origin" in window [ Wed Jan 01 21:00:00 2020 PST, Thu Jan 02 13:00:00 2020 PST ] psql:include/cagg_query_common.sql:685: LOG: merged 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_35" psql:include/cagg_query_common.sql:685: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_35" @@ -1628,9 +1631,9 @@ SELECT time_bucket('4 hour', time, '2000-01-01 01:00:00 PST'::timestamptz), max( -- Test invalidations TRUNCATE temperature; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); INSERT INTO temperature SELECT time, 5 FROM generate_series('2000-01-01 01:00:00 PST'::timestamptz, @@ -1643,8 +1646,9 @@ INSERT INTO temperature values('2020-01-02 01:05:00+01', 2222); INSERT INTO temperature values('2020-01-02 01:35:00+01', 5555); INSERT INTO temperature values('2020-01-02 05:05:00+01', 8888); SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -psql:include/cagg_query_common.sql:725: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:725: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:725: DEBUG: refreshing continuous aggregate "cagg_4_hours" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:725: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577952000000000 psql:include/cagg_query_common.sql:725: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Sat Jan 01 00:00:00 2000 PST, Sun Jan 02 00:00:00 2000 PST ] psql:include/cagg_query_common.sql:725: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" @@ -1652,8 +1656,9 @@ psql:include/cagg_query_common.sql:725: DEBUG: hypertable 33 existing watermark psql:include/cagg_query_common.sql:725: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Wed Jan 01 00:00:00 2020 PST, Thu Jan 02 00:00:00 2020 PST ] psql:include/cagg_query_common.sql:725: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" psql:include/cagg_query_common.sql:725: DEBUG: hypertable 33 existing watermark >= new watermark 1577995200000000 1577952000000000 -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -psql:include/cagg_query_common.sql:726: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:726: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:726: DEBUG: refreshing continuous aggregate "cagg_4_hours_offset" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:726: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577953800000000 psql:include/cagg_query_common.sql:726: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Sat Jan 01 00:30:00 2000 PST, Sun Jan 02 00:30:00 2000 PST ] psql:include/cagg_query_common.sql:726: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" @@ -1661,8 +1666,9 @@ psql:include/cagg_query_common.sql:726: DEBUG: hypertable 34 existing watermark psql:include/cagg_query_common.sql:726: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Tue Dec 31 20:30:00 2019 PST, Thu Jan 02 00:30:00 2020 PST ] psql:include/cagg_query_common.sql:726: LOG: inserted 7 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:726: DEBUG: hypertable 34 existing watermark >= new watermark 1577997000000000 1577953800000000 -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); -psql:include/cagg_query_common.sql:727: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:727: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:727: DEBUG: refreshing continuous aggregate "cagg_4_hours_origin" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:727: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577955600000000 psql:include/cagg_query_common.sql:727: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_origin" in window [ Sat Jan 01 01:00:00 2000 PST, Sun Jan 02 01:00:00 2000 PST ] psql:include/cagg_query_common.sql:727: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_35" @@ -1906,7 +1912,7 @@ ORDER BY 1, 2, 3; 33 | 1577995200000000 | 9223372036854775807 33 | 1577995200000000 | 9223372036854775807 -CALL refresh_continuous_aggregate('cagg_4_hours', '2000-01-01 00:00:00'::timestamptz, '2020-12-31 23:59:59'::timestamptz); +CALL refresh_continuous_aggregate('cagg_4_hours', '2000-01-01 00:00:00'::timestamptz, '2020-12-31 23:59:59'::timestamptz, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log WHERE materialization_id IN (SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'cagg_4_hours') @@ -2034,7 +2040,7 @@ SELECT * FROM cagg_int; 50 | 5 100 | 555 -CALL refresh_continuous_aggregate('cagg_int', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_int', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_int; time_bucket | value -------------+------- @@ -2083,7 +2089,7 @@ SELECT * FROM cagg_int_offset; -- the value 100 is part of the already serializ 45 | 30 95 | 555 -CALL refresh_continuous_aggregate('cagg_int_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_int_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_int_offset; time_bucket | value -------------+------- @@ -2138,8 +2144,9 @@ SELECT * FROM cagg_int_offset; -- Check that the refresh is properly aligned INSERT INTO table_int VALUES(114, 0); SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130); -psql:include/cagg_query_common.sql:845: LOG: statement: CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130); +CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:845: LOG: statement: CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:845: DEBUG: refreshing continuous aggregate "cagg_int_offset" from 100 to 130 psql:include/cagg_query_common.sql:845: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_int_offset" in window [ 105, 125 ] psql:include/cagg_query_common.sql:845: DEBUG: building index "_hyper_38_68_chunk__materialized_hypertable_38_time_bucket_idx" on table "_hyper_38_68_chunk" serially psql:include/cagg_query_common.sql:845: DEBUG: index "_hyper_38_68_chunk__materialized_hypertable_38_time_bucket_idx" can safely use deduplication @@ -2357,8 +2364,8 @@ SELECT time_bucket('1 week', time, origin=>'2000-01-02 01:00:00 PST'::timestampt Sun Dec 30 01:00:00 2029 PST | 55555 -- Test refresh -CALL refresh_continuous_aggregate('cagg_1_hour_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_1_week_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_1_hour_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_1_week_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); -- Everything should be now materailized ALTER MATERIALIZED VIEW cagg_1_hour_offset SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW cagg_1_week_offset SET (timescaledb.materialized_only=false); diff --git a/tsl/test/expected/cagg_refresh_incremental.out b/tsl/test/expected/cagg_refresh_incremental.out new file mode 100644 index 00000000000..f5146b346a0 --- /dev/null +++ b/tsl/test/expected/cagg_refresh_incremental.out @@ -0,0 +1,722 @@ +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license. +\c :TEST_DBNAME :ROLE_SUPERUSER +-- Create a user with specific timezone for deterministic output +CREATE ROLE test_cagg_refresh_manual_user WITH LOGIN; +ALTER ROLE test_cagg_refresh_manual_user SET timezone TO 'UTC'; +GRANT ALL ON SCHEMA public TO test_cagg_refresh_manual_user; +\c :TEST_DBNAME test_cagg_refresh_manual_user +SET timezone TO 'UTC'; +CREATE TABLE conditions ( + time TIMESTAMP WITH TIME ZONE NOT NULL, + device_id INTEGER, + temperature NUMERIC +); +SELECT FROM create_hypertable('conditions', by_range('time')); +-- + +INSERT INTO conditions +SELECT + t, d, 10 +FROM + generate_series( + '2025-02-05 00:00:00+00', + '2025-03-05 00:00:00+00', + '1 hour'::interval) AS t, + generate_series(1,5) AS d; +CREATE MATERIALIZED VIEW conditions_by_day +WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS +SELECT + time_bucket('1 day', time), + device_id, + count(*), + min(temperature), + max(temperature), + avg(temperature), + sum(temperature) +FROM + conditions +GROUP BY + 1, 2 +WITH NO DATA; +-- Issue an incremental manual refresh using JSONB options: +-- buckets_per_batch => 10. This is the manual equivalent of a +-- policy with the same setting. +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 10}'::jsonb); +LOG: statement: CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 10}'::jsonb); +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 25 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 50 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 50 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 20 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +SELECT count(*) FROM conditions_by_day; + count +------- + 145 + +CREATE MATERIALIZED VIEW conditions_by_day_atomic_refresh +WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS +SELECT + time_bucket('1 day', time), + device_id, + count(*), + min(temperature), + max(temperature), + avg(temperature), + sum(temperature) +FROM + conditions +GROUP BY + 1, 2 +WITH NO DATA; +CALL refresh_continuous_aggregate('conditions_by_day_atomic_refresh', NULL, NULL); +SELECT count(*) FROM conditions_by_day; + count +------- + 145 + +SELECT count(*) FROM conditions_by_day_atomic_refresh; + count +------- + 145 + +-- Should have no differences +SELECT + count(*) > 0 AS has_diff +FROM + ((SELECT * FROM conditions_by_day_atomic_refresh ORDER BY 1, 2) + EXCEPT + (SELECT * FROM conditions_by_day ORDER BY 1, 2)) AS diff; + has_diff +---------- + f + +-- buckets_per_batch => 0 is NOT incremental: the whole window is materialized +-- in a single pass. Under LOG this shows exactly one delete+insert pair, unlike +-- the multi-batch run above. The TRUNCATE invalidates the whole range so there +-- is data to re-materialize. +TRUNCATE conditions_by_day; +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 0}'::jsonb); +LOG: statement: CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 0}'::jsonb); +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 145 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +SELECT count(*) FROM conditions_by_day; + count +------- + 145 + +-- The continuous aggregate is now fully materialized with no pending +-- invalidations, so a normal refresh is a no-op (reports up-to-date). +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate('conditions_by_day', NULL, NULL); +LOG: statement: CALL refresh_continuous_aggregate('conditions_by_day', NULL, NULL); +NOTICE: continuous aggregate "conditions_by_day" is already up-to-date +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +-- Assert there really are no pending invalidations +SELECT + (SELECT count(*) + FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log + WHERE materialization_id = cagg.mat_hypertable_id + AND greatest_modified_value >= lowest_modified_value + AND lowest_modified_value != -9223372036854775808 + AND greatest_modified_value != 9223372036854775807) AS mat_invalidations, + (SELECT count(*) + FROM _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log + WHERE hypertable_id = cagg.raw_hypertable_id + AND greatest_modified_value >= lowest_modified_value) AS ht_invalidations +FROM _timescaledb_catalog.continuous_agg cagg +WHERE cagg.user_view_name = 'conditions_by_day'; + mat_invalidations | ht_invalidations +-------------------+------------------ + 0 | 0 + +-- A forced refresh is incremental but ignores the invalidation logs: +-- it re-materializes the entire window in batches (buckets_per_batch +-- defaults to 10) even though nothing is invalidated. +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate('conditions_by_day', NULL, NULL, force => true); +LOG: statement: CALL refresh_continuous_aggregate('conditions_by_day', NULL, NULL, force => true); +LOG: deleted 25 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 25 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: deleted 50 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 50 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: deleted 50 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 50 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: deleted 20 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 20 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +SELECT count(*) FROM conditions_by_day; + count +------- + 145 + +TRUNCATE conditions_by_day; +-- Run with max_batches_per_execution => 2. Manual refresh stops mid-window +-- after processing 2 batches, leaving the rest for subsequent calls. +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 10, "max_batches_per_execution": 2}'::jsonb); +LOG: statement: CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 10, "max_batches_per_execution": 2}'::jsonb); +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 25 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 50 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: reached maximum number of batches per execution (2), batches not processed (2) +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +SELECT count(*) FROM conditions_by_day; + count +------- + 75 + +SELECT count(*) FROM conditions_by_day_atomic_refresh; + count +------- + 145 + +-- Should have differences (partial materialization) +SELECT + count(*) > 0 AS has_diff +FROM + ((SELECT * FROM conditions_by_day_atomic_refresh ORDER BY 1, 2) + EXCEPT + (SELECT * FROM conditions_by_day ORDER BY 1, 2)) AS diff; + has_diff +---------- + t + +-- Run a second call (same options) to process more batches. +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 10, "max_batches_per_execution": 2}'::jsonb); +LOG: statement: CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 10, "max_batches_per_execution": 2}'::jsonb); +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 50 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 20 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +SELECT count(*) FROM conditions_by_day; + count +------- + 145 + +SELECT count(*) FROM conditions_by_day_atomic_refresh; + count +------- + 145 + +-- Should have no differences (all batches processed) +SELECT + count(*) > 0 AS has_diff +FROM + ((SELECT * FROM conditions_by_day_atomic_refresh ORDER BY 1, 2) + EXCEPT + (SELECT * FROM conditions_by_day ORDER BY 1, 2)) AS diff; + has_diff +---------- + f + +-- Set max_batches_per_execution to 10 (effectively unlimited for our window) +-- and insert data into the past so a new set of batches must be processed. +INSERT INTO conditions +SELECT + t, d, 10 +FROM + generate_series( + '2020-02-05 00:00:00+00', + '2020-03-05 00:00:00+00', + '1 hour'::interval) AS t, + generate_series(1,5) AS d; +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 10, "max_batches_per_execution": 10}'::jsonb); +LOG: statement: CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 10, "max_batches_per_execution": 10}'::jsonb); +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 50 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 50 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 50 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +SELECT count(*) FROM conditions_by_day; + count +------- + 295 + +SELECT count(*) FROM conditions_by_day_atomic_refresh; + count +------- + 145 + +CALL refresh_continuous_aggregate('conditions_by_day_atomic_refresh', NULL, NULL); +SELECT count(*) FROM conditions_by_day; + count +------- + 295 + +SELECT count(*) FROM conditions_by_day_atomic_refresh; + count +------- + 295 + +-- Should have no differences +SELECT + count(*) > 0 AS has_diff +FROM + ((SELECT * FROM conditions_by_day_atomic_refresh ORDER BY 1, 2) + EXCEPT + (SELECT * FROM conditions_by_day ORDER BY 1, 2)) AS diff; + has_diff +---------- + f + +-- Invalid configurations should be rejected +\set ON_ERROR_STOP 0 +\set VERBOSITY default +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"max_batches_per_execution": -1}'::jsonb); +ERROR: invalid max batches per execution +DETAIL: max_batches_per_execution: -1 +HINT: The max batches per execution should be greater than or equal to zero. +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": -1}'::jsonb); +ERROR: invalid buckets per batch +DETAIL: buckets_per_batch: -1 +HINT: The buckets per batch should be greater than or equal to zero. +\set VERBOSITY terse +\set ON_ERROR_STOP 1 +-- Truncate all data from the original hypertable. +TRUNCATE conditions; +-- Should fall back to single-batch processing because there's no data +-- to refresh on the source hypertable. +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 10}'::jsonb); +LOG: statement: CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 10}'::jsonb); +LOG: deleted 295 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 0 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +-- Should return zero rows +SELECT count(*) FROM conditions_by_day; + count +------- + 0 + +-- Insert 1 day of data +INSERT INTO conditions +SELECT + t, d, 10 +FROM + generate_series( + '2020-02-05 00:00:00+00', + '2020-02-06 00:00:00+00', + '1 hour'::interval) AS t, + generate_series(1,5) AS d; +-- Should fall back to single-batch processing because the refresh size +-- (1 day) is smaller than 10 buckets x 1 day. +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 10}'::jsonb); +LOG: statement: CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 10}'::jsonb); +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 10 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +-- Should return 10 rows because the bucket width is `1 day` and we +-- inserted across two boundary timestamps for 5 devices. +SELECT count(*) FROM conditions_by_day; + count +------- + 10 + +TRUNCATE conditions_by_day, conditions; +-- Less than 1 day of data (smaller than the bucket width) +INSERT INTO conditions +VALUES ('2020-02-05 00:00:00+00', 1, 10); +-- Should fall back to single-batch processing because the refresh size +-- is smaller than the bucket width. +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 10}'::jsonb); +LOG: statement: CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 10}'::jsonb); +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +-- Should return 1 row +SELECT count(*) FROM conditions_by_day; + count +------- + 1 + +-- Re-test with an explicit refresh_newest_first => true (default behavior). +TRUNCATE conditions_by_day, conditions_by_day_atomic_refresh, conditions; +INSERT INTO conditions +SELECT + t, d, 10 +FROM + generate_series( + '2025-03-11 00:00:00+00'::timestamptz - INTERVAL '30 days', + '2025-03-11 00:00:00+00'::timestamptz, + '1 hour'::interval) AS t, + generate_series(1,5) AS d; +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_day', + '2025-03-11 00:00:00+00'::timestamptz - INTERVAL '15 days', + NULL, + options => '{"buckets_per_batch": 5, "refresh_newest_first": true}'::jsonb); +LOG: statement: CALL refresh_continuous_aggregate( + 'conditions_by_day', + '2025-03-11 00:00:00+00'::timestamptz - INTERVAL '15 days', + NULL, + options => '{"buckets_per_batch": 5, "refresh_newest_first": true}'::jsonb); +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 5 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 25 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 25 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 25 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +CALL refresh_continuous_aggregate( + 'conditions_by_day_atomic_refresh', + '2025-03-11 00:00:00+00'::timestamptz - INTERVAL '15 days', + NULL); +-- Both continuous aggregates should have the same data +SELECT count(*) FROM conditions_by_day; + count +------- + 80 + +SELECT count(*) FROM conditions_by_day_atomic_refresh; + count +------- + 80 + +-- Should have no differences +SELECT + count(*) > 0 AS has_diff +FROM + ((SELECT * FROM conditions_by_day_atomic_refresh ORDER BY 1, 2) + EXCEPT + (SELECT * FROM conditions_by_day ORDER BY 1, 2)) AS diff; + has_diff +---------- + f + +-- refresh_newest_first => false (process from oldest to newest) +TRUNCATE conditions_by_day; +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_day', + '2025-03-11 00:00:00+00'::timestamptz - INTERVAL '15 days', + NULL, + options => '{"buckets_per_batch": 5, "refresh_newest_first": false}'::jsonb); +LOG: statement: CALL refresh_continuous_aggregate( + 'conditions_by_day', + '2025-03-11 00:00:00+00'::timestamptz - INTERVAL '15 days', + NULL, + options => '{"buckets_per_batch": 5, "refresh_newest_first": false}'::jsonb); +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 25 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 25 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 25 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 5 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +-- Both continuous aggregates should have the same data +SELECT count(*) FROM conditions_by_day; + count +------- + 80 + +SELECT count(*) FROM conditions_by_day_atomic_refresh; + count +------- + 80 + +-- Should have no differences +SELECT + count(*) > 0 AS has_diff +FROM + ((SELECT * FROM conditions_by_day_atomic_refresh ORDER BY 1, 2) + EXCEPT + (SELECT * FROM conditions_by_day ORDER BY 1, 2)) AS diff; + has_diff +---------- + f + +-- Tests with variable-sized bucket (monthly) +TRUNCATE conditions; +INSERT INTO conditions +SELECT + t, d, 10 +FROM + generate_series( + '2025-01-01 00:00:00+00', + '2025-10-08 00:00:00+00', + '1 hour'::interval) AS t, + generate_series(1,5) AS d; +CREATE MATERIALIZED VIEW conditions_by_month +WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS +SELECT + time_bucket('1 month', time), + device_id, + count(*), + min(temperature), + max(temperature), + avg(temperature), + sum(temperature) +FROM + conditions +GROUP BY + 1, 2 +WITH NO DATA; +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_month', + '2025-03-11 00:00:00+00'::timestamptz - INTERVAL '600 days', + '2025-03-11 00:00:00+00'::timestamptz - INTERVAL '7 days', + options => '{"refresh_newest_first": false}'::jsonb); +LOG: statement: CALL refresh_continuous_aggregate( + 'conditions_by_month', + '2025-03-11 00:00:00+00'::timestamptz - INTERVAL '600 days', + '2025-03-11 00:00:00+00'::timestamptz - INTERVAL '7 days', + options => '{"refresh_newest_first": false}'::jsonb); +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_4" +LOG: inserted 10 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_4" +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +SELECT count(*) FROM conditions_by_month; + count +------- + 10 + +------------------------------------------------------------------------------------------ +-- Test that batched refresh with variable-length buckets doesn't leave remainders +------------------------------------------------------------------------------------------ +CREATE TABLE test_data ( + time TIMESTAMPTZ NOT NULL, + value INT +); +SELECT public.create_hypertable( + relation => 'test_data', + time_column_name => 'time', + chunk_time_interval => interval '1 months' +); + create_hypertable +------------------------ + (5,public,test_data,t) + +-- Insert initial data +INSERT INTO test_data +SELECT time, 1 +FROM generate_series('2024-01-01'::timestamptz, '2024-12-31'::timestamptz, '1 day'::interval) time; +-- Create continuous aggregate with monthly buckets (variable-length) +CREATE MATERIALIZED VIEW batch_test_cagg +WITH (timescaledb.continuous) AS +SELECT + time_bucket('1 month'::interval, time) AS bucket, + count(*) as count +FROM test_data +GROUP BY bucket +WITH NO DATA; +-- Run incremental manual refresh, 1 bucket per batch +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'batch_test_cagg', NULL, '2024-12-01'::timestamptz, + options => '{"buckets_per_batch": 1}'::jsonb); +LOG: statement: CALL refresh_continuous_aggregate( + 'batch_test_cagg', NULL, '2024-12-01'::timestamptz, + options => '{"buckets_per_batch": 1}'::jsonb); +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: inserted 0 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_6" +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +-- Verify that the materialization invalidation log has no entries other than +-- the boundary -/+ infinity rows. +SELECT materialization_id, + _timescaledb_functions.to_timestamp(lowest_modified_value) as low, + _timescaledb_functions.to_timestamp(greatest_modified_value) as high +FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log +WHERE materialization_id IN + (SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg + WHERE user_view_name = 'batch_test_cagg') + AND lowest_modified_value != -9223372036854775808 + AND greatest_modified_value != 9223372036854775807 +ORDER BY low; + materialization_id | low | high +--------------------+-----+------ + +-- Running again should be a no-op +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'batch_test_cagg', NULL, '2024-12-01'::timestamptz, + options => '{"buckets_per_batch": 1}'::jsonb); +LOG: statement: CALL refresh_continuous_aggregate( + 'batch_test_cagg', NULL, '2024-12-01'::timestamptz, + options => '{"buckets_per_batch": 1}'::jsonb); +NOTICE: continuous aggregate "batch_test_cagg" is already up-to-date +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +DROP TABLE test_data CASCADE; +NOTICE: drop cascades to 2 other objects +NOTICE: drop cascades to 2 other objects +------------------------------------------------------------------------------------------ +-- Test incremental manual refresh crashing between batches +------------------------------------------------------------------------------------------ +-- Rows processed by a batch should be visible immediately after it finishes. +-- Inject an error after batch 1 completes and observe the cagg state. +\c :TEST_DBNAME :ROLE_SUPERUSER +TRUNCATE _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log, _timescaledb_catalog.continuous_aggs_materialization_invalidation_log; +\c :TEST_DBNAME test_cagg_refresh_manual_user +SET timezone TO 'UTC'; +TRUNCATE conditions, conditions_by_day, conditions_by_day_atomic_refresh; +INSERT INTO conditions +SELECT + t, d, 10 +FROM + generate_series( + '2025-02-05 00:00:00+00', + '2025-03-05 00:00:00+00', + '1 hour'::interval) AS t, + generate_series(1,5) AS d; +-- Crash after batch 1 finishes +SELECT debug_waitpoint_enable('cagg_policy_batch_1_after_refresh'); + debug_waitpoint_enable +------------------------ + + +-- Cagg state before refresh starts +SELECT min(time_bucket), max(time_bucket) FROM conditions_by_day; + min | max +-----+----- + | + +SET client_min_messages TO LOG; +\set ON_ERROR_STOP 0 +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, '1 hour'::interval, + options => '{"buckets_per_batch": 10}'::jsonb); +LOG: statement: CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, '1 hour'::interval, + options => '{"buckets_per_batch": 10}'::jsonb); +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 25 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +ERROR: error injected at debug point 'cagg_policy_batch_1_after_refresh' +\set ON_ERROR_STOP 1 +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +-- Rows processed by batch 1 should be materialized +SELECT count(*) AS rows_after_refresh FROM conditions_by_day; + rows_after_refresh +-------------------- + 25 + +SELECT min(time_bucket), max(time_bucket) FROM conditions_by_day; + min | max +------------------------------+------------------------------ + Sat Mar 01 00:00:00 2025 UTC | Wed Mar 05 00:00:00 2025 UTC + +-- Registered ranges should be cleaned up after the crash +SELECT count(*) AS registered_ranges FROM _timescaledb_catalog.continuous_aggs_jobs_refresh_ranges; + registered_ranges +------------------- + 0 + +SELECT debug_waitpoint_release('cagg_policy_batch_1_after_refresh'); + debug_waitpoint_release +------------------------- + + +-- Process remaining invalidations and verify +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, '1 hour'::interval, + options => '{"buckets_per_batch": 10}'::jsonb); +-- Verify against an atomic refresh +CALL refresh_continuous_aggregate('conditions_by_day_atomic_refresh', NULL, '1 hour'::interval); +SELECT + count(*) > 0 AS has_diff +FROM + ((SELECT * FROM conditions_by_day_atomic_refresh ORDER BY 1, 2) + EXCEPT + (SELECT * FROM conditions_by_day ORDER BY 1, 2)) AS diff; + has_diff +---------- + f + +\c :TEST_DBNAME :ROLE_SUPERUSER +REASSIGN OWNED BY test_cagg_refresh_manual_user TO :ROLE_SUPERUSER; +REVOKE ALL ON SCHEMA public FROM test_cagg_refresh_manual_user; +DROP ROLE test_cagg_refresh_manual_user; diff --git a/tsl/test/expected/cagg_refresh_using_merge.out b/tsl/test/expected/cagg_refresh_using_merge.out index b9a5e2d76cf..67533c5912d 100644 --- a/tsl/test/expected/cagg_refresh_using_merge.out +++ b/tsl/test/expected/cagg_refresh_using_merge.out @@ -813,7 +813,8 @@ WITH NO DATA; SET client_min_messages TO LOG; CALL refresh_continuous_aggregate('conditions_daily', NULL, '2018-11-01 23:59:59-08'); LOG: statement: CALL refresh_continuous_aggregate('conditions_daily', NULL, '2018-11-01 23:59:59-08'); -LOG: inserted 5 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_22" +LOG: inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_22" +LOG: inserted 4 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_22" SELECT * FROM conditions_daily ORDER BY 1, 2, 3 NULLS LAST, 4 NULLS LAST, 5 NULLS LAST; LOG: statement: SELECT * FROM conditions_daily ORDER BY 1, 2, 3 NULLS LAST, 4 NULLS LAST, 5 NULLS LAST; bucket | location | avg | max | min @@ -843,7 +844,7 @@ LOG: statement: SELECT * FROM conditions_daily ORDER BY 1, 2, 3 NULLS LAST, 4 N -- All data should be in the materialization hypertable CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL); LOG: statement: CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL); -NOTICE: continuous aggregate "conditions_daily" is already up-to-date +LOG: inserted 0 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_22" SELECT * FROM conditions_daily ORDER BY 1, 2, 3 NULLS LAST, 4 NULLS LAST, 5 NULLS LAST; LOG: statement: SELECT * FROM conditions_daily ORDER BY 1, 2, 3 NULLS LAST, 4 NULLS LAST, 5 NULLS LAST; bucket | location | avg | max | min diff --git a/tsl/test/expected/cagg_watermark.out b/tsl/test/expected/cagg_watermark.out index c2ee59c29c2..6935a5b984c 100644 --- a/tsl/test/expected/cagg_watermark.out +++ b/tsl/test/expected/cagg_watermark.out @@ -479,9 +479,9 @@ CALL refresh_continuous_aggregate('chunks_1h', '2000-01-01', '2021-06-01'); Append (actual rows=3.00 loops=1) -> Append (actual rows=3.00 loops=1) -> Seq Scan on _hyper_8_17_chunk (actual rows=1.00 loops=1) - -> Seq Scan on _hyper_8_20_chunk (actual rows=1.00 loops=1) - -> Index Scan using _hyper_8_21_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_21_chunk (actual rows=1.00 loops=1) + -> Index Scan using _hyper_8_20_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_20_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < 'Wed Jul 31 18:00:00 2002 PDT'::timestamp with time zone) + -> Seq Scan on _hyper_8_21_chunk (actual rows=1.00 loops=1) -> HashAggregate (actual rows=0.00 loops=1) Group Key: time_bucket('@ 1 hour'::interval, _hyper_7_19_chunk."time"), _hyper_7_19_chunk.device -> Result (actual rows=0.00 loops=1) @@ -493,9 +493,9 @@ CALL refresh_continuous_aggregate('chunks_1h', '2000-01-01', '2021-06-01'); Append (actual rows=3.00 loops=1) -> Append (actual rows=3.00 loops=1) -> Seq Scan on _hyper_8_17_chunk (actual rows=1.00 loops=1) - -> Seq Scan on _hyper_8_20_chunk (actual rows=1.00 loops=1) - -> Index Scan using _hyper_8_21_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_21_chunk (actual rows=1.00 loops=1) + -> Index Scan using _hyper_8_20_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_20_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < 'Wed Jul 31 18:00:00 2002 PDT'::timestamp with time zone) + -> Seq Scan on _hyper_8_21_chunk (actual rows=1.00 loops=1) -> HashAggregate (actual rows=0.00 loops=1) Group Key: time_bucket('@ 1 hour'::interval, _hyper_7_19_chunk."time"), _hyper_7_19_chunk.device -> Result (actual rows=0.00 loops=1) @@ -512,9 +512,9 @@ SET timescaledb.enable_constraint_aware_append = OFF; Append (actual rows=3.00 loops=1) -> Append (actual rows=3.00 loops=1) -> Seq Scan on _hyper_8_17_chunk (actual rows=1.00 loops=1) - -> Seq Scan on _hyper_8_20_chunk (actual rows=1.00 loops=1) - -> Index Scan using _hyper_8_21_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_21_chunk (actual rows=1.00 loops=1) + -> Index Scan using _hyper_8_20_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_20_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < 'Wed Jul 31 18:00:00 2002 PDT'::timestamp with time zone) + -> Seq Scan on _hyper_8_21_chunk (actual rows=1.00 loops=1) -> HashAggregate (actual rows=0.00 loops=1) Group Key: time_bucket('@ 1 hour'::interval, _hyper_7_19_chunk."time"), _hyper_7_19_chunk.device -> Result (actual rows=0.00 loops=1) @@ -551,16 +551,16 @@ EXECUTE cagg_scan_1h; bucket | device | max ------------------------------+--------+----- Mon Jul 31 17:00:00 2000 PDT | 1 | 2 - Tue Jul 31 17:00:00 2001 PDT | 1 | 2 Wed Jul 31 17:00:00 2002 PDT | 1 | 2 + Tue Jul 31 17:00:00 2001 PDT | 1 | 2 Thu Jul 31 17:00:00 2003 PDT | 1 | 2 SELECT * FROM chunks_1h; bucket | device | max ------------------------------+--------+----- Mon Jul 31 17:00:00 2000 PDT | 1 | 2 - Tue Jul 31 17:00:00 2001 PDT | 1 | 2 Wed Jul 31 17:00:00 2002 PDT | 1 | 2 + Tue Jul 31 17:00:00 2001 PDT | 1 | 2 Thu Jul 31 17:00:00 2003 PDT | 1 | 2 INSERT INTO chunks VALUES ('2004-08-01 01:01:01+01', 1, 2); @@ -677,8 +677,8 @@ EXECUTE cagg_scan_1h; bucket | device | max ------------------------------+--------+----- Mon Jul 31 17:00:00 2000 PDT | 1 | 2 - Tue Jul 31 17:00:00 2001 PDT | 1 | 2 Wed Jul 31 17:00:00 2002 PDT | 1 | 2 + Tue Jul 31 17:00:00 2001 PDT | 1 | 2 Thu Jul 31 17:00:00 2003 PDT | 1 | 2 Sat Jul 31 17:00:00 2004 PDT | 1 | 2 Sun Jul 31 17:00:00 2005 PDT | 1 | 2 @@ -689,8 +689,8 @@ SELECT * FROM chunks_1h; bucket | device | max ------------------------------+--------+----- Mon Jul 31 17:00:00 2000 PDT | 1 | 2 - Tue Jul 31 17:00:00 2001 PDT | 1 | 2 Wed Jul 31 17:00:00 2002 PDT | 1 | 2 + Tue Jul 31 17:00:00 2001 PDT | 1 | 2 Thu Jul 31 17:00:00 2003 PDT | 1 | 2 Sat Jul 31 17:00:00 2004 PDT | 1 | 2 Sun Jul 31 17:00:00 2005 PDT | 1 | 2 @@ -704,8 +704,8 @@ EXECUTE cagg_scan_1h; bucket | device | max ------------------------------+--------+----- Mon Jul 31 17:00:00 2000 PDT | 1 | 2 - Tue Jul 31 17:00:00 2001 PDT | 1 | 2 Wed Jul 31 17:00:00 2002 PDT | 1 | 2 + Tue Jul 31 17:00:00 2001 PDT | 1 | 2 Thu Jul 31 17:00:00 2003 PDT | 1 | 2 Sat Jul 31 17:00:00 2004 PDT | 1 | 2 Sun Jul 31 17:00:00 2005 PDT | 1 | 2 @@ -716,8 +716,8 @@ SELECT * FROM chunks_1h; bucket | device | max ------------------------------+--------+----- Mon Jul 31 17:00:00 2000 PDT | 1 | 2 - Tue Jul 31 17:00:00 2001 PDT | 1 | 2 Wed Jul 31 17:00:00 2002 PDT | 1 | 2 + Tue Jul 31 17:00:00 2001 PDT | 1 | 2 Thu Jul 31 17:00:00 2003 PDT | 1 | 2 Sat Jul 31 17:00:00 2004 PDT | 1 | 2 Sun Jul 31 17:00:00 2005 PDT | 1 | 2 @@ -963,8 +963,8 @@ INSERT INTO chunks VALUES ('2014-01-01 01:01:01+01', 1, 2); CALL refresh_continuous_aggregate('chunks_1h', '2000-01-01', '2021-06-01'); :EXPLAIN_ANALYZE EXECUTE ht_scan_realtime_1h; --- QUERY PLAN --- - Append (actual rows=2.00 loops=1) - -> Custom Scan (ChunkAppend) on _materialized_hypertable_8 (actual rows=2.00 loops=1) + Append (actual rows=8.00 loops=1) + -> Custom Scan (ChunkAppend) on _materialized_hypertable_8 (actual rows=8.00 loops=1) Chunks excluded during startup: 0 -> Index Scan using _hyper_8_17_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_17_chunk (actual rows=0.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) @@ -982,17 +982,17 @@ CALL refresh_continuous_aggregate('chunks_1h', '2000-01-01', '2021-06-01'); Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) -> Index Scan using _hyper_8_31_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_31_chunk (actual rows=0.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_33_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_33_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_33_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_33_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_36_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_36_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_36_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_36_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_39_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_39_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_39_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_39_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_42_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_42_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_42_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_42_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_44_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_44_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_44_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_44_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_46_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_46_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_46_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_46_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) -> Index Scan using _hyper_8_48_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_48_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) @@ -1013,10 +1013,10 @@ INSERT INTO chunks VALUES ('2015-01-01 01:01:01+01', 1, 2); CALL refresh_continuous_aggregate('chunks_1h', '2000-01-01', '2021-06-01'); :EXPLAIN_ANALYZE SELECT device FROM chunks_1h; --- QUERY PLAN --- - Append (actual rows=3.00 loops=1) - -> Subquery Scan on "*SELECT* 1" (actual rows=3.00 loops=1) - -> Result (actual rows=3.00 loops=1) - -> Append (actual rows=3.00 loops=1) + Append (actual rows=9.00 loops=1) + -> Subquery Scan on "*SELECT* 1" (actual rows=9.00 loops=1) + -> Result (actual rows=9.00 loops=1) + -> Append (actual rows=9.00 loops=1) -> Seq Scan on _hyper_8_17_chunk (actual rows=0.00 loops=1) -> Seq Scan on _hyper_8_20_chunk (actual rows=0.00 loops=1) -> Seq Scan on _hyper_8_21_chunk (actual rows=0.00 loops=1) @@ -1025,12 +1025,12 @@ CALL refresh_continuous_aggregate('chunks_1h', '2000-01-01', '2021-06-01'); -> Seq Scan on _hyper_8_27_chunk (actual rows=0.00 loops=1) -> Seq Scan on _hyper_8_29_chunk (actual rows=0.00 loops=1) -> Seq Scan on _hyper_8_31_chunk (actual rows=0.00 loops=1) - -> Seq Scan on _hyper_8_33_chunk (actual rows=0.00 loops=1) - -> Seq Scan on _hyper_8_36_chunk (actual rows=0.00 loops=1) - -> Seq Scan on _hyper_8_39_chunk (actual rows=0.00 loops=1) - -> Seq Scan on _hyper_8_42_chunk (actual rows=0.00 loops=1) - -> Seq Scan on _hyper_8_44_chunk (actual rows=0.00 loops=1) - -> Seq Scan on _hyper_8_46_chunk (actual rows=0.00 loops=1) + -> Seq Scan on _hyper_8_33_chunk (actual rows=1.00 loops=1) + -> Seq Scan on _hyper_8_36_chunk (actual rows=1.00 loops=1) + -> Seq Scan on _hyper_8_39_chunk (actual rows=1.00 loops=1) + -> Seq Scan on _hyper_8_42_chunk (actual rows=1.00 loops=1) + -> Seq Scan on _hyper_8_44_chunk (actual rows=1.00 loops=1) + -> Seq Scan on _hyper_8_46_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_8_48_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_8_50_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_8_52_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_52_chunk (actual rows=1.00 loops=1) @@ -1055,8 +1055,8 @@ CREATE TABLE continuous_agg_test(time int, data int); -- Query without COALESCE - should not be optimized :EXPLAIN_ANALYZE (SELECT * FROM chunks_1h AS t1) UNION ALL (SELECT * from chunks_1h AS t2 WHERE _timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(:MAT_HT_ID_1H)) IS NOT NULL); --- QUERY PLAN --- - Append (actual rows=6.00 loops=1) - -> Custom Scan (ChunkAppend) on _materialized_hypertable_8 (actual rows=3.00 loops=1) + Append (actual rows=18.00 loops=1) + -> Custom Scan (ChunkAppend) on _materialized_hypertable_8 (actual rows=9.00 loops=1) Chunks excluded during startup: 0 -> Index Scan using _hyper_8_17_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_17_chunk (actual rows=0.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) @@ -1074,17 +1074,17 @@ CREATE TABLE continuous_agg_test(time int, data int); Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) -> Index Scan using _hyper_8_31_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_31_chunk (actual rows=0.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_33_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_33_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_33_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_33_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_36_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_36_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_36_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_36_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_39_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_39_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_39_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_39_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_42_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_42_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_42_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_42_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_44_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_44_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_44_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_44_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_46_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_46_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_46_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_46_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) -> Index Scan using _hyper_8_48_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_48_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) @@ -1100,10 +1100,10 @@ CREATE TABLE continuous_agg_test(time int, data int); Group Key: time_bucket('@ 1 hour'::interval, _hyper_7_51_chunk."time"), _hyper_7_51_chunk.device -> Index Scan using _hyper_7_51_chunk_chunks_time_idx on _hyper_7_51_chunk (actual rows=0.00 loops=1) Index Cond: ("time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Append (actual rows=3.00 loops=1) - -> Result (actual rows=3.00 loops=1) + -> Append (actual rows=9.00 loops=1) + -> Result (actual rows=9.00 loops=1) One-Time Filter: (_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)) IS NOT NULL) - -> Custom Scan (ChunkAppend) on _materialized_hypertable_8 _materialized_hypertable_8_1 (actual rows=3.00 loops=1) + -> Custom Scan (ChunkAppend) on _materialized_hypertable_8 _materialized_hypertable_8_1 (actual rows=9.00 loops=1) Chunks excluded during startup: 0 -> Index Scan using _hyper_8_17_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_17_chunk _hyper_8_17_chunk_1 (actual rows=0.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) @@ -1121,17 +1121,17 @@ CREATE TABLE continuous_agg_test(time int, data int); Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) -> Index Scan using _hyper_8_31_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_31_chunk _hyper_8_31_chunk_1 (actual rows=0.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_33_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_33_chunk _hyper_8_33_chunk_1 (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_33_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_33_chunk _hyper_8_33_chunk_1 (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_36_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_36_chunk _hyper_8_36_chunk_1 (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_36_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_36_chunk _hyper_8_36_chunk_1 (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_39_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_39_chunk _hyper_8_39_chunk_1 (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_39_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_39_chunk _hyper_8_39_chunk_1 (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_42_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_42_chunk _hyper_8_42_chunk_1 (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_42_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_42_chunk _hyper_8_42_chunk_1 (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_44_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_44_chunk _hyper_8_44_chunk_1 (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_44_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_44_chunk _hyper_8_44_chunk_1 (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_46_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_46_chunk _hyper_8_46_chunk_1 (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_46_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_46_chunk _hyper_8_46_chunk_1 (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) -> Index Scan using _hyper_8_48_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_48_chunk _hyper_8_48_chunk_1 (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) @@ -1154,10 +1154,10 @@ CREATE TABLE continuous_agg_test(time int, data int); :EXPLAIN_ANALYZE SELECT max(device) from chunks_1h; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) - -> Append (actual rows=3.00 loops=1) - -> Subquery Scan on "*SELECT* 1" (actual rows=3.00 loops=1) - -> Result (actual rows=3.00 loops=1) - -> Append (actual rows=3.00 loops=1) + -> Append (actual rows=9.00 loops=1) + -> Subquery Scan on "*SELECT* 1" (actual rows=9.00 loops=1) + -> Result (actual rows=9.00 loops=1) + -> Append (actual rows=9.00 loops=1) -> Seq Scan on _hyper_8_17_chunk (actual rows=0.00 loops=1) -> Seq Scan on _hyper_8_20_chunk (actual rows=0.00 loops=1) -> Seq Scan on _hyper_8_21_chunk (actual rows=0.00 loops=1) @@ -1166,12 +1166,12 @@ CREATE TABLE continuous_agg_test(time int, data int); -> Seq Scan on _hyper_8_27_chunk (actual rows=0.00 loops=1) -> Seq Scan on _hyper_8_29_chunk (actual rows=0.00 loops=1) -> Seq Scan on _hyper_8_31_chunk (actual rows=0.00 loops=1) - -> Seq Scan on _hyper_8_33_chunk (actual rows=0.00 loops=1) - -> Seq Scan on _hyper_8_36_chunk (actual rows=0.00 loops=1) - -> Seq Scan on _hyper_8_39_chunk (actual rows=0.00 loops=1) - -> Seq Scan on _hyper_8_42_chunk (actual rows=0.00 loops=1) - -> Seq Scan on _hyper_8_44_chunk (actual rows=0.00 loops=1) - -> Seq Scan on _hyper_8_46_chunk (actual rows=0.00 loops=1) + -> Seq Scan on _hyper_8_33_chunk (actual rows=1.00 loops=1) + -> Seq Scan on _hyper_8_36_chunk (actual rows=1.00 loops=1) + -> Seq Scan on _hyper_8_39_chunk (actual rows=1.00 loops=1) + -> Seq Scan on _hyper_8_42_chunk (actual rows=1.00 loops=1) + -> Seq Scan on _hyper_8_44_chunk (actual rows=1.00 loops=1) + -> Seq Scan on _hyper_8_46_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_8_48_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_8_50_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_8_52_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_52_chunk (actual rows=1.00 loops=1) @@ -1364,8 +1364,8 @@ UNION ALL WHERE chunks_1h.bucket >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(COALESCE(:MAT_HT_ID_1H, :MAT_HT_ID_1H))), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 day'::interval, chunks_1h.bucket)), chunks_1h.device; --- QUERY PLAN --- - Append (actual rows=3.00 loops=1) - -> Custom Scan (ChunkAppend) on _materialized_hypertable_8 (actual rows=3.00 loops=1) + Append (actual rows=9.00 loops=1) + -> Custom Scan (ChunkAppend) on _materialized_hypertable_8 (actual rows=9.00 loops=1) Chunks excluded during startup: 0 -> Index Scan using _hyper_8_17_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_17_chunk (actual rows=0.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) @@ -1383,17 +1383,17 @@ UNION ALL Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) -> Index Scan using _hyper_8_31_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_31_chunk (actual rows=0.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_33_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_33_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_33_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_33_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_36_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_36_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_36_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_36_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_39_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_39_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_39_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_39_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_42_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_42_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_42_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_42_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_44_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_44_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_44_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_44_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_46_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_46_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_46_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_46_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) -> Index Scan using _hyper_8_48_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_48_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) @@ -1431,8 +1431,8 @@ UNION ALL WHERE chunks_1h.bucket >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(COALESCE(:MAT_HT_ID_1H, :MAT_HT_ID_1H))), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 day'::interval, chunks_1h.bucket)), chunks_1h.device; --- QUERY PLAN --- - Append (actual rows=3.00 loops=1) - -> Custom Scan (ChunkAppend) on _materialized_hypertable_8 (actual rows=3.00 loops=1) + Append (actual rows=9.00 loops=1) + -> Custom Scan (ChunkAppend) on _materialized_hypertable_8 (actual rows=9.00 loops=1) Chunks excluded during startup: 0 -> Index Scan using _hyper_8_17_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_17_chunk (actual rows=0.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) @@ -1450,17 +1450,17 @@ UNION ALL Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) -> Index Scan using _hyper_8_31_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_31_chunk (actual rows=0.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_33_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_33_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_33_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_33_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_36_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_36_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_36_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_36_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_39_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_39_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_39_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_39_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_42_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_42_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_42_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_42_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_44_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_44_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_44_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_44_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_46_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_46_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_46_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_46_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) -> Index Scan using _hyper_8_48_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_48_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) @@ -1498,8 +1498,8 @@ UNION ALL WHERE chunks_1h.bucket >= COALESCE(COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(:MAT_HT_ID_1H)), '-infinity'::timestamp with time zone), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 day'::interval, chunks_1h.bucket)), chunks_1h.device; --- QUERY PLAN --- - Append (actual rows=3.00 loops=1) - -> Append (actual rows=3.00 loops=1) + Append (actual rows=9.00 loops=1) + -> Append (actual rows=9.00 loops=1) -> Seq Scan on _hyper_8_17_chunk (actual rows=0.00 loops=1) -> Seq Scan on _hyper_8_20_chunk (actual rows=0.00 loops=1) -> Seq Scan on _hyper_8_21_chunk (actual rows=0.00 loops=1) @@ -1508,12 +1508,12 @@ UNION ALL -> Seq Scan on _hyper_8_27_chunk (actual rows=0.00 loops=1) -> Seq Scan on _hyper_8_29_chunk (actual rows=0.00 loops=1) -> Seq Scan on _hyper_8_31_chunk (actual rows=0.00 loops=1) - -> Seq Scan on _hyper_8_33_chunk (actual rows=0.00 loops=1) - -> Seq Scan on _hyper_8_36_chunk (actual rows=0.00 loops=1) - -> Seq Scan on _hyper_8_39_chunk (actual rows=0.00 loops=1) - -> Seq Scan on _hyper_8_42_chunk (actual rows=0.00 loops=1) - -> Seq Scan on _hyper_8_44_chunk (actual rows=0.00 loops=1) - -> Seq Scan on _hyper_8_46_chunk (actual rows=0.00 loops=1) + -> Seq Scan on _hyper_8_33_chunk (actual rows=1.00 loops=1) + -> Seq Scan on _hyper_8_36_chunk (actual rows=1.00 loops=1) + -> Seq Scan on _hyper_8_39_chunk (actual rows=1.00 loops=1) + -> Seq Scan on _hyper_8_42_chunk (actual rows=1.00 loops=1) + -> Seq Scan on _hyper_8_44_chunk (actual rows=1.00 loops=1) + -> Seq Scan on _hyper_8_46_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_8_48_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_8_50_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_8_52_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_52_chunk (actual rows=1.00 loops=1) diff --git a/tsl/test/expected/telemetry_stats.out b/tsl/test/expected/telemetry_stats.out index be01aef86c9..2be2efb3c2f 100644 --- a/tsl/test/expected/telemetry_stats.out +++ b/tsl/test/expected/telemetry_stats.out @@ -155,7 +155,7 @@ INSERT INTO hyper SELECT * FROM normal; INSERT INTO part SELECT * FROM normal; -CALL refresh_continuous_aggregate('contagg', NULL, NULL); +CALL refresh_continuous_aggregate('contagg', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); -- Reindex to avoid the dependency on the way the index is built (e.g. the caggs -- might get their rows inserted in different order during the refresh based on -- the underlying aggregation plan, and the index will be built differently, diff --git a/tsl/test/isolation/expected/cagg_cancel_kill_refresh.out b/tsl/test/isolation/expected/cagg_cancel_kill_refresh.out index d4a2c039c9b..54d8c9c8093 100644 --- a/tsl/test/isolation/expected/cagg_cancel_kill_refresh.out +++ b/tsl/test/isolation/expected/cagg_cancel_kill_refresh.out @@ -49,7 +49,7 @@ cagg_name|start_range|end_range starting permutation: wp1_enable r1_register_pid r1_refresh s1_registered_ranges k1_cancel wp1_release s1_registered_ranges step wp1_enable: - SELECT debug_waitpoint_enable('cagg_policy_batch_0_after_txn_1_wait'); + SELECT debug_waitpoint_enable('cagg_policy_batch_1_after_txn_1_wait'); debug_waitpoint_enable ---------------------- @@ -77,7 +77,7 @@ step k1_cancel: step r1_refresh: <... completed> ERROR: canceling statement due to user request step wp1_release: - SELECT debug_waitpoint_release('cagg_policy_batch_0_after_txn_1_wait'); + SELECT debug_waitpoint_release('cagg_policy_batch_1_after_txn_1_wait'); debug_waitpoint_release ----------------------- @@ -254,7 +254,7 @@ cagg_name|start_range|end_range starting permutation: wp1_enable tr2_register_pid tr2_refresh s1_registered_ranges t1_terminate wp1_release s1_registered_ranges r2_refresh s1_registered_ranges step wp1_enable: - SELECT debug_waitpoint_enable('cagg_policy_batch_0_after_txn_1_wait'); + SELECT debug_waitpoint_enable('cagg_policy_batch_1_after_txn_1_wait'); debug_waitpoint_enable ---------------------- @@ -286,7 +286,7 @@ server closed the connection unexpectedly before or while processing the request. step wp1_release: - SELECT debug_waitpoint_release('cagg_policy_batch_0_after_txn_1_wait'); + SELECT debug_waitpoint_release('cagg_policy_batch_1_after_txn_1_wait'); debug_waitpoint_release ----------------------- diff --git a/tsl/test/isolation/expected/cagg_concurrent_register.out b/tsl/test/isolation/expected/cagg_concurrent_register.out index 534aec5e3a1..c36e526728b 100644 --- a/tsl/test/isolation/expected/cagg_concurrent_register.out +++ b/tsl/test/isolation/expected/cagg_concurrent_register.out @@ -39,8 +39,8 @@ step s5_show_running_jobs: cagg_name| start_range| end_range|start_ts_utc |end_ts_utc ---------+----------------+----------------+------------------------+------------------------ -cagg_1 |1577836800000000|1578009600000000|Wed Jan 01 00:00:00 2020|Fri Jan 03 00:00:00 2020 -cagg_2 |1578009600000000|1578182400000000|Fri Jan 03 00:00:00 2020|Sun Jan 05 00:00:00 2020 +cagg_1 |1577980800000000|1578009600000000|Thu Jan 02 16:00:00 2020|Fri Jan 03 00:00:00 2020 +cagg_2 |1578153600000000|1578182400000000|Sat Jan 04 16:00:00 2020|Sun Jan 05 00:00:00 2020 step s4_release_before_process_cagg_invalidations: SELECT debug_waitpoint_release('before_process_cagg_invalidations_for_refresh_lock'); @@ -81,8 +81,6 @@ step s3_release_after_register: -- release lock on jobs_refresh_ranges table ROLLBACK; -step s2_run_cagg2_overlap_refresh: <... completed> -ERROR: could not refresh continuous aggregate "cagg_2" due to a concurrent refresh step s5_show_running_jobs: SELECT ca.user_view_name AS cagg_name, r.start_range, r.end_range, to_timestamp(r.start_range / 1000000) AT TIME ZONE 'UTC' AS start_ts_utc, @@ -93,7 +91,8 @@ step s5_show_running_jobs: cagg_name| start_range| end_range|start_ts_utc |end_ts_utc ---------+----------------+----------------+------------------------+------------------------ -cagg_2 |1577836800000000|1578355200000000|Wed Jan 01 00:00:00 2020|Tue Jan 07 00:00:00 2020 +cagg_2 |1578268800000000|1578355200000000|Mon Jan 06 00:00:00 2020|Tue Jan 07 00:00:00 2020 +cagg_2 |1578153600000000|1578182400000000|Sat Jan 04 16:00:00 2020|Sun Jan 05 00:00:00 2020 step s4_release_before_process_cagg_invalidations: SELECT debug_waitpoint_release('before_process_cagg_invalidations_for_refresh_lock'); @@ -103,6 +102,8 @@ debug_waitpoint_release step s1_run_cagg2_overlap_refresh: <... completed> +ERROR: could not refresh continuous aggregate "cagg_2" due to a concurrent refresh +step s2_run_cagg2_overlap_refresh: <... completed> starting permutation: s2_insert_new_data_2020 s3_lock_before_register s1_run_cagg2_nonoverlap_refresh s2_run_cagg2_overlap_refresh s4_enable_before_process_cagg_invalidations s3_release_after_register s5_show_running_jobs s4_release_before_process_cagg_invalidations step s2_insert_new_data_2020: @@ -144,7 +145,7 @@ step s5_show_running_jobs: cagg_name| start_range| end_range|start_ts_utc |end_ts_utc ---------+----------------+----------------+------------------------+------------------------ cagg_2 |1577836800000000|1577923200000000|Wed Jan 01 00:00:00 2020|Thu Jan 02 00:00:00 2020 -cagg_2 |1578009600000000|1578182400000000|Fri Jan 03 00:00:00 2020|Sun Jan 05 00:00:00 2020 +cagg_2 |1578153600000000|1578182400000000|Sat Jan 04 16:00:00 2020|Sun Jan 05 00:00:00 2020 step s4_release_before_process_cagg_invalidations: SELECT debug_waitpoint_release('before_process_cagg_invalidations_for_refresh_lock'); diff --git a/tsl/test/isolation/expected/cagg_hierarchical_concurrent_refresh.out b/tsl/test/isolation/expected/cagg_hierarchical_concurrent_refresh.out index 2e0ef6d71fd..df0af1d9921 100644 --- a/tsl/test/isolation/expected/cagg_hierarchical_concurrent_refresh.out +++ b/tsl/test/isolation/expected/cagg_hierarchical_concurrent_refresh.out @@ -509,7 +509,8 @@ step chk_hyper_invals: source |lowest |greatest -------+----------------------------+---------------------------- -cagg_6h|Thu Jan 01 00:00:00 2026 UTC|Sat Jan 03 18:00:00 2026 UTC +cagg_6h|Thu Jan 01 00:00:00 2026 UTC|Sat Jan 03 06:00:00 2026 UTC +cagg_6h|Sat Jan 03 12:00:00 2026 UTC|Sat Jan 03 18:00:00 2026 UTC step lock_L2_source: BEGIN; @@ -631,7 +632,8 @@ step chk_hyper_invals: source |lowest |greatest -------+----------------------------+---------------------------- -cagg_6h|Thu Jan 01 00:00:00 2026 UTC|Sat Jan 03 18:00:00 2026 UTC +cagg_6h|Thu Jan 01 00:00:00 2026 UTC|Sat Jan 03 06:00:00 2026 UTC +cagg_6h|Sat Jan 03 12:00:00 2026 UTC|Sat Jan 03 18:00:00 2026 UTC step lock_L2_source: BEGIN; @@ -908,7 +910,8 @@ step chk_hyper_invals: source |lowest |greatest -------+----------------------------+---------------------------- -cagg_6h|Thu Jan 01 00:00:00 2026 UTC|Sat Jan 03 18:00:00 2026 UTC +cagg_6h|Thu Jan 01 00:00:00 2026 UTC|Sat Jan 03 06:00:00 2026 UTC +cagg_6h|Sat Jan 03 12:00:00 2026 UTC|Sat Jan 03 18:00:00 2026 UTC step lock_L2_source: BEGIN; diff --git a/tsl/test/isolation/expected/cagg_incremental_concurrent.out b/tsl/test/isolation/expected/cagg_incremental_concurrent.out index eb9f25bae95..e4acd8bb438 100644 --- a/tsl/test/isolation/expected/cagg_incremental_concurrent.out +++ b/tsl/test/isolation/expected/cagg_incremental_concurrent.out @@ -1,4 +1,4 @@ -Parsed test spec with 7 sessions +Parsed test spec with 8 sessions starting permutation: wp_enable r1_run s1_refresh_ranges s1_count r2_refresh wp_release s1_count s1_check_duplicates step wp_enable: @@ -113,6 +113,11 @@ R3: LOG: statement: SET SESSION client_min_messages = 'LOG'; SET timescaledb.current_timestamp_mock TO '2026-04-01 00:30:00+00'; +R4: LOG: statement: + SET SESSION lock_timeout = '500ms'; + SET SESSION deadlock_timeout = '500ms'; + SET SESSION client_min_messages = 'LOG'; + step wp_enable: SELECT debug_waitpoint_enable('cagg_policy_batch_2_after_txn_1_wait'); @@ -241,6 +246,11 @@ R3: LOG: statement: SET SESSION client_min_messages = 'LOG'; SET timescaledb.current_timestamp_mock TO '2026-04-01 00:30:00+00'; +R4: LOG: statement: + SET SESSION lock_timeout = '500ms'; + SET SESSION deadlock_timeout = '500ms'; + SET SESSION client_min_messages = 'LOG'; + step wp_enable: SELECT debug_waitpoint_enable('cagg_policy_batch_2_after_txn_1_wait'); @@ -357,6 +367,11 @@ R3: LOG: statement: SET SESSION client_min_messages = 'LOG'; SET timescaledb.current_timestamp_mock TO '2026-04-01 00:30:00+00'; +R4: LOG: statement: + SET SESSION lock_timeout = '500ms'; + SET SESSION deadlock_timeout = '500ms'; + SET SESSION client_min_messages = 'LOG'; + step wp_enable: SELECT debug_waitpoint_enable('cagg_policy_batch_2_after_txn_1_wait'); @@ -448,6 +463,11 @@ R3: LOG: statement: SET SESSION client_min_messages = 'LOG'; SET timescaledb.current_timestamp_mock TO '2026-04-01 00:30:00+00'; +R4: LOG: statement: + SET SESSION lock_timeout = '500ms'; + SET SESSION deadlock_timeout = '500ms'; + SET SESSION client_min_messages = 'LOG'; + step wp_enable: SELECT debug_waitpoint_enable('cagg_policy_batch_2_after_txn_1_wait'); @@ -521,3 +541,203 @@ R3: LOG: continuous aggregate refresh (individual invalidation) on "sensor_hour R3: LOG: deleted 2 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_X" R3: LOG: inserted 2 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_X" step r3_run_oldest_first: <... completed> + +starting permutation: wp_enable r4_run s1_refresh_ranges s1_count r2_refresh wp_release s1_count s1_check_duplicates +R1: LOG: statement: + SET SESSION lock_timeout = '500ms'; + SET SESSION deadlock_timeout = '500ms'; + SET SESSION client_min_messages = 'LOG'; + +R2: LOG: statement: + SET SESSION lock_timeout = '500ms'; + SET SESSION deadlock_timeout = '500ms'; + SET SESSION client_min_messages = 'LOG'; + +R3: LOG: statement: + SET SESSION lock_timeout = '500ms'; + SET SESSION deadlock_timeout = '500ms'; + SET SESSION client_min_messages = 'LOG'; + SET timescaledb.current_timestamp_mock TO '2026-04-01 00:30:00+00'; + +R4: LOG: statement: + SET SESSION lock_timeout = '500ms'; + SET SESSION deadlock_timeout = '500ms'; + SET SESSION client_min_messages = 'LOG'; + +step wp_enable: + SELECT debug_waitpoint_enable('cagg_policy_batch_2_after_txn_1_wait'); + +debug_waitpoint_enable +---------------------- + + +R4: LOG: statement: + CALL refresh_continuous_aggregate( + 'cond_10', NULL, NULL, + options => jsonb_build_object('buckets_per_batch', 1)); + +R4: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_X" +R4: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_X" +step r4_run: + CALL refresh_continuous_aggregate( + 'cond_10', NULL, NULL, + options => jsonb_build_object('buckets_per_batch', 1)); + +step s1_refresh_ranges: + SELECT ca.user_view_name AS cagg_name, r.start_range, r.end_range + FROM _timescaledb_catalog.continuous_aggs_jobs_refresh_ranges r + JOIN _timescaledb_catalog.continuous_agg ca ON r.materialization_id = ca.mat_hypertable_id + ORDER BY ca.user_view_name; + +cagg_name|start_range|end_range +---------+-----------+--------- +cond_10 | 50| 60 + +step s1_count: + SELECT count(*) AS row_count FROM cond_10; + +row_count +--------- + 18 + +R2: LOG: statement: + -- Non-overlapping with R1's registered batch range [50, 60) + CALL refresh_continuous_aggregate('cond_10', 1, 50); + +R2: LOG: deleted 12 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_X" +R2: LOG: inserted 12 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_X" +step r2_refresh: + -- Non-overlapping with R1's registered batch range [50, 60) + CALL refresh_continuous_aggregate('cond_10', 1, 50); + +step wp_release: + SELECT debug_waitpoint_release('cagg_policy_batch_2_after_txn_1_wait'); + +debug_waitpoint_release +----------------------- + + +R4: LOG: deleted 3 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_X" +R4: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_X" +R4: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_X" +R4: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_X" +step r4_run: <... completed> +step s1_count: + SELECT count(*) AS row_count FROM cond_10; + +row_count +--------- + 21 + +step s1_check_duplicates: + -- Check for duplicate (bucket, device_id) rows in the materialization hypertable. + -- If duplicates exist, the same range was materialized twice without proper cleanup. + SELECT bucket, device_id, count(*) AS copies + FROM cond_10 + GROUP BY bucket, device_id + HAVING count(*) > 1 + ORDER BY bucket, device_id; + +bucket|device_id|copies +------+---------+------ + + +starting permutation: wp_enable r4_run s1_refresh_ranges r2_refresh_overlap wp_release s1_count s1_check_duplicates +R1: LOG: statement: + SET SESSION lock_timeout = '500ms'; + SET SESSION deadlock_timeout = '500ms'; + SET SESSION client_min_messages = 'LOG'; + +R2: LOG: statement: + SET SESSION lock_timeout = '500ms'; + SET SESSION deadlock_timeout = '500ms'; + SET SESSION client_min_messages = 'LOG'; + +R3: LOG: statement: + SET SESSION lock_timeout = '500ms'; + SET SESSION deadlock_timeout = '500ms'; + SET SESSION client_min_messages = 'LOG'; + SET timescaledb.current_timestamp_mock TO '2026-04-01 00:30:00+00'; + +R4: LOG: statement: + SET SESSION lock_timeout = '500ms'; + SET SESSION deadlock_timeout = '500ms'; + SET SESSION client_min_messages = 'LOG'; + +step wp_enable: + SELECT debug_waitpoint_enable('cagg_policy_batch_2_after_txn_1_wait'); + +debug_waitpoint_enable +---------------------- + + +R4: LOG: statement: + CALL refresh_continuous_aggregate( + 'cond_10', NULL, NULL, + options => jsonb_build_object('buckets_per_batch', 1)); + +R4: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_X" +R4: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_X" +step r4_run: + CALL refresh_continuous_aggregate( + 'cond_10', NULL, NULL, + options => jsonb_build_object('buckets_per_batch', 1)); + +step s1_refresh_ranges: + SELECT ca.user_view_name AS cagg_name, r.start_range, r.end_range + FROM _timescaledb_catalog.continuous_aggs_jobs_refresh_ranges r + JOIN _timescaledb_catalog.continuous_agg ca ON r.materialization_id = ca.mat_hypertable_id + ORDER BY ca.user_view_name; + +cagg_name|start_range|end_range +---------+-----------+--------- +cond_10 | 50| 60 + +R2: LOG: statement: + -- Overlapping with R1's registered batch range [50, 60) + CALL refresh_continuous_aggregate('cond_10', 40, 60); + +step r2_refresh_overlap: + -- Overlapping with R1's registered batch range [50, 60) + CALL refresh_continuous_aggregate('cond_10', 40, 60); + +ERROR: could not refresh continuous aggregate "cond_10" due to a concurrent refresh +step wp_release: + SELECT debug_waitpoint_release('cagg_policy_batch_2_after_txn_1_wait'); + +debug_waitpoint_release +----------------------- + + +R4: LOG: deleted 3 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_X" +R4: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_X" +R4: LOG: deleted 3 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_X" +R4: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_X" +R4: LOG: deleted 3 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_X" +R4: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_X" +R4: LOG: deleted 3 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_X" +R4: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_X" +R4: LOG: deleted 3 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_X" +R4: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_X" +R4: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_X" +R4: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_X" +step r4_run: <... completed> +step s1_count: + SELECT count(*) AS row_count FROM cond_10; + +row_count +--------- + 21 + +step s1_check_duplicates: + -- Check for duplicate (bucket, device_id) rows in the materialization hypertable. + -- If duplicates exist, the same range was materialized twice without proper cleanup. + SELECT bucket, device_id, count(*) AS copies + FROM cond_10 + GROUP BY bucket, device_id + HAVING count(*) > 1 + ORDER BY bucket, device_id; + +bucket|device_id|copies +------+---------+------ + diff --git a/tsl/test/isolation/expected/cagg_refresh_cleanup_register.out b/tsl/test/isolation/expected/cagg_refresh_cleanup_register.out index 19102a98690..4c39ffdf81f 100644 --- a/tsl/test/isolation/expected/cagg_refresh_cleanup_register.out +++ b/tsl/test/isolation/expected/cagg_refresh_cleanup_register.out @@ -1,6 +1,6 @@ Parsed test spec with 9 sessions -starting permutation: WP_mat_enable R2_refresh L1_lock WP_mat_disable R3_refresh R4_refresh check_locks check_jobs L1_unlock check_locks check_jobs +starting permutation: WP_mat_enable R2_refresh L1_lock WP_mat_disable WP_enable_after_refresh R3_refresh R4_refresh check_locks check_jobs L1_unlock WP_disable_after_refresh check_locks check_jobs step WP_mat_enable: SELECT debug_waitpoint_enable('after_process_cagg_materializations'); debug_waitpoint_enable ---------------------- @@ -19,6 +19,11 @@ debug_waitpoint_release ----------------------- +step WP_enable_after_refresh: SELECT debug_waitpoint_enable('after_cagg_refresh_window'); +debug_waitpoint_enable +---------------------- + + step R3_refresh: CALL refresh_continuous_aggregate('cond_daily', '2026-02-15', '2026-03-15'); @@ -50,11 +55,16 @@ step check_jobs: user_view_name|start_time |end_time --------------+----------------------------+---------------------------- -cond_daily |Mon Jan 05 16:00:00 2026 PST|Sat Feb 14 16:00:00 2026 PST +cond_daily |Sun Feb 08 16:00:00 2026 PST|Sat Feb 14 16:00:00 2026 PST step L1_unlock: COMMIT; +step WP_disable_after_refresh: SELECT debug_waitpoint_release('after_cagg_refresh_window'); +debug_waitpoint_release +----------------------- + + step R2_refresh: <... completed> step R3_refresh: <... completed> step R4_refresh: <... completed> @@ -81,7 +91,7 @@ user_view_name|start_time|end_time --------------+----------+-------- -starting permutation: WP_mat_enable R2_refresh L1_lock WP_mat_disable R3_refresh R4_overlapping_refresh check_locks check_jobs L1_unlock check_locks check_jobs +starting permutation: WP_mat_enable R2_refresh L1_lock WP_mat_disable WP_after_register_enable R3_refresh R4_overlapping_refresh check_locks check_jobs L1_unlock WP_after_register_disable check_locks check_jobs step WP_mat_enable: SELECT debug_waitpoint_enable('after_process_cagg_materializations'); debug_waitpoint_enable ---------------------- @@ -100,11 +110,16 @@ debug_waitpoint_release ----------------------- +step WP_after_register_enable: SELECT debug_waitpoint_enable('cagg_refresh_after_register'); +debug_waitpoint_enable +---------------------- + + step R3_refresh: CALL refresh_continuous_aggregate('cond_daily', '2026-02-15', '2026-03-15'); step R4_overlapping_refresh: - CALL refresh_continuous_aggregate('cond_daily', '2026-02-15', '2026-03-05'); + CALL refresh_continuous_aggregate('cond_daily', '2026-03-01', '2026-03-15'); step check_locks: SELECT l.mode, l.granted @@ -131,11 +146,16 @@ step check_jobs: user_view_name|start_time |end_time --------------+----------------------------+---------------------------- -cond_daily |Mon Jan 05 16:00:00 2026 PST|Sat Feb 14 16:00:00 2026 PST +cond_daily |Sun Feb 08 16:00:00 2026 PST|Sat Feb 14 16:00:00 2026 PST step L1_unlock: COMMIT; +step WP_after_register_disable: SELECT debug_waitpoint_release('cagg_refresh_after_register'); +debug_waitpoint_release +----------------------- + + step R2_refresh: <... completed> step R3_refresh: <... completed> step R4_overlapping_refresh: <... completed> @@ -183,7 +203,7 @@ step check_jobs: user_view_name|start_time |end_time --------------+----------------------------+---------------------------- -cond_daily |Sun Feb 15 16:00:00 2026 PST|Sat Mar 14 17:00:00 2026 PDT +cond_daily |Sat Mar 07 16:00:00 2026 PST|Sat Mar 14 17:00:00 2026 PDT step A1_revoke_perm: REVOKE SELECT on conditions FROM cagg_user; @@ -228,7 +248,7 @@ step check_jobs: user_view_name|start_time |end_time --------------+----------------------------+---------------------------- -cond_daily |Sun Feb 15 16:00:00 2026 PST|Sat Mar 14 17:00:00 2026 PDT +cond_daily |Sat Mar 07 16:00:00 2026 PST|Sat Mar 14 17:00:00 2026 PDT step A1_revoke_mat_perm: DO $$ @@ -363,7 +383,7 @@ step check_jobs: user_view_name|start_time |end_time --------------+----------------------------+---------------------------- -cond_daily |Mon Jan 05 16:00:00 2026 PST|Sat Mar 14 17:00:00 2026 PDT +cond_daily |Tue Mar 10 17:00:00 2026 PDT|Sat Mar 14 17:00:00 2026 PDT step K1_terminate: DO $$ diff --git a/tsl/test/isolation/specs/cagg_cancel_kill_refresh.spec b/tsl/test/isolation/specs/cagg_cancel_kill_refresh.spec index 601519fcadb..8da8d00d03b 100644 --- a/tsl/test/isolation/specs/cagg_cancel_kill_refresh.spec +++ b/tsl/test/isolation/specs/cagg_cancel_kill_refresh.spec @@ -101,11 +101,11 @@ step "wp0_release" session "WP1" step "wp1_enable" { - SELECT debug_waitpoint_enable('cagg_policy_batch_0_after_txn_1_wait'); + SELECT debug_waitpoint_enable('cagg_policy_batch_1_after_txn_1_wait'); } step "wp1_release" { - SELECT debug_waitpoint_release('cagg_policy_batch_0_after_txn_1_wait'); + SELECT debug_waitpoint_release('cagg_policy_batch_1_after_txn_1_wait'); } # Waitpoint after txn 2 (cagg invalidations processed) diff --git a/tsl/test/isolation/specs/cagg_incremental_concurrent.spec b/tsl/test/isolation/specs/cagg_incremental_concurrent.spec index 98f3e451577..e48b7c7a89a 100644 --- a/tsl/test/isolation/specs/cagg_incremental_concurrent.spec +++ b/tsl/test/isolation/specs/cagg_incremental_concurrent.spec @@ -295,6 +295,23 @@ step "r3_run_oldest_first" $$; } +# Session for batched manual refresh via JSONB options. Goes through the same +# continuous_agg_refresh_single_window path as the policy, so it pauses at the +# same waitpoint. +session "R4" +setup +{ + SET SESSION lock_timeout = '500ms'; + SET SESSION deadlock_timeout = '500ms'; + SET SESSION client_min_messages = 'LOG'; +} +step "r4_run" +{ + CALL refresh_continuous_aggregate( + 'cond_10', NULL, NULL, + options => jsonb_build_object('buckets_per_batch', 1)); +} + #insert data to create invalidations when refresh is stopped after batch 1 session "I2" step "i2_insert" @@ -325,4 +342,14 @@ permutation "wp_enable" "r1_run"("wp_enable") "s1_refresh_ranges" "r2_refresh_ov permutation "wp_enable" "r3_run"("wp_enable") "i2_insert" "wp_release" # Test 5: Same as test 5, but the policy refreshes oldest batch first -permutation "wp_enable" "r3_run_oldest_first"("wp_enable") "i2_insert" "wp_release" \ No newline at end of file +permutation "wp_enable" "r3_run_oldest_first"("wp_enable") "i2_insert" "wp_release" + +# Test 6: Manual incremental refresh pauses after batch 1. +# A non-overlapping atomic manual refresh proceeds successfully +# Mirrors test 1 but with batched manual refresh on the paused side instead of policy. +permutation "wp_enable" "r4_run"("wp_enable") "s1_refresh_ranges" "s1_count" "r2_refresh" "wp_release" "s1_count" "s1_check_duplicates" + +# Test 7: Manual incremental refresh pauses after batch 1. +# An overlapping atomic manual refresh must error with "concurrent refresh". +# Mirrors test 3 with batched manual refresh on the paused side. +permutation "wp_enable" "r4_run"("wp_enable") "s1_refresh_ranges" "r2_refresh_overlap" "wp_release" "s1_count" "s1_check_duplicates" \ No newline at end of file diff --git a/tsl/test/isolation/specs/cagg_refresh_cleanup_register.spec b/tsl/test/isolation/specs/cagg_refresh_cleanup_register.spec index 92f235dbaf7..c49c1c6087e 100644 --- a/tsl/test/isolation/specs/cagg_refresh_cleanup_register.spec +++ b/tsl/test/isolation/specs/cagg_refresh_cleanup_register.spec @@ -74,6 +74,8 @@ step "WP_before_txn2_start_enable" { SELECT debug_waitpoint_enable('cagg_refres step "WP_before_txn2_start_disable" { SELECT debug_waitpoint_release('cagg_refresh_after_register'); } step "WP_after_register_enable" { SELECT debug_waitpoint_enable('cagg_refresh_after_register'); } step "WP_after_register_disable" { SELECT debug_waitpoint_release('cagg_refresh_after_register'); } +step "WP_enable_after_refresh" { SELECT debug_waitpoint_enable('after_cagg_refresh_window'); } +step "WP_disable_after_refresh" { SELECT debug_waitpoint_release('after_cagg_refresh_window'); } # Session K1: terminate R1's backend so its PID becomes dead in the # registration table, then wait until the process is gone. @@ -123,7 +125,7 @@ step "R4_refresh" { CALL refresh_continuous_aggregate('cond_daily', '2026-03-15', '2026-03-30'); } step "R4_overlapping_refresh" { - CALL refresh_continuous_aggregate('cond_daily', '2026-02-15', '2026-03-05'); + CALL refresh_continuous_aggregate('cond_daily', '2026-03-01', '2026-03-15'); } session "A1" @@ -220,11 +222,11 @@ step "P1_run_policy" { # Two refreshes wait for registration, one waits for cleanup before exiting. All blocked on an AccessExclusiveLock on continuous_aggs_jobs_refresh_ranges. # None of those refreshes overlaps, so all should succeed. -permutation "WP_mat_enable" "R2_refresh" "L1_lock" "WP_mat_disable" "R3_refresh" "R4_refresh" "check_locks" "check_jobs" "L1_unlock" "check_locks" "check_jobs" +permutation "WP_mat_enable" "R2_refresh" "L1_lock" "WP_mat_disable" "WP_enable_after_refresh" "R3_refresh"("R2_refresh") "R4_refresh"("R3_refresh") "check_locks" "check_jobs" "L1_unlock" "WP_disable_after_refresh" "check_locks" "check_jobs" # Two refreshes wait for registration, one waits for cleanup before exiting. All blocked on an AccessExclusiveLock on continuous_aggs_jobs_refresh_ranges. # Refreshes waiting for registration overlap with each other, so one should fail. -permutation "WP_mat_enable" "R2_refresh" "L1_lock" "WP_mat_disable" "R3_refresh" "R4_overlapping_refresh" "check_locks" "check_jobs" "L1_unlock" "check_locks" "check_jobs" +permutation "WP_mat_enable" "R2_refresh" "L1_lock" "WP_mat_disable" "WP_after_register_enable" "R3_refresh"("R2_refresh") "R4_overlapping_refresh"("R3_refresh") "check_locks" "check_jobs" "L1_unlock" "WP_after_register_disable" "check_locks" "check_jobs" ## Refresh registers . But fails in txn2. Gets into catch block. Cleanup should succeed permutation "WP_before_txn2_start_enable" "R3_refresh" "check_jobs" "A1_revoke_perm" "WP_before_txn2_start_disable"("A1_revoke_perm") "check_jobs" diff --git a/tsl/test/shared/expected/constify_now-15.out b/tsl/test/shared/expected/constify_now-15.out index 7e4f202db01..0725ea404aa 100644 --- a/tsl/test/shared/expected/constify_now-15.out +++ b/tsl/test/shared/expected/constify_now-15.out @@ -588,11 +588,11 @@ CALL refresh_continuous_aggregate('cagg_now_view', NULL, '2004-01-01'); --- QUERY PLAN --- Append -> Append + -> Index Scan using _hyper_X_X_chunk__materialized_hypertable_X_bucket_idx on _hyper_X_X_chunk + Index Cond: (bucket < 'Wed Jan 01 01:00:00 2003 UTC'::timestamp with time zone) -> Seq Scan on _hyper_X_X_chunk -> Seq Scan on _hyper_X_X_chunk -> Seq Scan on _hyper_X_X_chunk - -> Index Scan using _hyper_X_X_chunk__materialized_hypertable_X_bucket_idx on _hyper_X_X_chunk - Index Cond: (bucket < 'Wed Jan 01 01:00:00 2003 UTC'::timestamp with time zone) -> Finalize HashAggregate Group Key: (time_bucket('@ 1 day'::interval, cagg_now_test."time")), cagg_now_test.device -> Append @@ -713,10 +713,10 @@ CALL refresh_continuous_aggregate('cagg_hierarch_l2', NULL, '2004-01-01'); --- QUERY PLAN --- Append -> Append - -> Index Scan using _hyper_X_X_chunk__materialized_hypertable_X_bucket_idx on _hyper_X_X_chunk - Index Cond: ((bucket < 'Wed Jan 01 01:00:00 2003 UTC'::timestamp with time zone) AND (bucket >= (ts_now_mock() - '@ 1 year'::interval))) -> Index Scan using _hyper_X_X_chunk__materialized_hypertable_X_bucket_idx on _hyper_X_X_chunk Index Cond: (bucket >= (ts_now_mock() - '@ 1 year'::interval)) + -> Index Scan using _hyper_X_X_chunk__materialized_hypertable_X_bucket_idx on _hyper_X_X_chunk + Index Cond: ((bucket < 'Wed Jan 01 01:00:00 2003 UTC'::timestamp with time zone) AND (bucket >= (ts_now_mock() - '@ 1 year'::interval))) -> HashAggregate Group Key: time_bucket('@ 1 mon'::interval, (time_bucket('@ 1 day'::interval, cagg_hierarch_test."time"))), cagg_hierarch_test.device -> Result diff --git a/tsl/test/shared/expected/constify_now-16.out b/tsl/test/shared/expected/constify_now-16.out index 7e4f202db01..0725ea404aa 100644 --- a/tsl/test/shared/expected/constify_now-16.out +++ b/tsl/test/shared/expected/constify_now-16.out @@ -588,11 +588,11 @@ CALL refresh_continuous_aggregate('cagg_now_view', NULL, '2004-01-01'); --- QUERY PLAN --- Append -> Append + -> Index Scan using _hyper_X_X_chunk__materialized_hypertable_X_bucket_idx on _hyper_X_X_chunk + Index Cond: (bucket < 'Wed Jan 01 01:00:00 2003 UTC'::timestamp with time zone) -> Seq Scan on _hyper_X_X_chunk -> Seq Scan on _hyper_X_X_chunk -> Seq Scan on _hyper_X_X_chunk - -> Index Scan using _hyper_X_X_chunk__materialized_hypertable_X_bucket_idx on _hyper_X_X_chunk - Index Cond: (bucket < 'Wed Jan 01 01:00:00 2003 UTC'::timestamp with time zone) -> Finalize HashAggregate Group Key: (time_bucket('@ 1 day'::interval, cagg_now_test."time")), cagg_now_test.device -> Append @@ -713,10 +713,10 @@ CALL refresh_continuous_aggregate('cagg_hierarch_l2', NULL, '2004-01-01'); --- QUERY PLAN --- Append -> Append - -> Index Scan using _hyper_X_X_chunk__materialized_hypertable_X_bucket_idx on _hyper_X_X_chunk - Index Cond: ((bucket < 'Wed Jan 01 01:00:00 2003 UTC'::timestamp with time zone) AND (bucket >= (ts_now_mock() - '@ 1 year'::interval))) -> Index Scan using _hyper_X_X_chunk__materialized_hypertable_X_bucket_idx on _hyper_X_X_chunk Index Cond: (bucket >= (ts_now_mock() - '@ 1 year'::interval)) + -> Index Scan using _hyper_X_X_chunk__materialized_hypertable_X_bucket_idx on _hyper_X_X_chunk + Index Cond: ((bucket < 'Wed Jan 01 01:00:00 2003 UTC'::timestamp with time zone) AND (bucket >= (ts_now_mock() - '@ 1 year'::interval))) -> HashAggregate Group Key: time_bucket('@ 1 mon'::interval, (time_bucket('@ 1 day'::interval, cagg_hierarch_test."time"))), cagg_hierarch_test.device -> Result diff --git a/tsl/test/shared/expected/constify_now-17.out b/tsl/test/shared/expected/constify_now-17.out index 7e4f202db01..0725ea404aa 100644 --- a/tsl/test/shared/expected/constify_now-17.out +++ b/tsl/test/shared/expected/constify_now-17.out @@ -588,11 +588,11 @@ CALL refresh_continuous_aggregate('cagg_now_view', NULL, '2004-01-01'); --- QUERY PLAN --- Append -> Append + -> Index Scan using _hyper_X_X_chunk__materialized_hypertable_X_bucket_idx on _hyper_X_X_chunk + Index Cond: (bucket < 'Wed Jan 01 01:00:00 2003 UTC'::timestamp with time zone) -> Seq Scan on _hyper_X_X_chunk -> Seq Scan on _hyper_X_X_chunk -> Seq Scan on _hyper_X_X_chunk - -> Index Scan using _hyper_X_X_chunk__materialized_hypertable_X_bucket_idx on _hyper_X_X_chunk - Index Cond: (bucket < 'Wed Jan 01 01:00:00 2003 UTC'::timestamp with time zone) -> Finalize HashAggregate Group Key: (time_bucket('@ 1 day'::interval, cagg_now_test."time")), cagg_now_test.device -> Append @@ -713,10 +713,10 @@ CALL refresh_continuous_aggregate('cagg_hierarch_l2', NULL, '2004-01-01'); --- QUERY PLAN --- Append -> Append - -> Index Scan using _hyper_X_X_chunk__materialized_hypertable_X_bucket_idx on _hyper_X_X_chunk - Index Cond: ((bucket < 'Wed Jan 01 01:00:00 2003 UTC'::timestamp with time zone) AND (bucket >= (ts_now_mock() - '@ 1 year'::interval))) -> Index Scan using _hyper_X_X_chunk__materialized_hypertable_X_bucket_idx on _hyper_X_X_chunk Index Cond: (bucket >= (ts_now_mock() - '@ 1 year'::interval)) + -> Index Scan using _hyper_X_X_chunk__materialized_hypertable_X_bucket_idx on _hyper_X_X_chunk + Index Cond: ((bucket < 'Wed Jan 01 01:00:00 2003 UTC'::timestamp with time zone) AND (bucket >= (ts_now_mock() - '@ 1 year'::interval))) -> HashAggregate Group Key: time_bucket('@ 1 mon'::interval, (time_bucket('@ 1 day'::interval, cagg_hierarch_test."time"))), cagg_hierarch_test.device -> Result diff --git a/tsl/test/shared/expected/constify_now-18.out b/tsl/test/shared/expected/constify_now-18.out index 6f193e5c590..25fb86957c0 100644 --- a/tsl/test/shared/expected/constify_now-18.out +++ b/tsl/test/shared/expected/constify_now-18.out @@ -578,11 +578,11 @@ CALL refresh_continuous_aggregate('cagg_now_view', NULL, '2004-01-01'); --- QUERY PLAN --- Append -> Append + -> Index Scan using _hyper_X_X_chunk__materialized_hypertable_X_bucket_idx on _hyper_X_X_chunk + Index Cond: (bucket < 'Wed Jan 01 01:00:00 2003 UTC'::timestamp with time zone) -> Seq Scan on _hyper_X_X_chunk -> Seq Scan on _hyper_X_X_chunk -> Seq Scan on _hyper_X_X_chunk - -> Index Scan using _hyper_X_X_chunk__materialized_hypertable_X_bucket_idx on _hyper_X_X_chunk - Index Cond: (bucket < 'Wed Jan 01 01:00:00 2003 UTC'::timestamp with time zone) -> Finalize HashAggregate Group Key: (time_bucket('@ 1 day'::interval, cagg_now_test."time")), cagg_now_test.device -> Append @@ -703,10 +703,10 @@ CALL refresh_continuous_aggregate('cagg_hierarch_l2', NULL, '2004-01-01'); --- QUERY PLAN --- Append -> Append - -> Index Scan using _hyper_X_X_chunk__materialized_hypertable_X_bucket_idx on _hyper_X_X_chunk - Index Cond: ((bucket < 'Wed Jan 01 01:00:00 2003 UTC'::timestamp with time zone) AND (bucket >= (ts_now_mock() - '@ 1 year'::interval))) -> Index Scan using _hyper_X_X_chunk__materialized_hypertable_X_bucket_idx on _hyper_X_X_chunk Index Cond: (bucket >= (ts_now_mock() - '@ 1 year'::interval)) + -> Index Scan using _hyper_X_X_chunk__materialized_hypertable_X_bucket_idx on _hyper_X_X_chunk + Index Cond: ((bucket < 'Wed Jan 01 01:00:00 2003 UTC'::timestamp with time zone) AND (bucket >= (ts_now_mock() - '@ 1 year'::interval))) -> HashAggregate Group Key: time_bucket('@ 1 mon'::interval, (time_bucket('@ 1 day'::interval, cagg_hierarch_test."time"))), cagg_hierarch_test.device -> Result diff --git a/tsl/test/sql/CMakeLists.txt b/tsl/test/sql/CMakeLists.txt index d1f31fc94dd..a5893f9dad9 100644 --- a/tsl/test/sql/CMakeLists.txt +++ b/tsl/test/sql/CMakeLists.txt @@ -136,6 +136,7 @@ if(CMAKE_BUILD_TYPE MATCHES Debug) cagg_policy_concurrent.sql cagg_policy_incremental.sql cagg_refresh_cleanup.sql + cagg_refresh_incremental.sql chunk_column_stats.sql compress_bgw_reorder_drop_chunks.sql compress_bloom_legacy_v1.sql diff --git a/tsl/test/sql/cagg_direct_compress.sql b/tsl/test/sql/cagg_direct_compress.sql index acc2d122993..a6d918bd366 100644 --- a/tsl/test/sql/cagg_direct_compress.sql +++ b/tsl/test/sql/cagg_direct_compress.sql @@ -30,7 +30,8 @@ FROM conditions GROUP BY 1, 2 WITH NO DATA; -CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL); +-- Setting buckets_per_batch to a high value to bypass "disabling direct compress because of too small batch size" situation +CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_hourly') chunk; -- Enable columnstore @@ -39,14 +40,14 @@ ALTER MATERIALIZED VIEW conditions_hourly SET (timescaledb.compress); -- Enable direct compress on cagg refresh SET timescaledb.enable_direct_compress_on_cagg_refresh TO on; -CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL); +CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_hourly') chunk; -- Backfill data and refresh again WITHOUT direct compress INSERT INTO conditions SELECT t, d::text, 1, 1 FROM generate_series('2025-12-15 00:00:00+00'::timestamptz - interval '1 year', '2025-12-15 00:00:00+00'::timestamptz, interval '1 hour') AS t, generate_series(1, 10) AS d; SET timescaledb.enable_direct_compress_on_cagg_refresh TO off; -CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL); +CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_hourly') chunk; -- Recompress all uncompressed chunks @@ -57,7 +58,7 @@ SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks INSERT INTO conditions SELECT t, d::text, 1, 1 FROM generate_series('2025-12-15 00:00:00+00'::timestamptz - interval '1 year', '2025-12-15 00:00:00+00'::timestamptz, interval '1 hour') AS t, generate_series(1, 10) AS d; SET timescaledb.enable_direct_compress_on_cagg_refresh TO on; -CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL); +CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_hourly') chunk; -- Cleanup @@ -85,11 +86,11 @@ SELECT t, d::text, 1, 1 FROM generate_series('2025-12-15 00:00:00+00'::timestamp SET timescaledb.enable_direct_compress_on_cagg_refresh TO on; -- Refresh the base CAgg -CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL); +CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_hourly') chunk; -- Refresh the hierarchical CAgg -CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL); +CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_daily') chunk; -- Produce some invalidations for the base CAgg @@ -97,18 +98,18 @@ INSERT INTO conditions SELECT t, d::text, 1, 1 FROM generate_series('2025-12-15 00:00:00+00'::timestamptz - interval '1 year', '2025-12-15 00:00:00+00'::timestamptz, interval '1 hour') AS t, generate_series(1, 10) AS d; -- Refresh the base CAgg -CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL); +CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_hourly') chunk; -- Refreshing again the base CAgg is a no-op since everything is up to date -CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL); +CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); -- Refresh the hierarchical CAgg with invalidations procuded by the base CAgg -CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL); +CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_daily') chunk; -- Refreshing again the hierarchical CAgg is a no-op since everything is up to date -CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL); +CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); -- Tests with custom segmentby and orderby CREATE MATERIALIZED VIEW conditions_weekly @@ -126,7 +127,7 @@ WITH NO DATA; ALTER MATERIALIZED VIEW conditions_weekly SET (timescaledb.compress_segmentby = 'device_id, location_id', timescaledb.compress_orderby = 'max, min, bucket DESC'); -CALL refresh_continuous_aggregate('conditions_weekly', NULL, NULL); +CALL refresh_continuous_aggregate('conditions_weekly', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_weekly') chunk; -- Test GROUP BY ROLLUP on compressed continuous aggregate (issue #9520) diff --git a/tsl/test/sql/cagg_invalidation.sql b/tsl/test/sql/cagg_invalidation.sql index 84cbf14ef63..f543798c0f6 100644 --- a/tsl/test/sql/cagg_invalidation.sql +++ b/tsl/test/sql/cagg_invalidation.sql @@ -1017,6 +1017,13 @@ SET timezone = 'UTC'; call refresh_continuous_aggregate ('test_cagg','2023-12-29 15:00:00', '2026-01-28 15:00:00'); call refresh_continuous_aggregate ('test_cagg_1d_offset','2023-12-29 18:00:00', '2024-01-01 18:00:00'); +-- test_cagg uses 1-month buckets with buckets_per_batch=10. The inscribed window +-- [2024-01-01, 2026-01-01) spans 24 months, producing two batches: +-- [2024-01-01, 2024-11-01) and [2024-11-01, 2025-09-01). +-- Data only exists through Dec 2024, so the split function finds no chunks past +-- Dec 2024 and stops at Sep 2025 (end of batch 2). The range [2025-09-01, 2026-01-01) +-- has no data and is left unprocessed, so the upper residual in the mat_inval_log +-- is 2025-09-01 rather than 2026-01-01 as it would be with single-pass refresh. SELECT materialization_id, _timescaledb_functions.to_timestamp(lowest_modified_value) as low, _timescaledb_functions.to_timestamp(greatest_modified_value) as high @@ -1043,9 +1050,10 @@ WHERE materialization_id = ( ) ORDER BY lowest_modified_value, greatest_modified_value; - ---now do the same refresh again, it should say the cagg is already up to date +-- Refresh the same range again, [2025-09-01 ─ 2026-01-01) was left. +-- This time refresh will go into the single batch path, so it will process the whole range. CALL refresh_continuous_aggregate ('test_cagg','2023-12-29 15:00:00', '2026-01-28 15:00:00'); +--Do the same refresh once again, it should say the cagg is already up to date CALL refresh_continuous_aggregate ('test_cagg','2023-12-29 15:00:00', '2026-01-28 15:00:00'); @@ -1095,7 +1103,8 @@ WHERE hypertable_id IN ( SELECT raw_hypertable_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'test_cagg_1d_offset'); INSERT INTO test_data values ('2026-01-05 00:00:00', 1); -CALL refresh_continuous_aggregate ('test_cagg_1d_offset','2023-12-29 15:00:00', NULL); +-- Setting buckets_per_batch to 0 as the intention is to test the capping when windew end is set to NULL, no need for batching. +CALL refresh_continuous_aggregate ('test_cagg_1d_offset','2023-12-29 15:00:00', NULL, options => '{"buckets_per_batch": 0}'::jsonb); --should be at the 18th hour SELECT _timescaledb_functions.to_timestamp(watermark) as invalidation_threshold @@ -1158,9 +1167,9 @@ WHERE hypertable_id = ( -- Now refresh cagg_4hrs with NULL,NULL. -- cagg_4hrs computes its own threshold = 04:00, but stored threshold = 06:00 > 04:00, -- so the stored (misaligned) value is used but capped to the start of the current bucket of cagg_4hrs, --- which is 2020-01-01 04:00 UTC. +-- which is 2020-01-01 04:00 UTC. Disabling incremental refresh to preseve the isolate the test intention. SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_4hrs', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4hrs', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); RESET client_min_messages; -- Show invalidations left in cagg_4hrs's invalidation log, diff --git a/tsl/test/sql/cagg_invalidation_variable_bucket.sql b/tsl/test/sql/cagg_invalidation_variable_bucket.sql index 038eb48d27e..0b272e2396b 100644 --- a/tsl/test/sql/cagg_invalidation_variable_bucket.sql +++ b/tsl/test/sql/cagg_invalidation_variable_bucket.sql @@ -217,7 +217,8 @@ FROM generate_series('2025-03-30 00:00:00'::timestamptz, '2025-03-31 23:59:59.999999'::timestamptz, '1 hour'::interval) ts; -CALL refresh_continuous_aggregate('cagg_dst_daily', '2025-03-01 00:00:00', '2025-05-01 00:00:00'); +-- Disabling incremental refresh as it's not the focus of the test and it's easier to read the test output without it. +CALL refresh_continuous_aggregate('cagg_dst_daily', '2025-03-01 00:00:00', '2025-05-01 00:00:00', options => '{"buckets_per_batch": 0}'::jsonb); -- March 30 should have 23 hours SELECT bucket, cnt FROM cagg_dst_daily ORDER BY bucket; @@ -234,7 +235,7 @@ FROM generate_series('2025-10-26 00:00:00'::timestamptz, '1 hour'::interval) ts; -- Wide window to cover all DST-shifted buckets -CALL refresh_continuous_aggregate('cagg_dst_daily', '2025-10-01 00:00:00', '2026-12-01 00:00:00'); +CALL refresh_continuous_aggregate('cagg_dst_daily', '2025-10-01 00:00:00', '2026-12-01 00:00:00', options => '{"buckets_per_batch": 0}'::jsonb); -- October bucket should have extra hour (25-hour day on Oct 26) SELECT bucket, cnt FROM cagg_dst_daily @@ -245,7 +246,7 @@ ORDER BY bucket; INSERT INTO dst_data VALUES ('2025-10-26 01:00:00', 888.0); -- 2:00 AM Europe/Berlin (after fall-back) INSERT INTO dst_data VALUES ('2025-10-26 00:30:00', 777.0); -- 2:30 AM Europe/Berlin (before fall-back) -CALL refresh_continuous_aggregate('cagg_dst_daily', '2025-09-01 00:00:00', '2026-02-01 00:00:00'); +CALL refresh_continuous_aggregate('cagg_dst_daily', '2025-09-01 00:00:00', '2026-02-01 00:00:00', options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_inval_log WHERE cagg_name = 'cagg_dst_daily'; SET timezone TO 'UTC'; diff --git a/tsl/test/sql/cagg_refresh_incremental.sql b/tsl/test/sql/cagg_refresh_incremental.sql new file mode 100644 index 00000000000..74d56014516 --- /dev/null +++ b/tsl/test/sql/cagg_refresh_incremental.sql @@ -0,0 +1,491 @@ +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license. + +\c :TEST_DBNAME :ROLE_SUPERUSER + +-- Create a user with specific timezone for deterministic output +CREATE ROLE test_cagg_refresh_manual_user WITH LOGIN; +ALTER ROLE test_cagg_refresh_manual_user SET timezone TO 'UTC'; +GRANT ALL ON SCHEMA public TO test_cagg_refresh_manual_user; + +\c :TEST_DBNAME test_cagg_refresh_manual_user +SET timezone TO 'UTC'; + +CREATE TABLE conditions ( + time TIMESTAMP WITH TIME ZONE NOT NULL, + device_id INTEGER, + temperature NUMERIC +); + +SELECT FROM create_hypertable('conditions', by_range('time')); + +INSERT INTO conditions +SELECT + t, d, 10 +FROM + generate_series( + '2025-02-05 00:00:00+00', + '2025-03-05 00:00:00+00', + '1 hour'::interval) AS t, + generate_series(1,5) AS d; + +CREATE MATERIALIZED VIEW conditions_by_day +WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS +SELECT + time_bucket('1 day', time), + device_id, + count(*), + min(temperature), + max(temperature), + avg(temperature), + sum(temperature) +FROM + conditions +GROUP BY + 1, 2 +WITH NO DATA; + +-- Issue an incremental manual refresh using JSONB options: +-- buckets_per_batch => 10. This is the manual equivalent of a +-- policy with the same setting. +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 10}'::jsonb); +RESET client_min_messages; + +SELECT count(*) FROM conditions_by_day; + +CREATE MATERIALIZED VIEW conditions_by_day_atomic_refresh +WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS +SELECT + time_bucket('1 day', time), + device_id, + count(*), + min(temperature), + max(temperature), + avg(temperature), + sum(temperature) +FROM + conditions +GROUP BY + 1, 2 +WITH NO DATA; + +CALL refresh_continuous_aggregate('conditions_by_day_atomic_refresh', NULL, NULL); + +SELECT count(*) FROM conditions_by_day; +SELECT count(*) FROM conditions_by_day_atomic_refresh; + +-- Should have no differences +SELECT + count(*) > 0 AS has_diff +FROM + ((SELECT * FROM conditions_by_day_atomic_refresh ORDER BY 1, 2) + EXCEPT + (SELECT * FROM conditions_by_day ORDER BY 1, 2)) AS diff; + +-- buckets_per_batch => 0 is NOT incremental: the whole window is materialized +-- in a single pass. Under LOG this shows exactly one delete+insert pair, unlike +-- the multi-batch run above. The TRUNCATE invalidates the whole range so there +-- is data to re-materialize. +TRUNCATE conditions_by_day; +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 0}'::jsonb); +RESET client_min_messages; +SELECT count(*) FROM conditions_by_day; + +-- The continuous aggregate is now fully materialized with no pending +-- invalidations, so a normal refresh is a no-op (reports up-to-date). +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate('conditions_by_day', NULL, NULL); +RESET client_min_messages; + +-- Assert there really are no pending invalidations +SELECT + (SELECT count(*) + FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log + WHERE materialization_id = cagg.mat_hypertable_id + AND greatest_modified_value >= lowest_modified_value + AND lowest_modified_value != -9223372036854775808 + AND greatest_modified_value != 9223372036854775807) AS mat_invalidations, + (SELECT count(*) + FROM _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log + WHERE hypertable_id = cagg.raw_hypertable_id + AND greatest_modified_value >= lowest_modified_value) AS ht_invalidations +FROM _timescaledb_catalog.continuous_agg cagg +WHERE cagg.user_view_name = 'conditions_by_day'; + +-- A forced refresh is incremental but ignores the invalidation logs: +-- it re-materializes the entire window in batches (buckets_per_batch +-- defaults to 10) even though nothing is invalidated. +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate('conditions_by_day', NULL, NULL, force => true); +RESET client_min_messages; +SELECT count(*) FROM conditions_by_day; + +TRUNCATE conditions_by_day; + +-- Run with max_batches_per_execution => 2. Manual refresh stops mid-window +-- after processing 2 batches, leaving the rest for subsequent calls. +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 10, "max_batches_per_execution": 2}'::jsonb); +RESET client_min_messages; + +SELECT count(*) FROM conditions_by_day; +SELECT count(*) FROM conditions_by_day_atomic_refresh; + +-- Should have differences (partial materialization) +SELECT + count(*) > 0 AS has_diff +FROM + ((SELECT * FROM conditions_by_day_atomic_refresh ORDER BY 1, 2) + EXCEPT + (SELECT * FROM conditions_by_day ORDER BY 1, 2)) AS diff; + +-- Run a second call (same options) to process more batches. +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 10, "max_batches_per_execution": 2}'::jsonb); +RESET client_min_messages; + +SELECT count(*) FROM conditions_by_day; +SELECT count(*) FROM conditions_by_day_atomic_refresh; + +-- Should have no differences (all batches processed) +SELECT + count(*) > 0 AS has_diff +FROM + ((SELECT * FROM conditions_by_day_atomic_refresh ORDER BY 1, 2) + EXCEPT + (SELECT * FROM conditions_by_day ORDER BY 1, 2)) AS diff; + +-- Set max_batches_per_execution to 10 (effectively unlimited for our window) +-- and insert data into the past so a new set of batches must be processed. +INSERT INTO conditions +SELECT + t, d, 10 +FROM + generate_series( + '2020-02-05 00:00:00+00', + '2020-03-05 00:00:00+00', + '1 hour'::interval) AS t, + generate_series(1,5) AS d; + +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 10, "max_batches_per_execution": 10}'::jsonb); +RESET client_min_messages; + +SELECT count(*) FROM conditions_by_day; +SELECT count(*) FROM conditions_by_day_atomic_refresh; + +CALL refresh_continuous_aggregate('conditions_by_day_atomic_refresh', NULL, NULL); + +SELECT count(*) FROM conditions_by_day; +SELECT count(*) FROM conditions_by_day_atomic_refresh; + +-- Should have no differences +SELECT + count(*) > 0 AS has_diff +FROM + ((SELECT * FROM conditions_by_day_atomic_refresh ORDER BY 1, 2) + EXCEPT + (SELECT * FROM conditions_by_day ORDER BY 1, 2)) AS diff; + +-- Invalid configurations should be rejected +\set ON_ERROR_STOP 0 +\set VERBOSITY default +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"max_batches_per_execution": -1}'::jsonb); +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": -1}'::jsonb); +\set VERBOSITY terse +\set ON_ERROR_STOP 1 + +-- Truncate all data from the original hypertable. +TRUNCATE conditions; + +-- Should fall back to single-batch processing because there's no data +-- to refresh on the source hypertable. +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 10}'::jsonb); +RESET client_min_messages; + +-- Should return zero rows +SELECT count(*) FROM conditions_by_day; + +-- Insert 1 day of data +INSERT INTO conditions +SELECT + t, d, 10 +FROM + generate_series( + '2020-02-05 00:00:00+00', + '2020-02-06 00:00:00+00', + '1 hour'::interval) AS t, + generate_series(1,5) AS d; + +-- Should fall back to single-batch processing because the refresh size +-- (1 day) is smaller than 10 buckets x 1 day. +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 10}'::jsonb); +RESET client_min_messages; + +-- Should return 10 rows because the bucket width is `1 day` and we +-- inserted across two boundary timestamps for 5 devices. +SELECT count(*) FROM conditions_by_day; + +TRUNCATE conditions_by_day, conditions; + +-- Less than 1 day of data (smaller than the bucket width) +INSERT INTO conditions +VALUES ('2020-02-05 00:00:00+00', 1, 10); + +-- Should fall back to single-batch processing because the refresh size +-- is smaller than the bucket width. +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 10}'::jsonb); +RESET client_min_messages; + +-- Should return 1 row +SELECT count(*) FROM conditions_by_day; + +-- Re-test with an explicit refresh_newest_first => true (default behavior). +TRUNCATE conditions_by_day, conditions_by_day_atomic_refresh, conditions; + +INSERT INTO conditions +SELECT + t, d, 10 +FROM + generate_series( + '2025-03-11 00:00:00+00'::timestamptz - INTERVAL '30 days', + '2025-03-11 00:00:00+00'::timestamptz, + '1 hour'::interval) AS t, + generate_series(1,5) AS d; + +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_day', + '2025-03-11 00:00:00+00'::timestamptz - INTERVAL '15 days', + NULL, + options => '{"buckets_per_batch": 5, "refresh_newest_first": true}'::jsonb); +RESET client_min_messages; + +CALL refresh_continuous_aggregate( + 'conditions_by_day_atomic_refresh', + '2025-03-11 00:00:00+00'::timestamptz - INTERVAL '15 days', + NULL); + +-- Both continuous aggregates should have the same data +SELECT count(*) FROM conditions_by_day; +SELECT count(*) FROM conditions_by_day_atomic_refresh; + +-- Should have no differences +SELECT + count(*) > 0 AS has_diff +FROM + ((SELECT * FROM conditions_by_day_atomic_refresh ORDER BY 1, 2) + EXCEPT + (SELECT * FROM conditions_by_day ORDER BY 1, 2)) AS diff; + +-- refresh_newest_first => false (process from oldest to newest) +TRUNCATE conditions_by_day; + +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_day', + '2025-03-11 00:00:00+00'::timestamptz - INTERVAL '15 days', + NULL, + options => '{"buckets_per_batch": 5, "refresh_newest_first": false}'::jsonb); +RESET client_min_messages; + +-- Both continuous aggregates should have the same data +SELECT count(*) FROM conditions_by_day; +SELECT count(*) FROM conditions_by_day_atomic_refresh; + +-- Should have no differences +SELECT + count(*) > 0 AS has_diff +FROM + ((SELECT * FROM conditions_by_day_atomic_refresh ORDER BY 1, 2) + EXCEPT + (SELECT * FROM conditions_by_day ORDER BY 1, 2)) AS diff; + +-- Tests with variable-sized bucket (monthly) +TRUNCATE conditions; + +INSERT INTO conditions +SELECT + t, d, 10 +FROM + generate_series( + '2025-01-01 00:00:00+00', + '2025-10-08 00:00:00+00', + '1 hour'::interval) AS t, + generate_series(1,5) AS d; + +CREATE MATERIALIZED VIEW conditions_by_month +WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS +SELECT + time_bucket('1 month', time), + device_id, + count(*), + min(temperature), + max(temperature), + avg(temperature), + sum(temperature) +FROM + conditions +GROUP BY + 1, 2 +WITH NO DATA; + +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_month', + '2025-03-11 00:00:00+00'::timestamptz - INTERVAL '600 days', + '2025-03-11 00:00:00+00'::timestamptz - INTERVAL '7 days', + options => '{"refresh_newest_first": false}'::jsonb); +RESET client_min_messages; + +SELECT count(*) FROM conditions_by_month; + +------------------------------------------------------------------------------------------ +-- Test that batched refresh with variable-length buckets doesn't leave remainders +------------------------------------------------------------------------------------------ +CREATE TABLE test_data ( + time TIMESTAMPTZ NOT NULL, + value INT +); + +SELECT public.create_hypertable( + relation => 'test_data', + time_column_name => 'time', + chunk_time_interval => interval '1 months' +); +-- Insert initial data +INSERT INTO test_data +SELECT time, 1 +FROM generate_series('2024-01-01'::timestamptz, '2024-12-31'::timestamptz, '1 day'::interval) time; + +-- Create continuous aggregate with monthly buckets (variable-length) +CREATE MATERIALIZED VIEW batch_test_cagg +WITH (timescaledb.continuous) AS +SELECT + time_bucket('1 month'::interval, time) AS bucket, + count(*) as count +FROM test_data +GROUP BY bucket +WITH NO DATA; + +-- Run incremental manual refresh, 1 bucket per batch +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'batch_test_cagg', NULL, '2024-12-01'::timestamptz, + options => '{"buckets_per_batch": 1}'::jsonb); +RESET client_min_messages; + +-- Verify that the materialization invalidation log has no entries other than +-- the boundary -/+ infinity rows. +SELECT materialization_id, + _timescaledb_functions.to_timestamp(lowest_modified_value) as low, + _timescaledb_functions.to_timestamp(greatest_modified_value) as high +FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log +WHERE materialization_id IN + (SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg + WHERE user_view_name = 'batch_test_cagg') + AND lowest_modified_value != -9223372036854775808 + AND greatest_modified_value != 9223372036854775807 +ORDER BY low; + +-- Running again should be a no-op +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'batch_test_cagg', NULL, '2024-12-01'::timestamptz, + options => '{"buckets_per_batch": 1}'::jsonb); +RESET client_min_messages; + +DROP TABLE test_data CASCADE; + +------------------------------------------------------------------------------------------ +-- Test incremental manual refresh crashing between batches +------------------------------------------------------------------------------------------ + +-- Rows processed by a batch should be visible immediately after it finishes. +-- Inject an error after batch 1 completes and observe the cagg state. +\c :TEST_DBNAME :ROLE_SUPERUSER +TRUNCATE _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log, _timescaledb_catalog.continuous_aggs_materialization_invalidation_log; +\c :TEST_DBNAME test_cagg_refresh_manual_user +SET timezone TO 'UTC'; + +TRUNCATE conditions, conditions_by_day, conditions_by_day_atomic_refresh; + +INSERT INTO conditions +SELECT + t, d, 10 +FROM + generate_series( + '2025-02-05 00:00:00+00', + '2025-03-05 00:00:00+00', + '1 hour'::interval) AS t, + generate_series(1,5) AS d; + +-- Crash after batch 1 finishes +SELECT debug_waitpoint_enable('cagg_policy_batch_1_after_refresh'); + +-- Cagg state before refresh starts +SELECT min(time_bucket), max(time_bucket) FROM conditions_by_day; + +SET client_min_messages TO LOG; +\set ON_ERROR_STOP 0 +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, '1 hour'::interval, + options => '{"buckets_per_batch": 10}'::jsonb); +\set ON_ERROR_STOP 1 +RESET client_min_messages; + +-- Rows processed by batch 1 should be materialized +SELECT count(*) AS rows_after_refresh FROM conditions_by_day; +SELECT min(time_bucket), max(time_bucket) FROM conditions_by_day; + +-- Registered ranges should be cleaned up after the crash +SELECT count(*) AS registered_ranges FROM _timescaledb_catalog.continuous_aggs_jobs_refresh_ranges; + +SELECT debug_waitpoint_release('cagg_policy_batch_1_after_refresh'); + +-- Process remaining invalidations and verify +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, '1 hour'::interval, + options => '{"buckets_per_batch": 10}'::jsonb); + +-- Verify against an atomic refresh +CALL refresh_continuous_aggregate('conditions_by_day_atomic_refresh', NULL, '1 hour'::interval); + +SELECT + count(*) > 0 AS has_diff +FROM + ((SELECT * FROM conditions_by_day_atomic_refresh ORDER BY 1, 2) + EXCEPT + (SELECT * FROM conditions_by_day ORDER BY 1, 2)) AS diff; + +\c :TEST_DBNAME :ROLE_SUPERUSER +REASSIGN OWNED BY test_cagg_refresh_manual_user TO :ROLE_SUPERUSER; +REVOKE ALL ON SCHEMA public FROM test_cagg_refresh_manual_user; + +DROP ROLE test_cagg_refresh_manual_user; diff --git a/tsl/test/sql/include/cagg_query_common.sql b/tsl/test/sql/include/cagg_query_common.sql index efdceb6b548..7dd4660dc72 100644 --- a/tsl/test/sql/include/cagg_query_common.sql +++ b/tsl/test/sql/include/cagg_query_common.sql @@ -54,7 +54,7 @@ group by time_bucket('1day', timec), location WITH NO DATA; --compute time_bucketted max+bucket_width for the materialized view SELECT time_bucket('1day' , q.timeval+ '1day'::interval) FROM ( select max(timec)as timeval from conditions ) as q; -CALL refresh_continuous_aggregate('mat_m1', NULL, NULL); +CALL refresh_continuous_aggregate('mat_m1', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); --test first/last create materialized view mat_m2(location, timec, firsth, lasth, maxtemp, mintemp) @@ -66,7 +66,7 @@ group by time_bucket('1day', timec), location WITH NO DATA; --time that refresh assumes as now() for repeatability SELECT time_bucket('1day' , q.timeval+ '1day'::interval) FROM ( select max(timec)as timeval from conditions ) as q; -CALL refresh_continuous_aggregate('mat_m2', NULL, NULL); +CALL refresh_continuous_aggregate('mat_m2', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); --normal view -- create or replace view regview( location, timec, minl, sumt , sumh) @@ -563,7 +563,7 @@ SET ROLE :ROLE_SUPERUSER; BEGIN; UPDATE _timescaledb_catalog.continuous_aggs_bucket_function SET bucket_func = 'func_does_not_exist()'; -- should error because function does not exist -CALL refresh_continuous_aggregate('cagg_bigint_offset2', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_bigint_offset2', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); ROLLBACK; \set ON_ERROR_STOP 1 SET ROLE :ROLE_DEFAULT_PERM_USER; @@ -644,9 +644,9 @@ SELECT * FROM cagg_4_hours_origin; -- Update the last bucket and re-materialize INSERT INTO temperature values('2020-01-01 23:55:00 PST', 10); -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_4_hours; SELECT * FROM cagg_4_hours_offset; @@ -680,9 +680,9 @@ SELECT * FROM cagg_4_hours_origin; -- Update materialized data SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); RESET client_min_messages; -- Query the CAggs and check that all buckets are materialized @@ -703,9 +703,9 @@ SELECT time_bucket('4 hour', time, '2000-01-01 01:00:00 PST'::timestamptz), max( -- Test invalidations TRUNCATE temperature; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); INSERT INTO temperature SELECT time, 5 @@ -722,9 +722,9 @@ INSERT INTO temperature values('2020-01-02 01:35:00+01', 5555); INSERT INTO temperature values('2020-01-02 05:05:00+01', 8888); SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); RESET client_min_messages; ALTER MATERIALIZED VIEW cagg_4_hours SET (timescaledb.materialized_only=true); @@ -780,7 +780,7 @@ WHERE materialization_id IN (SELECT mat_hypertable_id FROM _timescaledb_catalog. WHERE user_view_name = 'cagg_4_hours') ORDER BY 1, 2, 3; -CALL refresh_continuous_aggregate('cagg_4_hours', '2000-01-01 00:00:00'::timestamptz, '2020-12-31 23:59:59'::timestamptz); +CALL refresh_continuous_aggregate('cagg_4_hours', '2000-01-01 00:00:00'::timestamptz, '2020-12-31 23:59:59'::timestamptz, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log WHERE materialization_id IN (SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg @@ -823,12 +823,12 @@ INSERT INTO table_int VALUES(100, 555); -- Compare bucketing results SELECT time_bucket('10', time), SUM(data) FROM table_int GROUP BY 1 ORDER BY 1; SELECT * FROM cagg_int; -CALL refresh_continuous_aggregate('cagg_int', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_int', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_int; SELECT time_bucket('10', time, "offset"=>5), SUM(data) FROM table_int GROUP BY 1 ORDER BY 1; SELECT * FROM cagg_int_offset; -- the value 100 is part of the already serialized bucket, so it should not be visible -CALL refresh_continuous_aggregate('cagg_int_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_int_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_int_offset; -- Ensure everything was materialized @@ -842,7 +842,7 @@ SELECT * FROM cagg_int_offset; INSERT INTO table_int VALUES(114, 0); SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130); +CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130, options => '{"buckets_per_batch": 0}'::jsonb); RESET client_min_messages; SELECT * FROM cagg_int_offset; @@ -982,8 +982,8 @@ SELECT * FROM cagg_1_week_offset; SELECT time_bucket('1 week', time, origin=>'2000-01-02 01:00:00 PST'::timestamptz), max(value) FROM temperature GROUP BY 1 ORDER BY 1; -- Test refresh -CALL refresh_continuous_aggregate('cagg_1_hour_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_1_week_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_1_hour_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_1_week_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); -- Everything should be now materailized ALTER MATERIALIZED VIEW cagg_1_hour_offset SET (timescaledb.materialized_only=false); diff --git a/tsl/test/sql/telemetry_stats.sql b/tsl/test/sql/telemetry_stats.sql index 080f4343058..245fb1927a8 100644 --- a/tsl/test/sql/telemetry_stats.sql +++ b/tsl/test/sql/telemetry_stats.sql @@ -77,7 +77,7 @@ SELECT * FROM normal; INSERT INTO part SELECT * FROM normal; -CALL refresh_continuous_aggregate('contagg', NULL, NULL); +CALL refresh_continuous_aggregate('contagg', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); -- Reindex to avoid the dependency on the way the index is built (e.g. the caggs -- might get their rows inserted in different order during the refresh based on