diff --git a/.unreleased/pr_9903 b/.unreleased/pr_9903 new file mode 100644 index 00000000000..fdfdd9e1bf6 --- /dev/null +++ b/.unreleased/pr_9903 @@ -0,0 +1 @@ +Implements: #9903 Incremental refresh for refresh_continuous_aggregate() diff --git a/tsl/src/bgw_policy/continuous_aggregate_api.c b/tsl/src/bgw_policy/continuous_aggregate_api.c index 0e249157dc1..cc60a2ded5f 100644 --- a/tsl/src/bgw_policy/continuous_aggregate_api.c +++ b/tsl/src/bgw_policy/continuous_aggregate_api.c @@ -21,6 +21,7 @@ #include "bgw_policy/job_api.h" #include "bgw_policy/policies_v2.h" #include "bgw_policy/policy_utils.h" +#include "continuous_aggs/refresh.h" #include "dimension.h" #include "guc.h" #include "jsonb_utils.h" @@ -32,12 +33,8 @@ /* Default max runtime for a continuous aggregate jobs is unlimited for now */ #define DEFAULT_MAX_RUNTIME \ DatumGetIntervalP(DirectFunctionCall3(interval_in, CStringGetDatum("0"), InvalidOid, -1)) -/* Default buckets per batch is 1, which means that the job will refresh 1 bucket at a time */ -#define DEFAULT_BUCKETS_PER_BATCH 10 /* Default max batches per execution is 0, which means no limit */ #define DEFAULT_MAX_BATCHES_PER_EXECUTION 0 -/* Default refresh newest first is true, which means from newest data to the oldest */ -#define DEFAULT_REFRESH_NEWEST_FIRST true /* Default compress after refresh is false, which means compression does not run after refresh */ #define DEFAULT_COMPRESS_AFTER_REFRESH false diff --git a/tsl/src/bgw_policy/job.c b/tsl/src/bgw_policy/job.c index 9c2bd8950b8..4e9eb9a3dfd 100644 --- a/tsl/src/bgw_policy/job.c +++ b/tsl/src/bgw_policy/job.c @@ -422,74 +422,18 @@ policy_refresh_cagg_execute(int32 job_id, Jsonb *config) PGC_S_SESSION); } - ContinuousAggRefreshContext context = { .callctx = CAGG_REFRESH_POLICY, .job_id = job_id }; - - /* Try to split window range into a list of ranges */ - List *refresh_window_list = continuous_agg_split_refresh_window(policy_data.cagg, - &policy_data.refresh_window, - policy_data.buckets_per_batch); - if (refresh_window_list == NIL) - { - refresh_window_list = lappend(refresh_window_list, &policy_data.refresh_window); - } - else - { - context.callctx = CAGG_REFRESH_POLICY_BATCHED; - } - - context.number_of_batches = list_length(refresh_window_list); - - /* - * The list is always built oldest-first. When refresh_newest_first is true we - * iterate from the last element down to the first using index-based access so - * that no reversal copy of the list is needed. - */ - int32 processing_batch = 0; - int32 nbatches = list_length(refresh_window_list); - int32 batch_start = policy_data.refresh_newest_first ? nbatches - 1 : 0; - int32 batch_end = policy_data.refresh_newest_first ? -1 : nbatches; - int32 batch_step = policy_data.refresh_newest_first ? -1 : 1; - for (int32 batch_idx = batch_start; batch_idx != batch_end; batch_idx += batch_step) - { - InternalTimeRange *refresh_window = - (InternalTimeRange *) list_nth(refresh_window_list, batch_idx); - elog(DEBUG1, - "refreshing continuous aggregate \"%s\" from %s to %s", - NameStr(policy_data.cagg->data.user_view_name), - ts_internal_to_time_string(refresh_window->start, refresh_window->type), - ts_internal_to_time_string(refresh_window->end, refresh_window->type)); - - context.processing_batch = ++processing_batch; - - /* extend_last_bucket must only apply to the boundary batch — the one - * whose window abuts the adjacent policy. For newest-first ordering - * that is batch 1; for oldest-first it is the final batch. - * In non-batched mode (single batch) the one batch is always the boundary. */ - bool apply_extend = - extend_last_bucket && - (policy_data.refresh_newest_first ? processing_batch == 1 : - processing_batch == context.number_of_batches); - - continuous_agg_refresh_internal(policy_data.cagg, - refresh_window, - context, - refresh_window->start_isnull, - refresh_window->end_isnull, - (context.callctx != CAGG_REFRESH_POLICY_BATCHED), - false, /* force */ - apply_extend); - DEBUG_ERROR_INJECTION(psprintf("cagg_policy_batch_%d_after_refresh", processing_batch)); - if (processing_batch >= policy_data.max_batches_per_execution && - processing_batch < context.number_of_batches && - policy_data.max_batches_per_execution > 0) - { - elog(LOG, - "reached maximum number of batches per execution (%d), batches not processed (%d)", - policy_data.max_batches_per_execution, - context.number_of_batches - processing_batch); - break; - } - } + ContinuousAggRefreshContext context = { + .callctx = CAGG_REFRESH_POLICY, + .job_id = job_id, + .buckets_per_batch = policy_data.buckets_per_batch, + .max_batches_per_execution = policy_data.max_batches_per_execution, + .refresh_newest_first = policy_data.refresh_newest_first, + }; + + continuous_agg_refresh_batched(policy_data.cagg, + &policy_data.refresh_window, + context, + extend_last_bucket); if (!policy_data.include_tiered_data_isnull) { diff --git a/tsl/src/continuous_aggs/common.c b/tsl/src/continuous_aggs/common.c index 1f73f6a76fd..6095c3fb3ba 100644 --- a/tsl/src/continuous_aggs/common.c +++ b/tsl/src/continuous_aggs/common.c @@ -2102,3 +2102,21 @@ cagg_find_groupingcols(ContinuousAgg *agg, Hypertable *mat_ht) } return retlist; } + +void +emit_up_to_date_notice(const ContinuousAgg *cagg, const ContinuousAggRefreshContext context) +{ + switch (context.callctx) + { + case CAGG_REFRESH_WINDOW: + case CAGG_REFRESH_CREATION: + case CAGG_REFRESH_WINDOW_BATCHED: + elog(NOTICE, + "continuous aggregate \"%s\" is already up-to-date", + NameStr(cagg->data.user_view_name)); + break; + case CAGG_REFRESH_POLICY: + case CAGG_REFRESH_POLICY_BATCHED: + break; + } +} diff --git a/tsl/src/continuous_aggs/common.h b/tsl/src/continuous_aggs/common.h index b1f7cac7e82..2710dcccc4e 100644 --- a/tsl/src/continuous_aggs/common.h +++ b/tsl/src/continuous_aggs/common.h @@ -79,6 +79,7 @@ typedef enum ContinuousAggRefreshCallContext { CAGG_REFRESH_CREATION, CAGG_REFRESH_WINDOW, + CAGG_REFRESH_WINDOW_BATCHED, CAGG_REFRESH_POLICY, CAGG_REFRESH_POLICY_BATCHED } ContinuousAggRefreshCallContext; @@ -89,6 +90,11 @@ typedef struct ContinuousAggRefreshContext int32 job_id; int32 processing_batch; int32 number_of_batches; + /* Batch configuration */ + int32 buckets_per_batch; /* 0 = disabled */ + int32 max_batches_per_execution; /* 0 = no limit */ + bool refresh_newest_first; + bool force; /* re-materialize the whole window, ignoring invalidations */ } ContinuousAggRefreshContext; #define IS_TIME_BUCKET_INFO_TIME_BASED(bucket_function) \ @@ -173,3 +179,5 @@ extern bool caggtimebucket_validate_common(ContinuousAggBucketFunction *bf, List List *targetList, List *rtable, int ht_partcolno, StringInfo msg, bool is_cagg_create, const bool for_rewrites); +extern void emit_up_to_date_notice(const ContinuousAgg *cagg, + const ContinuousAggRefreshContext context); diff --git a/tsl/src/continuous_aggs/create.c b/tsl/src/continuous_aggs/create.c index 5a1b63ca4b6..a5f0bbe1e48 100644 --- a/tsl/src/continuous_aggs/create.c +++ b/tsl/src/continuous_aggs/create.c @@ -857,8 +857,11 @@ tsl_process_continuous_agg_viewstmt(Node *node, const char *query_string, void * if (!stmt->into->skipData) { + bool refreshed = false; InternalTimeRange refresh_window = { .type = InvalidOid, + .start_isnull = true, + .end_isnull = true, }; /* @@ -887,14 +890,15 @@ tsl_process_continuous_agg_viewstmt(Node *node, const char *query_string, void * refresh_window.end = ts_time_get_noend_or_max(refresh_window.type); ContinuousAggRefreshContext context = { .callctx = CAGG_REFRESH_CREATION }; - continuous_agg_refresh_internal(cagg, - &refresh_window, - context, - true, /* start_isnull */ - true, /* end_isnull */ - true, /* bucketing_refresh_window */ - false, /* force */ - false /*extend_last_bucket*/); + refreshed = continuous_agg_refresh_internal(cagg, + &refresh_window, + context, + true, /* bucketing_refresh_window */ + false /*extend_last_bucket*/); + if (!refreshed) + { + emit_up_to_date_notice(cagg, context); + } } return DDL_DONE; diff --git a/tsl/src/continuous_aggs/refresh.c b/tsl/src/continuous_aggs/refresh.c index c12e1b2db96..f281bdb7226 100644 --- a/tsl/src/continuous_aggs/refresh.c +++ b/tsl/src/continuous_aggs/refresh.c @@ -20,6 +20,7 @@ #include #include +#include "bgw_policy/policies_v2.h" #include "debug_point.h" #include "dimension.h" #include "dimension_slice.h" @@ -79,12 +80,10 @@ static void continuous_agg_refresh_with_window(const ContinuousAgg *cagg, const InvalidationStore *invalidations, const ContinuousAggRefreshContext context, bool bucketing_refresh_window); -static void emit_up_to_date_notice(const ContinuousAgg *cagg, - const ContinuousAggRefreshContext context); static bool process_cagg_invalidations_and_refresh(const ContinuousAgg *cagg, const InternalTimeRange *refresh_window, const ContinuousAggRefreshContext context, - bool bucketing_refresh_window, bool force); + bool bucketing_refresh_window); static Hypertable * cagg_get_hypertable_or_fail(int32 hypertable_id) { @@ -624,6 +623,111 @@ continuous_agg_refresh_with_window(const ContinuousAgg *cagg, } #define REFRESH_FUNCTION_NAME "refresh_continuous_aggregate()" + +/* + * Refresh a continuous aggregate over a window, splitting it into batches when + * incremental refresh is enabled. + * + * This is the shared entry point used by both the manual refresh + * (refresh_continuous_aggregate) and the continuous aggregate refresh policy. + * + * For a normal refresh, batching is driven by the invalidation logs, so it only + * produces batches for the regions that actually need to be refreshed. A forced + * refresh ignores the invalidation logs and instead batches every bucket-aligned + * chunk of the window that contains data, so the whole window is re-materialized. + * Set buckets_per_batch to 0 for a single atomic pass. + */ +void +continuous_agg_refresh_batched(ContinuousAgg *cagg, InternalTimeRange *refresh_window, + ContinuousAggRefreshContext context, bool extend_last_bucket) +{ + List *refresh_window_list = continuous_agg_split_refresh_window(cagg, + refresh_window, + context.buckets_per_batch, + context.force); + + bool batched = (refresh_window_list != NIL); + if (!batched) + { + /* No batching: refresh the whole window as a single batch */ + refresh_window_list = lappend(refresh_window_list, refresh_window); + } + else + { + /* Batches are already bucket-aligned by the split function */ + switch (context.callctx) + { + case CAGG_REFRESH_POLICY: + context.callctx = CAGG_REFRESH_POLICY_BATCHED; + break; + case CAGG_REFRESH_WINDOW: + context.callctx = CAGG_REFRESH_WINDOW_BATCHED; + break; + default: + break; + } + } + + context.number_of_batches = list_length(refresh_window_list); + + /* + * The list is always built oldest-first. When refresh_newest_first is true we + * iterate from the last element down to the first using index-based access so + * that no reversal copy of the list is needed. + */ + int32 processing_batch = 0; + int32 nbatches = context.number_of_batches; + int32 batch_start = context.refresh_newest_first ? nbatches - 1 : 0; + int32 batch_end = context.refresh_newest_first ? -1 : nbatches; + int32 batch_step = context.refresh_newest_first ? -1 : 1; + bool any_refreshed = false; + for (int32 batch_idx = batch_start; batch_idx != batch_end; batch_idx += batch_step) + { + InternalTimeRange *batch_window = + (InternalTimeRange *) list_nth(refresh_window_list, batch_idx); + + elog(DEBUG1, + "refreshing continuous aggregate \"%s\" from %s to %s", + NameStr(cagg->data.user_view_name), + ts_internal_to_time_string(batch_window->start, batch_window->type), + ts_internal_to_time_string(batch_window->end, batch_window->type)); + + context.processing_batch = ++processing_batch; + + /* extend_last_bucket must only apply to the boundary batch -- the one + * whose window abuts the adjacent policy. For newest-first ordering + * that is batch 1; for oldest-first it is the final batch. + * In non-batched mode (single batch) the one batch is always the boundary. */ + bool apply_extend = + extend_last_bucket && + (context.refresh_newest_first ? processing_batch == 1 : + processing_batch == context.number_of_batches); + + any_refreshed |= continuous_agg_refresh_internal(cagg, + batch_window, + context, + !batched, /* bucketing_refresh_window */ + apply_extend); + DEBUG_ERROR_INJECTION(psprintf("cagg_policy_batch_%d_after_refresh", processing_batch)); + + if (context.max_batches_per_execution > 0 && + processing_batch >= context.max_batches_per_execution && + processing_batch < context.number_of_batches) + { + elog(LOG, + "reached maximum number of batches per execution (%d), batches not processed (%d)", + context.max_batches_per_execution, + context.number_of_batches - processing_batch); + break; + } + } + + if (!any_refreshed) + { + emit_up_to_date_notice(cagg, context); + } +} + /* * Refresh a continuous aggregate across the given window. */ @@ -642,6 +746,17 @@ continuous_agg_refresh(PG_FUNCTION_ARGS) cagg = cagg_get_by_relid_or_fail(cagg_relid); refresh_window.type = cagg->partition_type; + /* + * Check ownership up front, before any work that touches the source + * hypertable (e.g. computing the batches for an incremental refresh). + */ + if (!object_ownercheck(RelationRelationId, cagg_relid, GetUserId())) + { + aclcheck_error(ACLCHECK_NOT_OWNER, + get_relkind_objtype(get_rel_relkind(cagg_relid)), + get_rel_name(cagg_relid)); + } + if (!PG_ARGISNULL(1)) { refresh_window.start = ts_time_value_from_arg(PG_GETARG_DATUM(1), @@ -653,6 +768,7 @@ continuous_agg_refresh(PG_FUNCTION_ARGS) { /* get min time for a cagg depending of the primary partition type */ refresh_window.start = cagg_get_time_min(cagg); + refresh_window.start_isnull = true; } if (!PG_ARGISNULL(2)) @@ -665,43 +781,83 @@ continuous_agg_refresh(PG_FUNCTION_ARGS) else { refresh_window.end = ts_time_get_noend_or_max(refresh_window.type); + refresh_window.end_isnull = true; } - ContinuousAggRefreshContext context = { .callctx = CAGG_REFRESH_WINDOW }; - continuous_agg_refresh_internal(cagg, - &refresh_window, - context, - PG_ARGISNULL(1), - PG_ARGISNULL(2), - true, - force, - false /*extend_last_bucket*/); + /* + * Manual refreshes batch by default (DEFAULT_BUCKETS_PER_BATCH), matching the + * continuous aggregate policy. Callers can override through the options JSONB; + * set buckets_per_batch to 0 to force a single-pass atomic refresh. + */ + int32 buckets_per_batch = DEFAULT_BUCKETS_PER_BATCH; + int32 max_batches_per_execution = 0; + bool refresh_newest_first = DEFAULT_REFRESH_NEWEST_FIRST; - PG_RETURN_VOID(); -} + if (!PG_ARGISNULL(4)) + { + Jsonb *options = PG_GETARG_JSONB_P(4); + bool found; -static void -emit_up_to_date_notice(const ContinuousAgg *cagg, const ContinuousAggRefreshContext context) -{ - switch (context.callctx) + int32 v = ts_jsonb_get_int32_field(options, POL_REFRESH_CONF_KEY_BUCKETS_PER_BATCH, &found); + if (found) + { + buckets_per_batch = v; + } + + v = ts_jsonb_get_int32_field(options, + POL_REFRESH_CONF_KEY_MAX_BATCHES_PER_EXECUTION, + &found); + if (found) + { + max_batches_per_execution = v; + } + + bool b = + ts_jsonb_get_bool_field(options, POL_REFRESH_CONF_KEY_REFRESH_NEWEST_FIRST, &found); + if (found) + { + refresh_newest_first = b; + } + } + + if (buckets_per_batch < 0) { - case CAGG_REFRESH_WINDOW: - case CAGG_REFRESH_CREATION: - elog(NOTICE, - "continuous aggregate \"%s\" is already up-to-date", - NameStr(cagg->data.user_view_name)); - break; - case CAGG_REFRESH_POLICY: - case CAGG_REFRESH_POLICY_BATCHED: - break; + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid buckets per batch"), + errdetail("buckets_per_batch: %d", buckets_per_batch), + errhint("The buckets per batch should be greater than or equal to zero."))); + } + + if (max_batches_per_execution < 0) + { + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid max batches per execution"), + errdetail("max_batches_per_execution: %d", max_batches_per_execution), + errhint( + "The max batches per execution should be greater than or equal to zero."))); } + + ContinuousAggRefreshContext context = { + .callctx = CAGG_REFRESH_WINDOW, + .buckets_per_batch = buckets_per_batch, + .max_batches_per_execution = max_batches_per_execution, + .refresh_newest_first = refresh_newest_first, + .force = force, + }; + + continuous_agg_refresh_batched(cagg, &refresh_window, context, false /*extend_last_bucket*/); + DEBUG_WAITPOINT("after_cagg_refresh_window"); + + PG_RETURN_VOID(); } static bool process_cagg_invalidations_and_refresh(const ContinuousAgg *cagg, const InternalTimeRange *refresh_window, const ContinuousAggRefreshContext context, - bool bucketing_refresh_window, bool force) + bool bucketing_refresh_window) { /* Lock the continuous aggregate's catalog table entry to protect against concurrent refreshes * on the same cagg processing the cagg invalidation logs for that CAgg. @@ -721,7 +877,7 @@ process_cagg_invalidations_and_refresh(const ContinuousAgg *cagg, DEBUG_WAITPOINT("after_process_cagg_invalidations_for_refresh_lock"); InvalidationStore *invalidations = - collect_and_delete_cagg_invalidations_in_window(cagg, refresh_window, force); + collect_and_delete_cagg_invalidations_in_window(cagg, refresh_window, context.force); if (invalidations != NULL) { @@ -840,12 +996,11 @@ rollback_and_error(const ContinuousAgg *cagg, CaggRefreshSpiContext *cagg_spi_ct ThrowErrorData(edata); } -void +bool continuous_agg_refresh_internal(const ContinuousAgg *cagg_arg, const InternalTimeRange *refresh_window_arg, - const ContinuousAggRefreshContext context, const bool start_isnull, - const bool end_isnull, bool bucketing_refresh_window, bool force, - bool extend_last_bucket) + const ContinuousAggRefreshContext context, + bool bucketing_refresh_window, bool extend_last_bucket) { const ContinuousAgg *volatile cagg = cagg_arg; int32 mat_id = cagg->data.mat_hypertable_id; @@ -862,7 +1017,8 @@ continuous_agg_refresh_internal(const ContinuousAgg *cagg_arg, continuous_agg_refresh_spi_setup_and_connect(&cagg_spi_ctx); /* No bucketing when open ended */ - if (bucketing_refresh_window && !(start_isnull && end_isnull)) + if (bucketing_refresh_window && + !(refresh_window_arg->start_isnull && refresh_window_arg->end_isnull)) { refresh_window = compute_inscribed_bucketed_refresh_window(refresh_window_arg, cagg->bucket_function); @@ -875,7 +1031,7 @@ continuous_agg_refresh_internal(const ContinuousAgg *cagg_arg, * bucket We don't need to do this when the CAgg is created WITH DATA, or manually * refreshed */ - if (extend_last_bucket && !(start_isnull && end_isnull)) + if (extend_last_bucket && !(refresh_window_arg->start_isnull && refresh_window_arg->end_isnull)) { refresh_window.end = cagg_next_bucket_start(refresh_window.end, refresh_window.type, cagg->bucket_function); @@ -1028,8 +1184,7 @@ continuous_agg_refresh_internal(const ContinuousAgg *cagg_arg, refreshed = process_cagg_invalidations_and_refresh(cagg, &refresh_window, context, - bucketing_refresh_window, - force); + bucketing_refresh_window); DEBUG_WAITPOINT("after_process_cagg_materializations"); } @@ -1050,11 +1205,7 @@ continuous_agg_refresh_internal(const ContinuousAgg *cagg_arg, if (edata) { rollback_and_error(cagg, &cagg_spi_ctx, (ErrorData *) edata); - return; - } - if (!refreshed) - { - emit_up_to_date_notice(cagg, context); + return false; } cleanup_before_cagg_refresh_exit(cagg, &cagg_spi_ctx); @@ -1065,6 +1216,8 @@ continuous_agg_refresh_internal(const ContinuousAgg *cagg_arg, { elog(ERROR, "SPI_finish failed: %s", SPI_result_code_string(rc)); } + + return refreshed; } static void @@ -1085,7 +1238,7 @@ debug_refresh_window(const ContinuousAgg *cagg, const InternalTimeRange *refresh List * continuous_agg_split_refresh_window(ContinuousAgg *cagg, InternalTimeRange *original_refresh_window, - int32 buckets_per_batch) + int32 buckets_per_batch, bool force) { /* Do not produce batches when the number of buckets per batch is zero (disabled) */ if (buckets_per_batch == 0) @@ -1224,42 +1377,61 @@ continuous_agg_split_refresh_window(ContinuousAgg *cagg, InternalTimeRange *orig " ORDER BY 1 ASC"; List *refresh_window_list = NIL; - int res; MemoryContext oldcontext = CurrentMemoryContext; - if (SPI_connect() != SPI_OK_CONNECT) - { - elog(ERROR, "could not connect to SPI"); - } + int inval_count; + TupleDesc inval_tupdesc = NULL; + int save_nestlevel = 0; + bool spi_connected = false; - int save_nestlevel = NewGUCNestLevel(); - RestrictSearchPath(); + /* + * A forced refresh must re-materialize the whole window regardless of the + * invalidation logs. We model that as a single synthetic invalidation + * spanning the entire refresh window and skip the invalidation-log query + * altogether. + */ + if (!force) + { + if (SPI_connect() != SPI_OK_CONNECT) + { + elog(ERROR, "could not connect to SPI"); + } + spi_connected = true; + + save_nestlevel = NewGUCNestLevel(); + RestrictSearchPath(); + + Oid inval_types[] = { INT4OID, INT4OID, INT8OID, INT8OID, INT8OID }; + Datum inval_values[] = { Int32GetDatum(cagg->data.mat_hypertable_id), + Int32GetDatum(ht->fd.id), + Int64GetDatum(CAGG_INVALIDATION_WRONG_GREATEST_VALUE), + Int64GetDatum(refresh_window.start), + Int64GetDatum(refresh_window.end) }; + char inval_nulls[] = { false, false, false, false, false }; + + int res = SPI_execute_with_args(inval_query, + 5, + inval_types, + inval_values, + inval_nulls, + true /* read_only */, + 0); + if (res < 0) + { + elog(ERROR, "%s: could not fetch invalidation ranges for cagg refresh", __func__); + } - Oid inval_types[] = { INT4OID, INT4OID, INT8OID, INT8OID, INT8OID }; - Datum inval_values[] = { Int32GetDatum(cagg->data.mat_hypertable_id), - Int32GetDatum(ht->fd.id), - Int64GetDatum(CAGG_INVALIDATION_WRONG_GREATEST_VALUE), - Int64GetDatum(refresh_window.start), - Int64GetDatum(refresh_window.end) }; - char inval_nulls[] = { false, false, false, false, false }; - - res = SPI_execute_with_args(inval_query, - 5, - inval_types, - inval_values, - inval_nulls, - true /* read_only */, - 0); - if (res < 0) + Assert(SPI_processed <= INT_MAX); + inval_count = (int) SPI_processed; + Assert(SPI_tuptable != NULL); + inval_tupdesc = SPI_tuptable->tupdesc; + } + else { - elog(ERROR, "%s: could not fetch invalidation ranges for cagg refresh", __func__); + /* Single synthetic invalidation covering the whole refresh window. */ + inval_count = 1; } - Assert(SPI_processed <= INT_MAX); - int inval_count = (int) SPI_processed; - Assert(SPI_tuptable != NULL); - TupleDesc inval_tupdesc = SPI_tuptable->tupdesc; - /* * Open a streaming catalog scan for dimension slices overlapping the refresh * window. @@ -1312,7 +1484,15 @@ continuous_agg_split_refresh_window(ContinuousAgg *cagg, InternalTimeRange *orig int inval_idx = 0; int64 inval_low = 0, inval_high = 0; - if (inval_idx < inval_count) + if (force) + { + /* Synthetic invalidation spanning the whole window. It stays active for + * the entire merge loop (inval_high == window end, so it is never + * advanced past), making every data-backed chunk eligible for a batch. */ + inval_low = refresh_window.start; + inval_high = refresh_window.end; + } + else if (inval_idx < inval_count) { bool isnull; inval_low = @@ -1413,13 +1593,16 @@ continuous_agg_split_refresh_window(ContinuousAgg *cagg, InternalTimeRange *orig ts_scan_iterator_close(&dim_it); - /* Done with SPI */ - /* Restore search_path */ - AtEOXact_GUC(false, save_nestlevel); - res = SPI_finish(); - if (res != SPI_OK_FINISH) + /* Done with SPI (only used for the invalidation-log query in the non-forced + * path; a forced refresh never connects). Restore search_path. */ + if (spi_connected) { - elog(ERROR, "SPI_finish failed: %s", SPI_result_code_string(res)); + AtEOXact_GUC(false, save_nestlevel); + int res = SPI_finish(); + if (res != SPI_OK_FINISH) + { + elog(ERROR, "SPI_finish failed: %s", SPI_result_code_string(res)); + } } /* diff --git a/tsl/src/continuous_aggs/refresh.h b/tsl/src/continuous_aggs/refresh.h index a10e7f0a17b..8540ffbdf82 100644 --- a/tsl/src/continuous_aggs/refresh.h +++ b/tsl/src/continuous_aggs/refresh.h @@ -12,16 +12,22 @@ #include "materialize.h" #include "ts_catalog/continuous_agg.h" +/* Default buckets per batch for incremental refresh */ +#define DEFAULT_BUCKETS_PER_BATCH 10 +/* Default refresh newest first */ +#define DEFAULT_REFRESH_NEWEST_FIRST true + extern Datum continuous_agg_refresh(PG_FUNCTION_ARGS); -extern void continuous_agg_refresh_internal(const ContinuousAgg *cagg_arg, +extern void continuous_agg_refresh_batched(ContinuousAgg *cagg, InternalTimeRange *refresh_window, + ContinuousAggRefreshContext context, + bool extend_last_bucket); +extern bool continuous_agg_refresh_internal(const ContinuousAgg *cagg_arg, const InternalTimeRange *refresh_window, const ContinuousAggRefreshContext context, - const bool start_isnull, const bool end_isnull, - bool bucketing_refresh_window, bool force, - bool extend_last_bucket); + bool bucketing_refresh_window, bool extend_last_bucket); extern List *continuous_agg_split_refresh_window(ContinuousAgg *cagg, InternalTimeRange *original_refresh_window, - int32 buckets_per_batch); + int32 buckets_per_batch, bool force); InternalTimeRange compute_circumscribed_bucketed_refresh_window(const InternalTimeRange *const refresh_window, const ContinuousAggBucketFunction *bucket_function); diff --git a/tsl/test/expected/cagg-15.out b/tsl/test/expected/cagg-15.out index 9abe697c706..9691854b5d5 100644 --- a/tsl/test/expected/cagg-15.out +++ b/tsl/test/expected/cagg-15.out @@ -1445,7 +1445,6 @@ insert into raw_data select '2000-05-01 00:00+0','Q3', 0, 0; \set ON_ERROR_STOP 0 CALL refresh_continuous_aggregate('search_query_count_3', NULL, '2000-06-01 00:00+0'::timestamptz); CALL refresh_continuous_aggregate('search_query_count_3', '2000-05-01 00:00+0'::timestamptz, '2000-06-01 00:00+0'::timestamptz); -NOTICE: continuous aggregate "search_query_count_3" is already up-to-date \set ON_ERROR_STOP 1 --insert row insert into raw_data select '2001-05-10 00:00+0','Q3', 100, 100; @@ -1471,8 +1470,9 @@ WHERE materialization_id = :'MAT_HTID' ORDER BY 1, 2,3; materialization_id | lowest_modified_value | greatest_modified_value --------------------+-----------------------+------------------------- 41 | -9223372036854775808 | -210866803200000001 - 41 | 959817600000000 | 988675199999999 - 41 | 991353600000000 | 9223372036854775807 + 41 | 947376060000000 | 956447999999999 + 41 | 959817600000000 | 988847999999999 + 41 | 990144000000000 | 9223372036854775807 SELECT * from search_query_count_3 WHERE bucket > '2001-01-01' diff --git a/tsl/test/expected/cagg-16.out b/tsl/test/expected/cagg-16.out index 0f6cde69817..7130e53a3e9 100644 --- a/tsl/test/expected/cagg-16.out +++ b/tsl/test/expected/cagg-16.out @@ -1445,7 +1445,6 @@ insert into raw_data select '2000-05-01 00:00+0','Q3', 0, 0; \set ON_ERROR_STOP 0 CALL refresh_continuous_aggregate('search_query_count_3', NULL, '2000-06-01 00:00+0'::timestamptz); CALL refresh_continuous_aggregate('search_query_count_3', '2000-05-01 00:00+0'::timestamptz, '2000-06-01 00:00+0'::timestamptz); -NOTICE: continuous aggregate "search_query_count_3" is already up-to-date \set ON_ERROR_STOP 1 --insert row insert into raw_data select '2001-05-10 00:00+0','Q3', 100, 100; @@ -1471,8 +1470,9 @@ WHERE materialization_id = :'MAT_HTID' ORDER BY 1, 2,3; materialization_id | lowest_modified_value | greatest_modified_value --------------------+-----------------------+------------------------- 41 | -9223372036854775808 | -210866803200000001 - 41 | 959817600000000 | 988675199999999 - 41 | 991353600000000 | 9223372036854775807 + 41 | 947376060000000 | 956447999999999 + 41 | 959817600000000 | 988847999999999 + 41 | 990144000000000 | 9223372036854775807 SELECT * from search_query_count_3 WHERE bucket > '2001-01-01' diff --git a/tsl/test/expected/cagg-17.out b/tsl/test/expected/cagg-17.out index 0f6cde69817..7130e53a3e9 100644 --- a/tsl/test/expected/cagg-17.out +++ b/tsl/test/expected/cagg-17.out @@ -1445,7 +1445,6 @@ insert into raw_data select '2000-05-01 00:00+0','Q3', 0, 0; \set ON_ERROR_STOP 0 CALL refresh_continuous_aggregate('search_query_count_3', NULL, '2000-06-01 00:00+0'::timestamptz); CALL refresh_continuous_aggregate('search_query_count_3', '2000-05-01 00:00+0'::timestamptz, '2000-06-01 00:00+0'::timestamptz); -NOTICE: continuous aggregate "search_query_count_3" is already up-to-date \set ON_ERROR_STOP 1 --insert row insert into raw_data select '2001-05-10 00:00+0','Q3', 100, 100; @@ -1471,8 +1470,9 @@ WHERE materialization_id = :'MAT_HTID' ORDER BY 1, 2,3; materialization_id | lowest_modified_value | greatest_modified_value --------------------+-----------------------+------------------------- 41 | -9223372036854775808 | -210866803200000001 - 41 | 959817600000000 | 988675199999999 - 41 | 991353600000000 | 9223372036854775807 + 41 | 947376060000000 | 956447999999999 + 41 | 959817600000000 | 988847999999999 + 41 | 990144000000000 | 9223372036854775807 SELECT * from search_query_count_3 WHERE bucket > '2001-01-01' diff --git a/tsl/test/expected/cagg-18.out b/tsl/test/expected/cagg-18.out index 0f6cde69817..7130e53a3e9 100644 --- a/tsl/test/expected/cagg-18.out +++ b/tsl/test/expected/cagg-18.out @@ -1445,7 +1445,6 @@ insert into raw_data select '2000-05-01 00:00+0','Q3', 0, 0; \set ON_ERROR_STOP 0 CALL refresh_continuous_aggregate('search_query_count_3', NULL, '2000-06-01 00:00+0'::timestamptz); CALL refresh_continuous_aggregate('search_query_count_3', '2000-05-01 00:00+0'::timestamptz, '2000-06-01 00:00+0'::timestamptz); -NOTICE: continuous aggregate "search_query_count_3" is already up-to-date \set ON_ERROR_STOP 1 --insert row insert into raw_data select '2001-05-10 00:00+0','Q3', 100, 100; @@ -1471,8 +1470,9 @@ WHERE materialization_id = :'MAT_HTID' ORDER BY 1, 2,3; materialization_id | lowest_modified_value | greatest_modified_value --------------------+-----------------------+------------------------- 41 | -9223372036854775808 | -210866803200000001 - 41 | 959817600000000 | 988675199999999 - 41 | 991353600000000 | 9223372036854775807 + 41 | 947376060000000 | 956447999999999 + 41 | 959817600000000 | 988847999999999 + 41 | 990144000000000 | 9223372036854775807 SELECT * from search_query_count_3 WHERE bucket > '2001-01-01' diff --git a/tsl/test/expected/cagg_direct_compress.out b/tsl/test/expected/cagg_direct_compress.out index d037efd8685..4e370218cc3 100644 --- a/tsl/test/expected/cagg_direct_compress.out +++ b/tsl/test/expected/cagg_direct_compress.out @@ -26,7 +26,8 @@ SELECT FROM conditions GROUP BY 1, 2 WITH NO DATA; -CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL); +-- Setting buckets_per_batch to a high value to bypass "disabling direct compress because of too small batch size" situation +CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_hourly') chunk; chunk_status_text ------------------- @@ -38,7 +39,7 @@ ALTER MATERIALIZED VIEW conditions_hourly SET (timescaledb.compress); NOTICE: defaulting compress_orderby to bucket,device_id -- Enable direct compress on cagg refresh SET timescaledb.enable_direct_compress_on_cagg_refresh TO on; -CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL); +CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_hourly') chunk; chunk_status_text ------------------- @@ -48,7 +49,7 @@ SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks INSERT INTO conditions SELECT t, d::text, 1, 1 FROM generate_series('2025-12-15 00:00:00+00'::timestamptz - interval '1 year', '2025-12-15 00:00:00+00'::timestamptz, interval '1 hour') AS t, generate_series(1, 10) AS d; SET timescaledb.enable_direct_compress_on_cagg_refresh TO off; -CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL); +CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_hourly') chunk; chunk_status_text ---------------------- @@ -71,7 +72,7 @@ SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks INSERT INTO conditions SELECT t, d::text, 1, 1 FROM generate_series('2025-12-15 00:00:00+00'::timestamptz - interval '1 year', '2025-12-15 00:00:00+00'::timestamptz, interval '1 hour') AS t, generate_series(1, 10) AS d; SET timescaledb.enable_direct_compress_on_cagg_refresh TO on; -CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL); +CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_hourly') chunk; chunk_status_text ------------------- @@ -98,14 +99,14 @@ INSERT INTO conditions SELECT t, d::text, 1, 1 FROM generate_series('2025-12-15 00:00:00+00'::timestamptz - interval '1 year', '2025-12-15 00:00:00+00'::timestamptz, interval '1 hour') AS t, generate_series(1, 10) AS d; SET timescaledb.enable_direct_compress_on_cagg_refresh TO on; -- Refresh the base CAgg -CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL); +CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_hourly') chunk; chunk_status_text ------------------- {COMPRESSED} -- Refresh the hierarchical CAgg -CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL); +CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_daily') chunk; chunk_status_text ------------------- @@ -115,24 +116,24 @@ SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks INSERT INTO conditions SELECT t, d::text, 1, 1 FROM generate_series('2025-12-15 00:00:00+00'::timestamptz - interval '1 year', '2025-12-15 00:00:00+00'::timestamptz, interval '1 hour') AS t, generate_series(1, 10) AS d; -- Refresh the base CAgg -CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL); +CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_hourly') chunk; chunk_status_text ------------------- {COMPRESSED} -- Refreshing again the base CAgg is a no-op since everything is up to date -CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL); +CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); NOTICE: continuous aggregate "conditions_hourly" is already up-to-date -- Refresh the hierarchical CAgg with invalidations procuded by the base CAgg -CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL); +CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_daily') chunk; chunk_status_text ------------------- {COMPRESSED} -- Refreshing again the hierarchical CAgg is a no-op since everything is up to date -CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL); +CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); NOTICE: continuous aggregate "conditions_daily" is already up-to-date -- Tests with custom segmentby and orderby CREATE MATERIALIZED VIEW conditions_weekly @@ -148,7 +149,7 @@ FROM conditions GROUP BY 1, 2, 3 WITH NO DATA; ALTER MATERIALIZED VIEW conditions_weekly SET (timescaledb.compress_segmentby = 'device_id, location_id', timescaledb.compress_orderby = 'max, min, bucket DESC'); -CALL refresh_continuous_aggregate('conditions_weekly', NULL, NULL); +CALL refresh_continuous_aggregate('conditions_weekly', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_weekly') chunk; chunk_status_text ------------------- diff --git a/tsl/test/expected/cagg_invalidation.out b/tsl/test/expected/cagg_invalidation.out index d7930c60e13..b41fb41f1e6 100644 --- a/tsl/test/expected/cagg_invalidation.out +++ b/tsl/test/expected/cagg_invalidation.out @@ -1687,6 +1687,13 @@ SET timezone = 'UTC'; --Do the first refresh and check materialization invalidation log call refresh_continuous_aggregate ('test_cagg','2023-12-29 15:00:00', '2026-01-28 15:00:00'); call refresh_continuous_aggregate ('test_cagg_1d_offset','2023-12-29 18:00:00', '2024-01-01 18:00:00'); +-- test_cagg uses 1-month buckets with buckets_per_batch=10. The inscribed window +-- [2024-01-01, 2026-01-01) spans 24 months, producing two batches: +-- [2024-01-01, 2024-11-01) and [2024-11-01, 2025-09-01). +-- Data only exists through Dec 2024, so the split function finds no chunks past +-- Dec 2024 and stops at Sep 2025 (end of batch 2). The range [2025-09-01, 2026-01-01) +-- has no data and is left unprocessed, so the upper residual in the mat_inval_log +-- is 2025-09-01 rather than 2026-01-01 as it would be with single-pass refresh. SELECT materialization_id, _timescaledb_functions.to_timestamp(lowest_modified_value) as low, _timescaledb_functions.to_timestamp(greatest_modified_value) as high @@ -1698,7 +1705,7 @@ ORDER BY low; materialization_id | low | high --------------------+------------------------+------------------------------- 29 | -infinity | 2023-12-31 23:59:59.999999+00 - 29 | 2026-01-01 00:00:00+00 | infinity + 29 | 2025-09-01 00:00:00+00 | infinity SELECT CASE WHEN lowest_modified_value <= _timescaledb_functions.get_internal_time_min('timestamptz'::regtype) @@ -1720,9 +1727,10 @@ ORDER BY lowest_modified_value, greatest_modified_value; -infinity | 2023-12-29 17:59:59.999999+00 2024-01-01 18:00:00+00 | infinity ---now do the same refresh again, it should say the cagg is already up to date +-- Refresh the same range again, [2025-09-01 ─ 2026-01-01) was left. +-- This time refresh will go into the single batch path, so it will process the whole range. CALL refresh_continuous_aggregate ('test_cagg','2023-12-29 15:00:00', '2026-01-28 15:00:00'); -NOTICE: continuous aggregate "test_cagg" is already up-to-date +--Do the same refresh once again, it should say the cagg is already up to date CALL refresh_continuous_aggregate ('test_cagg','2023-12-29 15:00:00', '2026-01-28 15:00:00'); NOTICE: continuous aggregate "test_cagg" is already up-to-date --Insert data to test that invalidation is moved correctly from hypertable invalidation log @@ -1792,7 +1800,8 @@ WHERE hypertable_id IN ( 2026-01-01 00:00:00+00 INSERT INTO test_data values ('2026-01-05 00:00:00', 1); -CALL refresh_continuous_aggregate ('test_cagg_1d_offset','2023-12-29 15:00:00', NULL); +-- Setting buckets_per_batch to 0 as the intention is to test the capping when windew end is set to NULL, no need for batching. +CALL refresh_continuous_aggregate ('test_cagg_1d_offset','2023-12-29 15:00:00', NULL, options => '{"buckets_per_batch": 0}'::jsonb); --should be at the 18th hour SELECT _timescaledb_functions.to_timestamp(watermark) as invalidation_threshold FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold @@ -1860,10 +1869,11 @@ WHERE hypertable_id = ( -- Now refresh cagg_4hrs with NULL,NULL. -- cagg_4hrs computes its own threshold = 04:00, but stored threshold = 06:00 > 04:00, -- so the stored (misaligned) value is used but capped to the start of the current bucket of cagg_4hrs, --- which is 2020-01-01 04:00 UTC. +-- which is 2020-01-01 04:00 UTC. Disabling incremental refresh to preseve the isolate the test intention. SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_4hrs', NULL, NULL); -LOG: statement: CALL refresh_continuous_aggregate('cagg_4hrs', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4hrs', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +LOG: statement: CALL refresh_continuous_aggregate('cagg_4hrs', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +DEBUG: refreshing continuous aggregate "cagg_4hrs" from 4714-11-24 00:00:00+00 BC to infinity DEBUG: hypertable 31 existing watermark >= new invalidation threshold 1577858400000000 1577851200000000 DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4hrs" in window [ 4714-11-24 00:00:00+00 BC, 2020-01-01 04:00:00+00 ] LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_32" diff --git a/tsl/test/expected/cagg_invalidation_variable_bucket.out b/tsl/test/expected/cagg_invalidation_variable_bucket.out index 76d4d8e1070..32e12a35f2b 100644 --- a/tsl/test/expected/cagg_invalidation_variable_bucket.out +++ b/tsl/test/expected/cagg_invalidation_variable_bucket.out @@ -273,7 +273,8 @@ SELECT ts, 1.0 FROM generate_series('2025-03-30 00:00:00'::timestamptz, '2025-03-31 23:59:59.999999'::timestamptz, '1 hour'::interval) ts; -CALL refresh_continuous_aggregate('cagg_dst_daily', '2025-03-01 00:00:00', '2025-05-01 00:00:00'); +-- Disabling incremental refresh as it's not the focus of the test and it's easier to read the test output without it. +CALL refresh_continuous_aggregate('cagg_dst_daily', '2025-03-01 00:00:00', '2025-05-01 00:00:00', options => '{"buckets_per_batch": 0}'::jsonb); -- March 30 should have 23 hours SELECT bucket, cnt FROM cagg_dst_daily ORDER BY bucket; @@ -292,7 +293,7 @@ FROM generate_series('2025-10-26 00:00:00'::timestamptz, '2025-10-27 23:59:59.999999'::timestamptz, '1 hour'::interval) ts; -- Wide window to cover all DST-shifted buckets -CALL refresh_continuous_aggregate('cagg_dst_daily', '2025-10-01 00:00:00', '2026-12-01 00:00:00'); +CALL refresh_continuous_aggregate('cagg_dst_daily', '2025-10-01 00:00:00', '2026-12-01 00:00:00', options => '{"buckets_per_batch": 0}'::jsonb); -- October bucket should have extra hour (25-hour day on Oct 26) SELECT bucket, cnt FROM cagg_dst_daily WHERE bucket >= '2025-10-01 00:00:00' AND bucket < '2026-01-01 00:00:00' @@ -305,7 +306,7 @@ ORDER BY bucket; -- Insert near the fall-back boundary INSERT INTO dst_data VALUES ('2025-10-26 01:00:00', 888.0); -- 2:00 AM Europe/Berlin (after fall-back) INSERT INTO dst_data VALUES ('2025-10-26 00:30:00', 777.0); -- 2:30 AM Europe/Berlin (before fall-back) -CALL refresh_continuous_aggregate('cagg_dst_daily', '2025-09-01 00:00:00', '2026-02-01 00:00:00'); +CALL refresh_continuous_aggregate('cagg_dst_daily', '2025-09-01 00:00:00', '2026-02-01 00:00:00', options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_inval_log WHERE cagg_name = 'cagg_dst_daily'; cagg_name | inval_start | inval_end ----------------+------------------------+------------------------------- @@ -675,7 +676,7 @@ SELECT bucket, cnt FROM cagg_origin_tz ORDER BY bucket; SELECT * FROM cagg_inval_log WHERE cagg_name = 'cagg_origin_tz'; cagg_name | inval_start | inval_end ----------------+------------------------+------------------------------- - cagg_origin_tz | -infinity | 2025-02-01 04:59:59.999999+01 + cagg_origin_tz | -infinity | 2025-03-12 04:59:59.999999+01 cagg_origin_tz | 2025-03-31 05:00:00+02 | infinity SET timezone TO 'UTC'; diff --git a/tsl/test/expected/cagg_policy_run.out b/tsl/test/expected/cagg_policy_run.out index 7ecfdb487bd..229e7f576e3 100644 --- a/tsl/test/expected/cagg_policy_run.out +++ b/tsl/test/expected/cagg_policy_run.out @@ -198,19 +198,19 @@ ORDER BY range_start; SELECT * FROM measurements_chunks; chunk_name | range_start | is_compressed -------------------+-------------+--------------- - _hyper_9_33_chunk | 01-01-2025 | f - _hyper_9_34_chunk | 01-02-2025 | f - _hyper_9_35_chunk | 01-03-2025 | f - _hyper_9_32_chunk | 01-04-2025 | f - _hyper_9_29_chunk | 01-05-2025 | f - _hyper_9_37_chunk | 01-06-2025 | f - _hyper_9_31_chunk | 01-07-2025 | f - _hyper_9_38_chunk | 01-08-2025 | f - _hyper_9_40_chunk | 01-09-2025 | f - _hyper_9_39_chunk | 01-10-2025 | f - _hyper_9_30_chunk | 01-11-2025 | f - _hyper_9_36_chunk | 01-12-2025 | f - _hyper_9_41_chunk | 01-13-2025 | f + _hyper_9_35_chunk | 01-01-2025 | f + _hyper_9_36_chunk | 01-02-2025 | f + _hyper_9_37_chunk | 01-03-2025 | f + _hyper_9_34_chunk | 01-04-2025 | f + _hyper_9_32_chunk | 01-05-2025 | f + _hyper_9_38_chunk | 01-06-2025 | f + _hyper_9_33_chunk | 01-07-2025 | f + _hyper_9_39_chunk | 01-08-2025 | f + _hyper_9_41_chunk | 01-09-2025 | f + _hyper_9_40_chunk | 01-10-2025 | f + _hyper_9_31_chunk | 01-11-2025 | f + _hyper_9_29_chunk | 01-12-2025 | f + _hyper_9_30_chunk | 01-13-2025 | f -- Mock now() at 2025-01-15. With start_offset = 8 days and end_offset = 4 days -- the refresh window is [2025-01-07, 2025-01-11). Chunks 01-07..01-10 should @@ -233,19 +233,19 @@ CALL run_job(:job_id); SELECT * FROM measurements_chunks; chunk_name | range_start | is_compressed -------------------+-------------+--------------- - _hyper_9_33_chunk | 01-01-2025 | f - _hyper_9_34_chunk | 01-02-2025 | f - _hyper_9_35_chunk | 01-03-2025 | f - _hyper_9_32_chunk | 01-04-2025 | f - _hyper_9_29_chunk | 01-05-2025 | f - _hyper_9_37_chunk | 01-06-2025 | f - _hyper_9_31_chunk | 01-07-2025 | t - _hyper_9_38_chunk | 01-08-2025 | t - _hyper_9_40_chunk | 01-09-2025 | t - _hyper_9_39_chunk | 01-10-2025 | t - _hyper_9_30_chunk | 01-11-2025 | f - _hyper_9_36_chunk | 01-12-2025 | f - _hyper_9_41_chunk | 01-13-2025 | f + _hyper_9_35_chunk | 01-01-2025 | f + _hyper_9_36_chunk | 01-02-2025 | f + _hyper_9_37_chunk | 01-03-2025 | f + _hyper_9_34_chunk | 01-04-2025 | f + _hyper_9_32_chunk | 01-05-2025 | f + _hyper_9_38_chunk | 01-06-2025 | f + _hyper_9_33_chunk | 01-07-2025 | t + _hyper_9_39_chunk | 01-08-2025 | t + _hyper_9_41_chunk | 01-09-2025 | t + _hyper_9_40_chunk | 01-10-2025 | t + _hyper_9_31_chunk | 01-11-2025 | f + _hyper_9_29_chunk | 01-12-2025 | f + _hyper_9_30_chunk | 01-13-2025 | f SELECT delete_job(:job_id); delete_job diff --git a/tsl/test/expected/cagg_query-15.out b/tsl/test/expected/cagg_query-15.out index c07e1ea487d..8c6fb5657d2 100644 --- a/tsl/test/expected/cagg_query-15.out +++ b/tsl/test/expected/cagg_query-15.out @@ -62,7 +62,7 @@ FROM ( select max(timec)as timeval from conditions ) as q; ------------------------------ Sat Nov 03 17:00:00 2018 PDT -CALL refresh_continuous_aggregate('mat_m1', NULL, NULL); +CALL refresh_continuous_aggregate('mat_m1', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); --test first/last create materialized view mat_m2(location, timec, firsth, lasth, maxtemp, mintemp) WITH (timescaledb.continuous, timescaledb.materialized_only=false) @@ -77,7 +77,7 @@ FROM ( select max(timec)as timeval from conditions ) as q; ------------------------------ Sat Nov 03 17:00:00 2018 PDT -CALL refresh_continuous_aggregate('mat_m2', NULL, NULL); +CALL refresh_continuous_aggregate('mat_m2', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); --normal view -- create or replace view regview( location, timec, minl, sumt , sumh) as @@ -1087,7 +1087,7 @@ SET ROLE :ROLE_SUPERUSER; BEGIN; UPDATE _timescaledb_catalog.continuous_aggs_bucket_function SET bucket_func = 'func_does_not_exist()'; -- should error because function does not exist -CALL refresh_continuous_aggregate('cagg_bigint_offset2', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_bigint_offset2', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); psql:include/cagg_query_common.sql:566: ERROR: function "func_does_not_exist()" does not exist ROLLBACK; \set ON_ERROR_STOP 1 @@ -1304,9 +1304,9 @@ SELECT * FROM cagg_4_hours_origin; -- Update the last bucket and re-materialize INSERT INTO temperature values('2020-01-01 23:55:00 PST', 10); -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_4_hours; time_bucket | max ------------------------------+----- @@ -1439,18 +1439,21 @@ SELECT * FROM cagg_4_hours_origin; -- Update materialized data SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -psql:include/cagg_query_common.sql:683: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:683: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:683: DEBUG: refreshing continuous aggregate "cagg_4_hours" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:683: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Thu Jan 02 00:00:00 2020 PST, Thu Jan 02 12:00:00 2020 PST ] psql:include/cagg_query_common.sql:683: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_33" psql:include/cagg_query_common.sql:683: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -psql:include/cagg_query_common.sql:684: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:684: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:684: DEBUG: refreshing continuous aggregate "cagg_4_hours_offset" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:684: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Wed Jan 01 20:30:00 2020 PST, Thu Jan 02 12:30:00 2020 PST ] psql:include/cagg_query_common.sql:684: LOG: deleted 1 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:684: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); -psql:include/cagg_query_common.sql:685: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:685: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:685: DEBUG: refreshing continuous aggregate "cagg_4_hours_origin" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:685: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_origin" in window [ Wed Jan 01 21:00:00 2020 PST, Thu Jan 02 13:00:00 2020 PST ] psql:include/cagg_query_common.sql:685: LOG: deleted 1 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_35" psql:include/cagg_query_common.sql:685: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_35" @@ -1633,9 +1636,9 @@ SELECT time_bucket('4 hour', time, '2000-01-01 01:00:00 PST'::timestamptz), max( -- Test invalidations TRUNCATE temperature; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); INSERT INTO temperature SELECT time, 5 FROM generate_series('2000-01-01 01:00:00 PST'::timestamptz, @@ -1648,8 +1651,9 @@ INSERT INTO temperature values('2020-01-02 01:05:00+01', 2222); INSERT INTO temperature values('2020-01-02 01:35:00+01', 5555); INSERT INTO temperature values('2020-01-02 05:05:00+01', 8888); SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -psql:include/cagg_query_common.sql:725: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:725: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:725: DEBUG: refreshing continuous aggregate "cagg_4_hours" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:725: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577952000000000 psql:include/cagg_query_common.sql:725: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Sat Jan 01 00:00:00 2000 PST, Sun Jan 02 00:00:00 2000 PST ] psql:include/cagg_query_common.sql:725: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_33" @@ -1659,8 +1663,9 @@ psql:include/cagg_query_common.sql:725: DEBUG: continuous aggregate refresh (in psql:include/cagg_query_common.sql:725: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_33" psql:include/cagg_query_common.sql:725: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" psql:include/cagg_query_common.sql:725: DEBUG: hypertable 33 existing watermark >= new watermark 1577995200000000 1577952000000000 -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -psql:include/cagg_query_common.sql:726: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:726: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:726: DEBUG: refreshing continuous aggregate "cagg_4_hours_offset" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:726: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577953800000000 psql:include/cagg_query_common.sql:726: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Sat Jan 01 00:30:00 2000 PST, Sun Jan 02 00:30:00 2000 PST ] psql:include/cagg_query_common.sql:726: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_34" @@ -1670,8 +1675,9 @@ psql:include/cagg_query_common.sql:726: DEBUG: continuous aggregate refresh (in psql:include/cagg_query_common.sql:726: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:726: LOG: inserted 7 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:726: DEBUG: hypertable 34 existing watermark >= new watermark 1577997000000000 1577953800000000 -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); -psql:include/cagg_query_common.sql:727: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:727: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:727: DEBUG: refreshing continuous aggregate "cagg_4_hours_origin" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:727: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577955600000000 psql:include/cagg_query_common.sql:727: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_origin" in window [ Sat Jan 01 01:00:00 2000 PST, Sun Jan 02 01:00:00 2000 PST ] psql:include/cagg_query_common.sql:727: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_35" @@ -1917,7 +1923,7 @@ ORDER BY 1, 2, 3; 33 | 1577995200000000 | 9223372036854775807 33 | 1577995200000000 | 9223372036854775807 -CALL refresh_continuous_aggregate('cagg_4_hours', '2000-01-01 00:00:00'::timestamptz, '2020-12-31 23:59:59'::timestamptz); +CALL refresh_continuous_aggregate('cagg_4_hours', '2000-01-01 00:00:00'::timestamptz, '2020-12-31 23:59:59'::timestamptz, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log WHERE materialization_id IN (SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'cagg_4_hours') @@ -2045,7 +2051,7 @@ SELECT * FROM cagg_int; 50 | 5 100 | 555 -CALL refresh_continuous_aggregate('cagg_int', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_int', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_int; time_bucket | value -------------+------- @@ -2094,7 +2100,7 @@ SELECT * FROM cagg_int_offset; -- the value 100 is part of the already serializ 45 | 30 95 | 555 -CALL refresh_continuous_aggregate('cagg_int_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_int_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_int_offset; time_bucket | value -------------+------- @@ -2149,8 +2155,9 @@ SELECT * FROM cagg_int_offset; -- Check that the refresh is properly aligned INSERT INTO table_int VALUES(114, 0); SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130); -psql:include/cagg_query_common.sql:845: LOG: statement: CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130); +CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:845: LOG: statement: CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:845: DEBUG: refreshing continuous aggregate "cagg_int_offset" from 100 to 130 psql:include/cagg_query_common.sql:845: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_int_offset" in window [ 105, 125 ] psql:include/cagg_query_common.sql:845: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_38" psql:include/cagg_query_common.sql:845: DEBUG: building index "_hyper_38_68_chunk__materialized_hypertable_38_time_bucket_idx" on table "_hyper_38_68_chunk" serially @@ -2369,8 +2376,8 @@ SELECT time_bucket('1 week', time, origin=>'2000-01-02 01:00:00 PST'::timestampt Sun Dec 30 01:00:00 2029 PST | 55555 -- Test refresh -CALL refresh_continuous_aggregate('cagg_1_hour_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_1_week_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_1_hour_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_1_week_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); -- Everything should be now materailized ALTER MATERIALIZED VIEW cagg_1_hour_offset SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW cagg_1_week_offset SET (timescaledb.materialized_only=false); diff --git a/tsl/test/expected/cagg_query-16.out b/tsl/test/expected/cagg_query-16.out index 29cfcf539fa..7db97c7fd4c 100644 --- a/tsl/test/expected/cagg_query-16.out +++ b/tsl/test/expected/cagg_query-16.out @@ -62,7 +62,7 @@ FROM ( select max(timec)as timeval from conditions ) as q; ------------------------------ Sat Nov 03 17:00:00 2018 PDT -CALL refresh_continuous_aggregate('mat_m1', NULL, NULL); +CALL refresh_continuous_aggregate('mat_m1', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); --test first/last create materialized view mat_m2(location, timec, firsth, lasth, maxtemp, mintemp) WITH (timescaledb.continuous, timescaledb.materialized_only=false) @@ -77,7 +77,7 @@ FROM ( select max(timec)as timeval from conditions ) as q; ------------------------------ Sat Nov 03 17:00:00 2018 PDT -CALL refresh_continuous_aggregate('mat_m2', NULL, NULL); +CALL refresh_continuous_aggregate('mat_m2', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); --normal view -- create or replace view regview( location, timec, minl, sumt , sumh) as @@ -1081,7 +1081,7 @@ SET ROLE :ROLE_SUPERUSER; BEGIN; UPDATE _timescaledb_catalog.continuous_aggs_bucket_function SET bucket_func = 'func_does_not_exist()'; -- should error because function does not exist -CALL refresh_continuous_aggregate('cagg_bigint_offset2', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_bigint_offset2', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); psql:include/cagg_query_common.sql:566: ERROR: function "func_does_not_exist()" does not exist ROLLBACK; \set ON_ERROR_STOP 1 @@ -1298,9 +1298,9 @@ SELECT * FROM cagg_4_hours_origin; -- Update the last bucket and re-materialize INSERT INTO temperature values('2020-01-01 23:55:00 PST', 10); -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_4_hours; time_bucket | max ------------------------------+----- @@ -1433,18 +1433,21 @@ SELECT * FROM cagg_4_hours_origin; -- Update materialized data SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -psql:include/cagg_query_common.sql:683: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:683: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:683: DEBUG: refreshing continuous aggregate "cagg_4_hours" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:683: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Thu Jan 02 00:00:00 2020 PST, Thu Jan 02 12:00:00 2020 PST ] psql:include/cagg_query_common.sql:683: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_33" psql:include/cagg_query_common.sql:683: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -psql:include/cagg_query_common.sql:684: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:684: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:684: DEBUG: refreshing continuous aggregate "cagg_4_hours_offset" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:684: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Wed Jan 01 20:30:00 2020 PST, Thu Jan 02 12:30:00 2020 PST ] psql:include/cagg_query_common.sql:684: LOG: deleted 1 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:684: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); -psql:include/cagg_query_common.sql:685: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:685: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:685: DEBUG: refreshing continuous aggregate "cagg_4_hours_origin" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:685: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_origin" in window [ Wed Jan 01 21:00:00 2020 PST, Thu Jan 02 13:00:00 2020 PST ] psql:include/cagg_query_common.sql:685: LOG: deleted 1 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_35" psql:include/cagg_query_common.sql:685: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_35" @@ -1627,9 +1630,9 @@ SELECT time_bucket('4 hour', time, '2000-01-01 01:00:00 PST'::timestamptz), max( -- Test invalidations TRUNCATE temperature; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); INSERT INTO temperature SELECT time, 5 FROM generate_series('2000-01-01 01:00:00 PST'::timestamptz, @@ -1642,8 +1645,9 @@ INSERT INTO temperature values('2020-01-02 01:05:00+01', 2222); INSERT INTO temperature values('2020-01-02 01:35:00+01', 5555); INSERT INTO temperature values('2020-01-02 05:05:00+01', 8888); SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -psql:include/cagg_query_common.sql:725: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:725: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:725: DEBUG: refreshing continuous aggregate "cagg_4_hours" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:725: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577952000000000 psql:include/cagg_query_common.sql:725: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Sat Jan 01 00:00:00 2000 PST, Sun Jan 02 00:00:00 2000 PST ] psql:include/cagg_query_common.sql:725: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_33" @@ -1653,8 +1657,9 @@ psql:include/cagg_query_common.sql:725: DEBUG: continuous aggregate refresh (in psql:include/cagg_query_common.sql:725: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_33" psql:include/cagg_query_common.sql:725: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" psql:include/cagg_query_common.sql:725: DEBUG: hypertable 33 existing watermark >= new watermark 1577995200000000 1577952000000000 -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -psql:include/cagg_query_common.sql:726: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:726: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:726: DEBUG: refreshing continuous aggregate "cagg_4_hours_offset" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:726: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577953800000000 psql:include/cagg_query_common.sql:726: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Sat Jan 01 00:30:00 2000 PST, Sun Jan 02 00:30:00 2000 PST ] psql:include/cagg_query_common.sql:726: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_34" @@ -1664,8 +1669,9 @@ psql:include/cagg_query_common.sql:726: DEBUG: continuous aggregate refresh (in psql:include/cagg_query_common.sql:726: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:726: LOG: inserted 7 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:726: DEBUG: hypertable 34 existing watermark >= new watermark 1577997000000000 1577953800000000 -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); -psql:include/cagg_query_common.sql:727: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:727: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:727: DEBUG: refreshing continuous aggregate "cagg_4_hours_origin" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:727: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577955600000000 psql:include/cagg_query_common.sql:727: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_origin" in window [ Sat Jan 01 01:00:00 2000 PST, Sun Jan 02 01:00:00 2000 PST ] psql:include/cagg_query_common.sql:727: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_35" @@ -1911,7 +1917,7 @@ ORDER BY 1, 2, 3; 33 | 1577995200000000 | 9223372036854775807 33 | 1577995200000000 | 9223372036854775807 -CALL refresh_continuous_aggregate('cagg_4_hours', '2000-01-01 00:00:00'::timestamptz, '2020-12-31 23:59:59'::timestamptz); +CALL refresh_continuous_aggregate('cagg_4_hours', '2000-01-01 00:00:00'::timestamptz, '2020-12-31 23:59:59'::timestamptz, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log WHERE materialization_id IN (SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'cagg_4_hours') @@ -2039,7 +2045,7 @@ SELECT * FROM cagg_int; 50 | 5 100 | 555 -CALL refresh_continuous_aggregate('cagg_int', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_int', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_int; time_bucket | value -------------+------- @@ -2088,7 +2094,7 @@ SELECT * FROM cagg_int_offset; -- the value 100 is part of the already serializ 45 | 30 95 | 555 -CALL refresh_continuous_aggregate('cagg_int_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_int_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_int_offset; time_bucket | value -------------+------- @@ -2143,8 +2149,9 @@ SELECT * FROM cagg_int_offset; -- Check that the refresh is properly aligned INSERT INTO table_int VALUES(114, 0); SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130); -psql:include/cagg_query_common.sql:845: LOG: statement: CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130); +CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:845: LOG: statement: CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:845: DEBUG: refreshing continuous aggregate "cagg_int_offset" from 100 to 130 psql:include/cagg_query_common.sql:845: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_int_offset" in window [ 105, 125 ] psql:include/cagg_query_common.sql:845: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_38" psql:include/cagg_query_common.sql:845: DEBUG: building index "_hyper_38_68_chunk__materialized_hypertable_38_time_bucket_idx" on table "_hyper_38_68_chunk" serially @@ -2363,8 +2370,8 @@ SELECT time_bucket('1 week', time, origin=>'2000-01-02 01:00:00 PST'::timestampt Sun Dec 30 01:00:00 2029 PST | 55555 -- Test refresh -CALL refresh_continuous_aggregate('cagg_1_hour_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_1_week_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_1_hour_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_1_week_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); -- Everything should be now materailized ALTER MATERIALIZED VIEW cagg_1_hour_offset SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW cagg_1_week_offset SET (timescaledb.materialized_only=false); diff --git a/tsl/test/expected/cagg_query-17.out b/tsl/test/expected/cagg_query-17.out index 29cfcf539fa..7db97c7fd4c 100644 --- a/tsl/test/expected/cagg_query-17.out +++ b/tsl/test/expected/cagg_query-17.out @@ -62,7 +62,7 @@ FROM ( select max(timec)as timeval from conditions ) as q; ------------------------------ Sat Nov 03 17:00:00 2018 PDT -CALL refresh_continuous_aggregate('mat_m1', NULL, NULL); +CALL refresh_continuous_aggregate('mat_m1', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); --test first/last create materialized view mat_m2(location, timec, firsth, lasth, maxtemp, mintemp) WITH (timescaledb.continuous, timescaledb.materialized_only=false) @@ -77,7 +77,7 @@ FROM ( select max(timec)as timeval from conditions ) as q; ------------------------------ Sat Nov 03 17:00:00 2018 PDT -CALL refresh_continuous_aggregate('mat_m2', NULL, NULL); +CALL refresh_continuous_aggregate('mat_m2', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); --normal view -- create or replace view regview( location, timec, minl, sumt , sumh) as @@ -1081,7 +1081,7 @@ SET ROLE :ROLE_SUPERUSER; BEGIN; UPDATE _timescaledb_catalog.continuous_aggs_bucket_function SET bucket_func = 'func_does_not_exist()'; -- should error because function does not exist -CALL refresh_continuous_aggregate('cagg_bigint_offset2', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_bigint_offset2', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); psql:include/cagg_query_common.sql:566: ERROR: function "func_does_not_exist()" does not exist ROLLBACK; \set ON_ERROR_STOP 1 @@ -1298,9 +1298,9 @@ SELECT * FROM cagg_4_hours_origin; -- Update the last bucket and re-materialize INSERT INTO temperature values('2020-01-01 23:55:00 PST', 10); -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_4_hours; time_bucket | max ------------------------------+----- @@ -1433,18 +1433,21 @@ SELECT * FROM cagg_4_hours_origin; -- Update materialized data SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -psql:include/cagg_query_common.sql:683: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:683: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:683: DEBUG: refreshing continuous aggregate "cagg_4_hours" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:683: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Thu Jan 02 00:00:00 2020 PST, Thu Jan 02 12:00:00 2020 PST ] psql:include/cagg_query_common.sql:683: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_33" psql:include/cagg_query_common.sql:683: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -psql:include/cagg_query_common.sql:684: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:684: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:684: DEBUG: refreshing continuous aggregate "cagg_4_hours_offset" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:684: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Wed Jan 01 20:30:00 2020 PST, Thu Jan 02 12:30:00 2020 PST ] psql:include/cagg_query_common.sql:684: LOG: deleted 1 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:684: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); -psql:include/cagg_query_common.sql:685: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:685: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:685: DEBUG: refreshing continuous aggregate "cagg_4_hours_origin" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:685: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_origin" in window [ Wed Jan 01 21:00:00 2020 PST, Thu Jan 02 13:00:00 2020 PST ] psql:include/cagg_query_common.sql:685: LOG: deleted 1 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_35" psql:include/cagg_query_common.sql:685: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_35" @@ -1627,9 +1630,9 @@ SELECT time_bucket('4 hour', time, '2000-01-01 01:00:00 PST'::timestamptz), max( -- Test invalidations TRUNCATE temperature; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); INSERT INTO temperature SELECT time, 5 FROM generate_series('2000-01-01 01:00:00 PST'::timestamptz, @@ -1642,8 +1645,9 @@ INSERT INTO temperature values('2020-01-02 01:05:00+01', 2222); INSERT INTO temperature values('2020-01-02 01:35:00+01', 5555); INSERT INTO temperature values('2020-01-02 05:05:00+01', 8888); SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -psql:include/cagg_query_common.sql:725: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:725: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:725: DEBUG: refreshing continuous aggregate "cagg_4_hours" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:725: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577952000000000 psql:include/cagg_query_common.sql:725: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Sat Jan 01 00:00:00 2000 PST, Sun Jan 02 00:00:00 2000 PST ] psql:include/cagg_query_common.sql:725: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_33" @@ -1653,8 +1657,9 @@ psql:include/cagg_query_common.sql:725: DEBUG: continuous aggregate refresh (in psql:include/cagg_query_common.sql:725: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_33" psql:include/cagg_query_common.sql:725: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" psql:include/cagg_query_common.sql:725: DEBUG: hypertable 33 existing watermark >= new watermark 1577995200000000 1577952000000000 -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -psql:include/cagg_query_common.sql:726: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:726: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:726: DEBUG: refreshing continuous aggregate "cagg_4_hours_offset" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:726: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577953800000000 psql:include/cagg_query_common.sql:726: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Sat Jan 01 00:30:00 2000 PST, Sun Jan 02 00:30:00 2000 PST ] psql:include/cagg_query_common.sql:726: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_34" @@ -1664,8 +1669,9 @@ psql:include/cagg_query_common.sql:726: DEBUG: continuous aggregate refresh (in psql:include/cagg_query_common.sql:726: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:726: LOG: inserted 7 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:726: DEBUG: hypertable 34 existing watermark >= new watermark 1577997000000000 1577953800000000 -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); -psql:include/cagg_query_common.sql:727: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:727: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:727: DEBUG: refreshing continuous aggregate "cagg_4_hours_origin" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:727: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577955600000000 psql:include/cagg_query_common.sql:727: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_origin" in window [ Sat Jan 01 01:00:00 2000 PST, Sun Jan 02 01:00:00 2000 PST ] psql:include/cagg_query_common.sql:727: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_35" @@ -1911,7 +1917,7 @@ ORDER BY 1, 2, 3; 33 | 1577995200000000 | 9223372036854775807 33 | 1577995200000000 | 9223372036854775807 -CALL refresh_continuous_aggregate('cagg_4_hours', '2000-01-01 00:00:00'::timestamptz, '2020-12-31 23:59:59'::timestamptz); +CALL refresh_continuous_aggregate('cagg_4_hours', '2000-01-01 00:00:00'::timestamptz, '2020-12-31 23:59:59'::timestamptz, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log WHERE materialization_id IN (SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'cagg_4_hours') @@ -2039,7 +2045,7 @@ SELECT * FROM cagg_int; 50 | 5 100 | 555 -CALL refresh_continuous_aggregate('cagg_int', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_int', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_int; time_bucket | value -------------+------- @@ -2088,7 +2094,7 @@ SELECT * FROM cagg_int_offset; -- the value 100 is part of the already serializ 45 | 30 95 | 555 -CALL refresh_continuous_aggregate('cagg_int_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_int_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_int_offset; time_bucket | value -------------+------- @@ -2143,8 +2149,9 @@ SELECT * FROM cagg_int_offset; -- Check that the refresh is properly aligned INSERT INTO table_int VALUES(114, 0); SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130); -psql:include/cagg_query_common.sql:845: LOG: statement: CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130); +CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:845: LOG: statement: CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:845: DEBUG: refreshing continuous aggregate "cagg_int_offset" from 100 to 130 psql:include/cagg_query_common.sql:845: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_int_offset" in window [ 105, 125 ] psql:include/cagg_query_common.sql:845: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_38" psql:include/cagg_query_common.sql:845: DEBUG: building index "_hyper_38_68_chunk__materialized_hypertable_38_time_bucket_idx" on table "_hyper_38_68_chunk" serially @@ -2363,8 +2370,8 @@ SELECT time_bucket('1 week', time, origin=>'2000-01-02 01:00:00 PST'::timestampt Sun Dec 30 01:00:00 2029 PST | 55555 -- Test refresh -CALL refresh_continuous_aggregate('cagg_1_hour_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_1_week_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_1_hour_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_1_week_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); -- Everything should be now materailized ALTER MATERIALIZED VIEW cagg_1_hour_offset SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW cagg_1_week_offset SET (timescaledb.materialized_only=false); diff --git a/tsl/test/expected/cagg_query-18.out b/tsl/test/expected/cagg_query-18.out index 29cfcf539fa..7db97c7fd4c 100644 --- a/tsl/test/expected/cagg_query-18.out +++ b/tsl/test/expected/cagg_query-18.out @@ -62,7 +62,7 @@ FROM ( select max(timec)as timeval from conditions ) as q; ------------------------------ Sat Nov 03 17:00:00 2018 PDT -CALL refresh_continuous_aggregate('mat_m1', NULL, NULL); +CALL refresh_continuous_aggregate('mat_m1', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); --test first/last create materialized view mat_m2(location, timec, firsth, lasth, maxtemp, mintemp) WITH (timescaledb.continuous, timescaledb.materialized_only=false) @@ -77,7 +77,7 @@ FROM ( select max(timec)as timeval from conditions ) as q; ------------------------------ Sat Nov 03 17:00:00 2018 PDT -CALL refresh_continuous_aggregate('mat_m2', NULL, NULL); +CALL refresh_continuous_aggregate('mat_m2', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); --normal view -- create or replace view regview( location, timec, minl, sumt , sumh) as @@ -1081,7 +1081,7 @@ SET ROLE :ROLE_SUPERUSER; BEGIN; UPDATE _timescaledb_catalog.continuous_aggs_bucket_function SET bucket_func = 'func_does_not_exist()'; -- should error because function does not exist -CALL refresh_continuous_aggregate('cagg_bigint_offset2', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_bigint_offset2', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); psql:include/cagg_query_common.sql:566: ERROR: function "func_does_not_exist()" does not exist ROLLBACK; \set ON_ERROR_STOP 1 @@ -1298,9 +1298,9 @@ SELECT * FROM cagg_4_hours_origin; -- Update the last bucket and re-materialize INSERT INTO temperature values('2020-01-01 23:55:00 PST', 10); -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_4_hours; time_bucket | max ------------------------------+----- @@ -1433,18 +1433,21 @@ SELECT * FROM cagg_4_hours_origin; -- Update materialized data SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -psql:include/cagg_query_common.sql:683: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:683: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:683: DEBUG: refreshing continuous aggregate "cagg_4_hours" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:683: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Thu Jan 02 00:00:00 2020 PST, Thu Jan 02 12:00:00 2020 PST ] psql:include/cagg_query_common.sql:683: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_33" psql:include/cagg_query_common.sql:683: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -psql:include/cagg_query_common.sql:684: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:684: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:684: DEBUG: refreshing continuous aggregate "cagg_4_hours_offset" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:684: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Wed Jan 01 20:30:00 2020 PST, Thu Jan 02 12:30:00 2020 PST ] psql:include/cagg_query_common.sql:684: LOG: deleted 1 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:684: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); -psql:include/cagg_query_common.sql:685: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:685: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:685: DEBUG: refreshing continuous aggregate "cagg_4_hours_origin" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:685: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_origin" in window [ Wed Jan 01 21:00:00 2020 PST, Thu Jan 02 13:00:00 2020 PST ] psql:include/cagg_query_common.sql:685: LOG: deleted 1 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_35" psql:include/cagg_query_common.sql:685: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_35" @@ -1627,9 +1630,9 @@ SELECT time_bucket('4 hour', time, '2000-01-01 01:00:00 PST'::timestamptz), max( -- Test invalidations TRUNCATE temperature; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); INSERT INTO temperature SELECT time, 5 FROM generate_series('2000-01-01 01:00:00 PST'::timestamptz, @@ -1642,8 +1645,9 @@ INSERT INTO temperature values('2020-01-02 01:05:00+01', 2222); INSERT INTO temperature values('2020-01-02 01:35:00+01', 5555); INSERT INTO temperature values('2020-01-02 05:05:00+01', 8888); SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -psql:include/cagg_query_common.sql:725: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:725: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:725: DEBUG: refreshing continuous aggregate "cagg_4_hours" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:725: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577952000000000 psql:include/cagg_query_common.sql:725: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Sat Jan 01 00:00:00 2000 PST, Sun Jan 02 00:00:00 2000 PST ] psql:include/cagg_query_common.sql:725: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_33" @@ -1653,8 +1657,9 @@ psql:include/cagg_query_common.sql:725: DEBUG: continuous aggregate refresh (in psql:include/cagg_query_common.sql:725: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_33" psql:include/cagg_query_common.sql:725: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" psql:include/cagg_query_common.sql:725: DEBUG: hypertable 33 existing watermark >= new watermark 1577995200000000 1577952000000000 -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -psql:include/cagg_query_common.sql:726: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:726: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:726: DEBUG: refreshing continuous aggregate "cagg_4_hours_offset" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:726: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577953800000000 psql:include/cagg_query_common.sql:726: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Sat Jan 01 00:30:00 2000 PST, Sun Jan 02 00:30:00 2000 PST ] psql:include/cagg_query_common.sql:726: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_34" @@ -1664,8 +1669,9 @@ psql:include/cagg_query_common.sql:726: DEBUG: continuous aggregate refresh (in psql:include/cagg_query_common.sql:726: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:726: LOG: inserted 7 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:726: DEBUG: hypertable 34 existing watermark >= new watermark 1577997000000000 1577953800000000 -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); -psql:include/cagg_query_common.sql:727: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:727: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:727: DEBUG: refreshing continuous aggregate "cagg_4_hours_origin" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:727: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577955600000000 psql:include/cagg_query_common.sql:727: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_origin" in window [ Sat Jan 01 01:00:00 2000 PST, Sun Jan 02 01:00:00 2000 PST ] psql:include/cagg_query_common.sql:727: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_35" @@ -1911,7 +1917,7 @@ ORDER BY 1, 2, 3; 33 | 1577995200000000 | 9223372036854775807 33 | 1577995200000000 | 9223372036854775807 -CALL refresh_continuous_aggregate('cagg_4_hours', '2000-01-01 00:00:00'::timestamptz, '2020-12-31 23:59:59'::timestamptz); +CALL refresh_continuous_aggregate('cagg_4_hours', '2000-01-01 00:00:00'::timestamptz, '2020-12-31 23:59:59'::timestamptz, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log WHERE materialization_id IN (SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'cagg_4_hours') @@ -2039,7 +2045,7 @@ SELECT * FROM cagg_int; 50 | 5 100 | 555 -CALL refresh_continuous_aggregate('cagg_int', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_int', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_int; time_bucket | value -------------+------- @@ -2088,7 +2094,7 @@ SELECT * FROM cagg_int_offset; -- the value 100 is part of the already serializ 45 | 30 95 | 555 -CALL refresh_continuous_aggregate('cagg_int_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_int_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_int_offset; time_bucket | value -------------+------- @@ -2143,8 +2149,9 @@ SELECT * FROM cagg_int_offset; -- Check that the refresh is properly aligned INSERT INTO table_int VALUES(114, 0); SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130); -psql:include/cagg_query_common.sql:845: LOG: statement: CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130); +CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:845: LOG: statement: CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:845: DEBUG: refreshing continuous aggregate "cagg_int_offset" from 100 to 130 psql:include/cagg_query_common.sql:845: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_int_offset" in window [ 105, 125 ] psql:include/cagg_query_common.sql:845: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_38" psql:include/cagg_query_common.sql:845: DEBUG: building index "_hyper_38_68_chunk__materialized_hypertable_38_time_bucket_idx" on table "_hyper_38_68_chunk" serially @@ -2363,8 +2370,8 @@ SELECT time_bucket('1 week', time, origin=>'2000-01-02 01:00:00 PST'::timestampt Sun Dec 30 01:00:00 2029 PST | 55555 -- Test refresh -CALL refresh_continuous_aggregate('cagg_1_hour_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_1_week_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_1_hour_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_1_week_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); -- Everything should be now materailized ALTER MATERIALIZED VIEW cagg_1_hour_offset SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW cagg_1_week_offset SET (timescaledb.materialized_only=false); diff --git a/tsl/test/expected/cagg_query_using_merge-15.out b/tsl/test/expected/cagg_query_using_merge-15.out index 80d8b5aa0f5..7db63d3950e 100644 --- a/tsl/test/expected/cagg_query_using_merge-15.out +++ b/tsl/test/expected/cagg_query_using_merge-15.out @@ -64,7 +64,7 @@ FROM ( select max(timec)as timeval from conditions ) as q; ------------------------------ Sat Nov 03 17:00:00 2018 PDT -CALL refresh_continuous_aggregate('mat_m1', NULL, NULL); +CALL refresh_continuous_aggregate('mat_m1', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); --test first/last create materialized view mat_m2(location, timec, firsth, lasth, maxtemp, mintemp) WITH (timescaledb.continuous, timescaledb.materialized_only=false) @@ -79,7 +79,7 @@ FROM ( select max(timec)as timeval from conditions ) as q; ------------------------------ Sat Nov 03 17:00:00 2018 PDT -CALL refresh_continuous_aggregate('mat_m2', NULL, NULL); +CALL refresh_continuous_aggregate('mat_m2', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); --normal view -- create or replace view regview( location, timec, minl, sumt , sumh) as @@ -1089,7 +1089,7 @@ SET ROLE :ROLE_SUPERUSER; BEGIN; UPDATE _timescaledb_catalog.continuous_aggs_bucket_function SET bucket_func = 'func_does_not_exist()'; -- should error because function does not exist -CALL refresh_continuous_aggregate('cagg_bigint_offset2', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_bigint_offset2', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); psql:include/cagg_query_common.sql:566: ERROR: function "func_does_not_exist()" does not exist ROLLBACK; \set ON_ERROR_STOP 1 @@ -1306,9 +1306,9 @@ SELECT * FROM cagg_4_hours_origin; -- Update the last bucket and re-materialize INSERT INTO temperature values('2020-01-01 23:55:00 PST', 10); -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_4_hours; time_bucket | max ------------------------------+----- @@ -1441,17 +1441,20 @@ SELECT * FROM cagg_4_hours_origin; -- Update materialized data SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -psql:include/cagg_query_common.sql:683: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:683: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:683: DEBUG: refreshing continuous aggregate "cagg_4_hours" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:683: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Thu Jan 02 00:00:00 2020 PST, Thu Jan 02 12:00:00 2020 PST ] psql:include/cagg_query_common.sql:683: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -psql:include/cagg_query_common.sql:684: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:684: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:684: DEBUG: refreshing continuous aggregate "cagg_4_hours_offset" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:684: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Wed Jan 01 20:30:00 2020 PST, Thu Jan 02 12:30:00 2020 PST ] psql:include/cagg_query_common.sql:684: LOG: merged 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:684: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_34" -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); -psql:include/cagg_query_common.sql:685: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:685: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:685: DEBUG: refreshing continuous aggregate "cagg_4_hours_origin" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:685: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_origin" in window [ Wed Jan 01 21:00:00 2020 PST, Thu Jan 02 13:00:00 2020 PST ] psql:include/cagg_query_common.sql:685: LOG: merged 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_35" psql:include/cagg_query_common.sql:685: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_35" @@ -1634,9 +1637,9 @@ SELECT time_bucket('4 hour', time, '2000-01-01 01:00:00 PST'::timestamptz), max( -- Test invalidations TRUNCATE temperature; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); INSERT INTO temperature SELECT time, 5 FROM generate_series('2000-01-01 01:00:00 PST'::timestamptz, @@ -1649,8 +1652,9 @@ INSERT INTO temperature values('2020-01-02 01:05:00+01', 2222); INSERT INTO temperature values('2020-01-02 01:35:00+01', 5555); INSERT INTO temperature values('2020-01-02 05:05:00+01', 8888); SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -psql:include/cagg_query_common.sql:725: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:725: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:725: DEBUG: refreshing continuous aggregate "cagg_4_hours" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:725: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577952000000000 psql:include/cagg_query_common.sql:725: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Sat Jan 01 00:00:00 2000 PST, Sun Jan 02 00:00:00 2000 PST ] psql:include/cagg_query_common.sql:725: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" @@ -1658,8 +1662,9 @@ psql:include/cagg_query_common.sql:725: DEBUG: hypertable 33 existing watermark psql:include/cagg_query_common.sql:725: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Wed Jan 01 00:00:00 2020 PST, Thu Jan 02 00:00:00 2020 PST ] psql:include/cagg_query_common.sql:725: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" psql:include/cagg_query_common.sql:725: DEBUG: hypertable 33 existing watermark >= new watermark 1577995200000000 1577952000000000 -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -psql:include/cagg_query_common.sql:726: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:726: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:726: DEBUG: refreshing continuous aggregate "cagg_4_hours_offset" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:726: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577953800000000 psql:include/cagg_query_common.sql:726: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Sat Jan 01 00:30:00 2000 PST, Sun Jan 02 00:30:00 2000 PST ] psql:include/cagg_query_common.sql:726: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" @@ -1667,8 +1672,9 @@ psql:include/cagg_query_common.sql:726: DEBUG: hypertable 34 existing watermark psql:include/cagg_query_common.sql:726: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Tue Dec 31 20:30:00 2019 PST, Thu Jan 02 00:30:00 2020 PST ] psql:include/cagg_query_common.sql:726: LOG: inserted 7 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:726: DEBUG: hypertable 34 existing watermark >= new watermark 1577997000000000 1577953800000000 -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); -psql:include/cagg_query_common.sql:727: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:727: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:727: DEBUG: refreshing continuous aggregate "cagg_4_hours_origin" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:727: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577955600000000 psql:include/cagg_query_common.sql:727: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_origin" in window [ Sat Jan 01 01:00:00 2000 PST, Sun Jan 02 01:00:00 2000 PST ] psql:include/cagg_query_common.sql:727: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_35" @@ -1912,7 +1918,7 @@ ORDER BY 1, 2, 3; 33 | 1577995200000000 | 9223372036854775807 33 | 1577995200000000 | 9223372036854775807 -CALL refresh_continuous_aggregate('cagg_4_hours', '2000-01-01 00:00:00'::timestamptz, '2020-12-31 23:59:59'::timestamptz); +CALL refresh_continuous_aggregate('cagg_4_hours', '2000-01-01 00:00:00'::timestamptz, '2020-12-31 23:59:59'::timestamptz, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log WHERE materialization_id IN (SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'cagg_4_hours') @@ -2040,7 +2046,7 @@ SELECT * FROM cagg_int; 50 | 5 100 | 555 -CALL refresh_continuous_aggregate('cagg_int', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_int', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_int; time_bucket | value -------------+------- @@ -2089,7 +2095,7 @@ SELECT * FROM cagg_int_offset; -- the value 100 is part of the already serializ 45 | 30 95 | 555 -CALL refresh_continuous_aggregate('cagg_int_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_int_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_int_offset; time_bucket | value -------------+------- @@ -2144,8 +2150,9 @@ SELECT * FROM cagg_int_offset; -- Check that the refresh is properly aligned INSERT INTO table_int VALUES(114, 0); SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130); -psql:include/cagg_query_common.sql:845: LOG: statement: CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130); +CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:845: LOG: statement: CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:845: DEBUG: refreshing continuous aggregate "cagg_int_offset" from 100 to 130 psql:include/cagg_query_common.sql:845: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_int_offset" in window [ 105, 125 ] psql:include/cagg_query_common.sql:845: DEBUG: building index "_hyper_38_68_chunk__materialized_hypertable_38_time_bucket_idx" on table "_hyper_38_68_chunk" serially psql:include/cagg_query_common.sql:845: DEBUG: index "_hyper_38_68_chunk__materialized_hypertable_38_time_bucket_idx" can safely use deduplication @@ -2363,8 +2370,8 @@ SELECT time_bucket('1 week', time, origin=>'2000-01-02 01:00:00 PST'::timestampt Sun Dec 30 01:00:00 2029 PST | 55555 -- Test refresh -CALL refresh_continuous_aggregate('cagg_1_hour_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_1_week_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_1_hour_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_1_week_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); -- Everything should be now materailized ALTER MATERIALIZED VIEW cagg_1_hour_offset SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW cagg_1_week_offset SET (timescaledb.materialized_only=false); diff --git a/tsl/test/expected/cagg_query_using_merge-16.out b/tsl/test/expected/cagg_query_using_merge-16.out index 1fdf3c7bc2f..f2a29c63102 100644 --- a/tsl/test/expected/cagg_query_using_merge-16.out +++ b/tsl/test/expected/cagg_query_using_merge-16.out @@ -64,7 +64,7 @@ FROM ( select max(timec)as timeval from conditions ) as q; ------------------------------ Sat Nov 03 17:00:00 2018 PDT -CALL refresh_continuous_aggregate('mat_m1', NULL, NULL); +CALL refresh_continuous_aggregate('mat_m1', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); --test first/last create materialized view mat_m2(location, timec, firsth, lasth, maxtemp, mintemp) WITH (timescaledb.continuous, timescaledb.materialized_only=false) @@ -79,7 +79,7 @@ FROM ( select max(timec)as timeval from conditions ) as q; ------------------------------ Sat Nov 03 17:00:00 2018 PDT -CALL refresh_continuous_aggregate('mat_m2', NULL, NULL); +CALL refresh_continuous_aggregate('mat_m2', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); --normal view -- create or replace view regview( location, timec, minl, sumt , sumh) as @@ -1083,7 +1083,7 @@ SET ROLE :ROLE_SUPERUSER; BEGIN; UPDATE _timescaledb_catalog.continuous_aggs_bucket_function SET bucket_func = 'func_does_not_exist()'; -- should error because function does not exist -CALL refresh_continuous_aggregate('cagg_bigint_offset2', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_bigint_offset2', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); psql:include/cagg_query_common.sql:566: ERROR: function "func_does_not_exist()" does not exist ROLLBACK; \set ON_ERROR_STOP 1 @@ -1300,9 +1300,9 @@ SELECT * FROM cagg_4_hours_origin; -- Update the last bucket and re-materialize INSERT INTO temperature values('2020-01-01 23:55:00 PST', 10); -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_4_hours; time_bucket | max ------------------------------+----- @@ -1435,17 +1435,20 @@ SELECT * FROM cagg_4_hours_origin; -- Update materialized data SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -psql:include/cagg_query_common.sql:683: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:683: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:683: DEBUG: refreshing continuous aggregate "cagg_4_hours" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:683: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Thu Jan 02 00:00:00 2020 PST, Thu Jan 02 12:00:00 2020 PST ] psql:include/cagg_query_common.sql:683: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -psql:include/cagg_query_common.sql:684: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:684: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:684: DEBUG: refreshing continuous aggregate "cagg_4_hours_offset" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:684: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Wed Jan 01 20:30:00 2020 PST, Thu Jan 02 12:30:00 2020 PST ] psql:include/cagg_query_common.sql:684: LOG: merged 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:684: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_34" -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); -psql:include/cagg_query_common.sql:685: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:685: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:685: DEBUG: refreshing continuous aggregate "cagg_4_hours_origin" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:685: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_origin" in window [ Wed Jan 01 21:00:00 2020 PST, Thu Jan 02 13:00:00 2020 PST ] psql:include/cagg_query_common.sql:685: LOG: merged 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_35" psql:include/cagg_query_common.sql:685: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_35" @@ -1628,9 +1631,9 @@ SELECT time_bucket('4 hour', time, '2000-01-01 01:00:00 PST'::timestamptz), max( -- Test invalidations TRUNCATE temperature; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); INSERT INTO temperature SELECT time, 5 FROM generate_series('2000-01-01 01:00:00 PST'::timestamptz, @@ -1643,8 +1646,9 @@ INSERT INTO temperature values('2020-01-02 01:05:00+01', 2222); INSERT INTO temperature values('2020-01-02 01:35:00+01', 5555); INSERT INTO temperature values('2020-01-02 05:05:00+01', 8888); SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -psql:include/cagg_query_common.sql:725: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:725: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:725: DEBUG: refreshing continuous aggregate "cagg_4_hours" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:725: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577952000000000 psql:include/cagg_query_common.sql:725: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Sat Jan 01 00:00:00 2000 PST, Sun Jan 02 00:00:00 2000 PST ] psql:include/cagg_query_common.sql:725: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" @@ -1652,8 +1656,9 @@ psql:include/cagg_query_common.sql:725: DEBUG: hypertable 33 existing watermark psql:include/cagg_query_common.sql:725: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Wed Jan 01 00:00:00 2020 PST, Thu Jan 02 00:00:00 2020 PST ] psql:include/cagg_query_common.sql:725: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" psql:include/cagg_query_common.sql:725: DEBUG: hypertable 33 existing watermark >= new watermark 1577995200000000 1577952000000000 -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -psql:include/cagg_query_common.sql:726: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:726: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:726: DEBUG: refreshing continuous aggregate "cagg_4_hours_offset" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:726: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577953800000000 psql:include/cagg_query_common.sql:726: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Sat Jan 01 00:30:00 2000 PST, Sun Jan 02 00:30:00 2000 PST ] psql:include/cagg_query_common.sql:726: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" @@ -1661,8 +1666,9 @@ psql:include/cagg_query_common.sql:726: DEBUG: hypertable 34 existing watermark psql:include/cagg_query_common.sql:726: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Tue Dec 31 20:30:00 2019 PST, Thu Jan 02 00:30:00 2020 PST ] psql:include/cagg_query_common.sql:726: LOG: inserted 7 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:726: DEBUG: hypertable 34 existing watermark >= new watermark 1577997000000000 1577953800000000 -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); -psql:include/cagg_query_common.sql:727: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:727: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:727: DEBUG: refreshing continuous aggregate "cagg_4_hours_origin" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:727: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577955600000000 psql:include/cagg_query_common.sql:727: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_origin" in window [ Sat Jan 01 01:00:00 2000 PST, Sun Jan 02 01:00:00 2000 PST ] psql:include/cagg_query_common.sql:727: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_35" @@ -1906,7 +1912,7 @@ ORDER BY 1, 2, 3; 33 | 1577995200000000 | 9223372036854775807 33 | 1577995200000000 | 9223372036854775807 -CALL refresh_continuous_aggregate('cagg_4_hours', '2000-01-01 00:00:00'::timestamptz, '2020-12-31 23:59:59'::timestamptz); +CALL refresh_continuous_aggregate('cagg_4_hours', '2000-01-01 00:00:00'::timestamptz, '2020-12-31 23:59:59'::timestamptz, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log WHERE materialization_id IN (SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'cagg_4_hours') @@ -2034,7 +2040,7 @@ SELECT * FROM cagg_int; 50 | 5 100 | 555 -CALL refresh_continuous_aggregate('cagg_int', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_int', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_int; time_bucket | value -------------+------- @@ -2083,7 +2089,7 @@ SELECT * FROM cagg_int_offset; -- the value 100 is part of the already serializ 45 | 30 95 | 555 -CALL refresh_continuous_aggregate('cagg_int_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_int_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_int_offset; time_bucket | value -------------+------- @@ -2138,8 +2144,9 @@ SELECT * FROM cagg_int_offset; -- Check that the refresh is properly aligned INSERT INTO table_int VALUES(114, 0); SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130); -psql:include/cagg_query_common.sql:845: LOG: statement: CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130); +CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:845: LOG: statement: CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:845: DEBUG: refreshing continuous aggregate "cagg_int_offset" from 100 to 130 psql:include/cagg_query_common.sql:845: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_int_offset" in window [ 105, 125 ] psql:include/cagg_query_common.sql:845: DEBUG: building index "_hyper_38_68_chunk__materialized_hypertable_38_time_bucket_idx" on table "_hyper_38_68_chunk" serially psql:include/cagg_query_common.sql:845: DEBUG: index "_hyper_38_68_chunk__materialized_hypertable_38_time_bucket_idx" can safely use deduplication @@ -2357,8 +2364,8 @@ SELECT time_bucket('1 week', time, origin=>'2000-01-02 01:00:00 PST'::timestampt Sun Dec 30 01:00:00 2029 PST | 55555 -- Test refresh -CALL refresh_continuous_aggregate('cagg_1_hour_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_1_week_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_1_hour_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_1_week_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); -- Everything should be now materailized ALTER MATERIALIZED VIEW cagg_1_hour_offset SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW cagg_1_week_offset SET (timescaledb.materialized_only=false); diff --git a/tsl/test/expected/cagg_query_using_merge-17.out b/tsl/test/expected/cagg_query_using_merge-17.out index 1fdf3c7bc2f..f2a29c63102 100644 --- a/tsl/test/expected/cagg_query_using_merge-17.out +++ b/tsl/test/expected/cagg_query_using_merge-17.out @@ -64,7 +64,7 @@ FROM ( select max(timec)as timeval from conditions ) as q; ------------------------------ Sat Nov 03 17:00:00 2018 PDT -CALL refresh_continuous_aggregate('mat_m1', NULL, NULL); +CALL refresh_continuous_aggregate('mat_m1', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); --test first/last create materialized view mat_m2(location, timec, firsth, lasth, maxtemp, mintemp) WITH (timescaledb.continuous, timescaledb.materialized_only=false) @@ -79,7 +79,7 @@ FROM ( select max(timec)as timeval from conditions ) as q; ------------------------------ Sat Nov 03 17:00:00 2018 PDT -CALL refresh_continuous_aggregate('mat_m2', NULL, NULL); +CALL refresh_continuous_aggregate('mat_m2', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); --normal view -- create or replace view regview( location, timec, minl, sumt , sumh) as @@ -1083,7 +1083,7 @@ SET ROLE :ROLE_SUPERUSER; BEGIN; UPDATE _timescaledb_catalog.continuous_aggs_bucket_function SET bucket_func = 'func_does_not_exist()'; -- should error because function does not exist -CALL refresh_continuous_aggregate('cagg_bigint_offset2', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_bigint_offset2', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); psql:include/cagg_query_common.sql:566: ERROR: function "func_does_not_exist()" does not exist ROLLBACK; \set ON_ERROR_STOP 1 @@ -1300,9 +1300,9 @@ SELECT * FROM cagg_4_hours_origin; -- Update the last bucket and re-materialize INSERT INTO temperature values('2020-01-01 23:55:00 PST', 10); -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_4_hours; time_bucket | max ------------------------------+----- @@ -1435,17 +1435,20 @@ SELECT * FROM cagg_4_hours_origin; -- Update materialized data SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -psql:include/cagg_query_common.sql:683: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:683: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:683: DEBUG: refreshing continuous aggregate "cagg_4_hours" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:683: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Thu Jan 02 00:00:00 2020 PST, Thu Jan 02 12:00:00 2020 PST ] psql:include/cagg_query_common.sql:683: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -psql:include/cagg_query_common.sql:684: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:684: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:684: DEBUG: refreshing continuous aggregate "cagg_4_hours_offset" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:684: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Wed Jan 01 20:30:00 2020 PST, Thu Jan 02 12:30:00 2020 PST ] psql:include/cagg_query_common.sql:684: LOG: merged 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:684: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_34" -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); -psql:include/cagg_query_common.sql:685: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:685: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:685: DEBUG: refreshing continuous aggregate "cagg_4_hours_origin" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:685: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_origin" in window [ Wed Jan 01 21:00:00 2020 PST, Thu Jan 02 13:00:00 2020 PST ] psql:include/cagg_query_common.sql:685: LOG: merged 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_35" psql:include/cagg_query_common.sql:685: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_35" @@ -1628,9 +1631,9 @@ SELECT time_bucket('4 hour', time, '2000-01-01 01:00:00 PST'::timestamptz), max( -- Test invalidations TRUNCATE temperature; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); INSERT INTO temperature SELECT time, 5 FROM generate_series('2000-01-01 01:00:00 PST'::timestamptz, @@ -1643,8 +1646,9 @@ INSERT INTO temperature values('2020-01-02 01:05:00+01', 2222); INSERT INTO temperature values('2020-01-02 01:35:00+01', 5555); INSERT INTO temperature values('2020-01-02 05:05:00+01', 8888); SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -psql:include/cagg_query_common.sql:725: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:725: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:725: DEBUG: refreshing continuous aggregate "cagg_4_hours" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:725: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577952000000000 psql:include/cagg_query_common.sql:725: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Sat Jan 01 00:00:00 2000 PST, Sun Jan 02 00:00:00 2000 PST ] psql:include/cagg_query_common.sql:725: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" @@ -1652,8 +1656,9 @@ psql:include/cagg_query_common.sql:725: DEBUG: hypertable 33 existing watermark psql:include/cagg_query_common.sql:725: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Wed Jan 01 00:00:00 2020 PST, Thu Jan 02 00:00:00 2020 PST ] psql:include/cagg_query_common.sql:725: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" psql:include/cagg_query_common.sql:725: DEBUG: hypertable 33 existing watermark >= new watermark 1577995200000000 1577952000000000 -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -psql:include/cagg_query_common.sql:726: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:726: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:726: DEBUG: refreshing continuous aggregate "cagg_4_hours_offset" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:726: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577953800000000 psql:include/cagg_query_common.sql:726: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Sat Jan 01 00:30:00 2000 PST, Sun Jan 02 00:30:00 2000 PST ] psql:include/cagg_query_common.sql:726: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" @@ -1661,8 +1666,9 @@ psql:include/cagg_query_common.sql:726: DEBUG: hypertable 34 existing watermark psql:include/cagg_query_common.sql:726: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Tue Dec 31 20:30:00 2019 PST, Thu Jan 02 00:30:00 2020 PST ] psql:include/cagg_query_common.sql:726: LOG: inserted 7 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:726: DEBUG: hypertable 34 existing watermark >= new watermark 1577997000000000 1577953800000000 -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); -psql:include/cagg_query_common.sql:727: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:727: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:727: DEBUG: refreshing continuous aggregate "cagg_4_hours_origin" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:727: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577955600000000 psql:include/cagg_query_common.sql:727: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_origin" in window [ Sat Jan 01 01:00:00 2000 PST, Sun Jan 02 01:00:00 2000 PST ] psql:include/cagg_query_common.sql:727: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_35" @@ -1906,7 +1912,7 @@ ORDER BY 1, 2, 3; 33 | 1577995200000000 | 9223372036854775807 33 | 1577995200000000 | 9223372036854775807 -CALL refresh_continuous_aggregate('cagg_4_hours', '2000-01-01 00:00:00'::timestamptz, '2020-12-31 23:59:59'::timestamptz); +CALL refresh_continuous_aggregate('cagg_4_hours', '2000-01-01 00:00:00'::timestamptz, '2020-12-31 23:59:59'::timestamptz, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log WHERE materialization_id IN (SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'cagg_4_hours') @@ -2034,7 +2040,7 @@ SELECT * FROM cagg_int; 50 | 5 100 | 555 -CALL refresh_continuous_aggregate('cagg_int', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_int', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_int; time_bucket | value -------------+------- @@ -2083,7 +2089,7 @@ SELECT * FROM cagg_int_offset; -- the value 100 is part of the already serializ 45 | 30 95 | 555 -CALL refresh_continuous_aggregate('cagg_int_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_int_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_int_offset; time_bucket | value -------------+------- @@ -2138,8 +2144,9 @@ SELECT * FROM cagg_int_offset; -- Check that the refresh is properly aligned INSERT INTO table_int VALUES(114, 0); SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130); -psql:include/cagg_query_common.sql:845: LOG: statement: CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130); +CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:845: LOG: statement: CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:845: DEBUG: refreshing continuous aggregate "cagg_int_offset" from 100 to 130 psql:include/cagg_query_common.sql:845: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_int_offset" in window [ 105, 125 ] psql:include/cagg_query_common.sql:845: DEBUG: building index "_hyper_38_68_chunk__materialized_hypertable_38_time_bucket_idx" on table "_hyper_38_68_chunk" serially psql:include/cagg_query_common.sql:845: DEBUG: index "_hyper_38_68_chunk__materialized_hypertable_38_time_bucket_idx" can safely use deduplication @@ -2357,8 +2364,8 @@ SELECT time_bucket('1 week', time, origin=>'2000-01-02 01:00:00 PST'::timestampt Sun Dec 30 01:00:00 2029 PST | 55555 -- Test refresh -CALL refresh_continuous_aggregate('cagg_1_hour_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_1_week_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_1_hour_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_1_week_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); -- Everything should be now materailized ALTER MATERIALIZED VIEW cagg_1_hour_offset SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW cagg_1_week_offset SET (timescaledb.materialized_only=false); diff --git a/tsl/test/expected/cagg_query_using_merge-18.out b/tsl/test/expected/cagg_query_using_merge-18.out index 1fdf3c7bc2f..f2a29c63102 100644 --- a/tsl/test/expected/cagg_query_using_merge-18.out +++ b/tsl/test/expected/cagg_query_using_merge-18.out @@ -64,7 +64,7 @@ FROM ( select max(timec)as timeval from conditions ) as q; ------------------------------ Sat Nov 03 17:00:00 2018 PDT -CALL refresh_continuous_aggregate('mat_m1', NULL, NULL); +CALL refresh_continuous_aggregate('mat_m1', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); --test first/last create materialized view mat_m2(location, timec, firsth, lasth, maxtemp, mintemp) WITH (timescaledb.continuous, timescaledb.materialized_only=false) @@ -79,7 +79,7 @@ FROM ( select max(timec)as timeval from conditions ) as q; ------------------------------ Sat Nov 03 17:00:00 2018 PDT -CALL refresh_continuous_aggregate('mat_m2', NULL, NULL); +CALL refresh_continuous_aggregate('mat_m2', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); --normal view -- create or replace view regview( location, timec, minl, sumt , sumh) as @@ -1083,7 +1083,7 @@ SET ROLE :ROLE_SUPERUSER; BEGIN; UPDATE _timescaledb_catalog.continuous_aggs_bucket_function SET bucket_func = 'func_does_not_exist()'; -- should error because function does not exist -CALL refresh_continuous_aggregate('cagg_bigint_offset2', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_bigint_offset2', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); psql:include/cagg_query_common.sql:566: ERROR: function "func_does_not_exist()" does not exist ROLLBACK; \set ON_ERROR_STOP 1 @@ -1300,9 +1300,9 @@ SELECT * FROM cagg_4_hours_origin; -- Update the last bucket and re-materialize INSERT INTO temperature values('2020-01-01 23:55:00 PST', 10); -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_4_hours; time_bucket | max ------------------------------+----- @@ -1435,17 +1435,20 @@ SELECT * FROM cagg_4_hours_origin; -- Update materialized data SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -psql:include/cagg_query_common.sql:683: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:683: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:683: DEBUG: refreshing continuous aggregate "cagg_4_hours" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:683: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Thu Jan 02 00:00:00 2020 PST, Thu Jan 02 12:00:00 2020 PST ] psql:include/cagg_query_common.sql:683: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -psql:include/cagg_query_common.sql:684: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:684: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:684: DEBUG: refreshing continuous aggregate "cagg_4_hours_offset" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:684: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Wed Jan 01 20:30:00 2020 PST, Thu Jan 02 12:30:00 2020 PST ] psql:include/cagg_query_common.sql:684: LOG: merged 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:684: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_34" -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); -psql:include/cagg_query_common.sql:685: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:685: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:685: DEBUG: refreshing continuous aggregate "cagg_4_hours_origin" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:685: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_origin" in window [ Wed Jan 01 21:00:00 2020 PST, Thu Jan 02 13:00:00 2020 PST ] psql:include/cagg_query_common.sql:685: LOG: merged 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_35" psql:include/cagg_query_common.sql:685: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_35" @@ -1628,9 +1631,9 @@ SELECT time_bucket('4 hour', time, '2000-01-01 01:00:00 PST'::timestamptz), max( -- Test invalidations TRUNCATE temperature; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); INSERT INTO temperature SELECT time, 5 FROM generate_series('2000-01-01 01:00:00 PST'::timestamptz, @@ -1643,8 +1646,9 @@ INSERT INTO temperature values('2020-01-02 01:05:00+01', 2222); INSERT INTO temperature values('2020-01-02 01:35:00+01', 5555); INSERT INTO temperature values('2020-01-02 05:05:00+01', 8888); SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -psql:include/cagg_query_common.sql:725: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:725: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:725: DEBUG: refreshing continuous aggregate "cagg_4_hours" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:725: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577952000000000 psql:include/cagg_query_common.sql:725: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Sat Jan 01 00:00:00 2000 PST, Sun Jan 02 00:00:00 2000 PST ] psql:include/cagg_query_common.sql:725: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" @@ -1652,8 +1656,9 @@ psql:include/cagg_query_common.sql:725: DEBUG: hypertable 33 existing watermark psql:include/cagg_query_common.sql:725: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Wed Jan 01 00:00:00 2020 PST, Thu Jan 02 00:00:00 2020 PST ] psql:include/cagg_query_common.sql:725: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" psql:include/cagg_query_common.sql:725: DEBUG: hypertable 33 existing watermark >= new watermark 1577995200000000 1577952000000000 -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -psql:include/cagg_query_common.sql:726: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:726: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:726: DEBUG: refreshing continuous aggregate "cagg_4_hours_offset" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:726: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577953800000000 psql:include/cagg_query_common.sql:726: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Sat Jan 01 00:30:00 2000 PST, Sun Jan 02 00:30:00 2000 PST ] psql:include/cagg_query_common.sql:726: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" @@ -1661,8 +1666,9 @@ psql:include/cagg_query_common.sql:726: DEBUG: hypertable 34 existing watermark psql:include/cagg_query_common.sql:726: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Tue Dec 31 20:30:00 2019 PST, Thu Jan 02 00:30:00 2020 PST ] psql:include/cagg_query_common.sql:726: LOG: inserted 7 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:726: DEBUG: hypertable 34 existing watermark >= new watermark 1577997000000000 1577953800000000 -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); -psql:include/cagg_query_common.sql:727: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:727: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:727: DEBUG: refreshing continuous aggregate "cagg_4_hours_origin" from Sun Nov 23 16:07:02 4714 LMT BC to infinity psql:include/cagg_query_common.sql:727: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577955600000000 psql:include/cagg_query_common.sql:727: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_origin" in window [ Sat Jan 01 01:00:00 2000 PST, Sun Jan 02 01:00:00 2000 PST ] psql:include/cagg_query_common.sql:727: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_35" @@ -1906,7 +1912,7 @@ ORDER BY 1, 2, 3; 33 | 1577995200000000 | 9223372036854775807 33 | 1577995200000000 | 9223372036854775807 -CALL refresh_continuous_aggregate('cagg_4_hours', '2000-01-01 00:00:00'::timestamptz, '2020-12-31 23:59:59'::timestamptz); +CALL refresh_continuous_aggregate('cagg_4_hours', '2000-01-01 00:00:00'::timestamptz, '2020-12-31 23:59:59'::timestamptz, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log WHERE materialization_id IN (SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'cagg_4_hours') @@ -2034,7 +2040,7 @@ SELECT * FROM cagg_int; 50 | 5 100 | 555 -CALL refresh_continuous_aggregate('cagg_int', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_int', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_int; time_bucket | value -------------+------- @@ -2083,7 +2089,7 @@ SELECT * FROM cagg_int_offset; -- the value 100 is part of the already serializ 45 | 30 95 | 555 -CALL refresh_continuous_aggregate('cagg_int_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_int_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_int_offset; time_bucket | value -------------+------- @@ -2138,8 +2144,9 @@ SELECT * FROM cagg_int_offset; -- Check that the refresh is properly aligned INSERT INTO table_int VALUES(114, 0); SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130); -psql:include/cagg_query_common.sql:845: LOG: statement: CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130); +CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:845: LOG: statement: CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130, options => '{"buckets_per_batch": 0}'::jsonb); +psql:include/cagg_query_common.sql:845: DEBUG: refreshing continuous aggregate "cagg_int_offset" from 100 to 130 psql:include/cagg_query_common.sql:845: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_int_offset" in window [ 105, 125 ] psql:include/cagg_query_common.sql:845: DEBUG: building index "_hyper_38_68_chunk__materialized_hypertable_38_time_bucket_idx" on table "_hyper_38_68_chunk" serially psql:include/cagg_query_common.sql:845: DEBUG: index "_hyper_38_68_chunk__materialized_hypertable_38_time_bucket_idx" can safely use deduplication @@ -2357,8 +2364,8 @@ SELECT time_bucket('1 week', time, origin=>'2000-01-02 01:00:00 PST'::timestampt Sun Dec 30 01:00:00 2029 PST | 55555 -- Test refresh -CALL refresh_continuous_aggregate('cagg_1_hour_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_1_week_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_1_hour_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_1_week_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); -- Everything should be now materailized ALTER MATERIALIZED VIEW cagg_1_hour_offset SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW cagg_1_week_offset SET (timescaledb.materialized_only=false); diff --git a/tsl/test/expected/cagg_refresh_incremental.out b/tsl/test/expected/cagg_refresh_incremental.out new file mode 100644 index 00000000000..f5146b346a0 --- /dev/null +++ b/tsl/test/expected/cagg_refresh_incremental.out @@ -0,0 +1,722 @@ +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license. +\c :TEST_DBNAME :ROLE_SUPERUSER +-- Create a user with specific timezone for deterministic output +CREATE ROLE test_cagg_refresh_manual_user WITH LOGIN; +ALTER ROLE test_cagg_refresh_manual_user SET timezone TO 'UTC'; +GRANT ALL ON SCHEMA public TO test_cagg_refresh_manual_user; +\c :TEST_DBNAME test_cagg_refresh_manual_user +SET timezone TO 'UTC'; +CREATE TABLE conditions ( + time TIMESTAMP WITH TIME ZONE NOT NULL, + device_id INTEGER, + temperature NUMERIC +); +SELECT FROM create_hypertable('conditions', by_range('time')); +-- + +INSERT INTO conditions +SELECT + t, d, 10 +FROM + generate_series( + '2025-02-05 00:00:00+00', + '2025-03-05 00:00:00+00', + '1 hour'::interval) AS t, + generate_series(1,5) AS d; +CREATE MATERIALIZED VIEW conditions_by_day +WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS +SELECT + time_bucket('1 day', time), + device_id, + count(*), + min(temperature), + max(temperature), + avg(temperature), + sum(temperature) +FROM + conditions +GROUP BY + 1, 2 +WITH NO DATA; +-- Issue an incremental manual refresh using JSONB options: +-- buckets_per_batch => 10. This is the manual equivalent of a +-- policy with the same setting. +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 10}'::jsonb); +LOG: statement: CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 10}'::jsonb); +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 25 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 50 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 50 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 20 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +SELECT count(*) FROM conditions_by_day; + count +------- + 145 + +CREATE MATERIALIZED VIEW conditions_by_day_atomic_refresh +WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS +SELECT + time_bucket('1 day', time), + device_id, + count(*), + min(temperature), + max(temperature), + avg(temperature), + sum(temperature) +FROM + conditions +GROUP BY + 1, 2 +WITH NO DATA; +CALL refresh_continuous_aggregate('conditions_by_day_atomic_refresh', NULL, NULL); +SELECT count(*) FROM conditions_by_day; + count +------- + 145 + +SELECT count(*) FROM conditions_by_day_atomic_refresh; + count +------- + 145 + +-- Should have no differences +SELECT + count(*) > 0 AS has_diff +FROM + ((SELECT * FROM conditions_by_day_atomic_refresh ORDER BY 1, 2) + EXCEPT + (SELECT * FROM conditions_by_day ORDER BY 1, 2)) AS diff; + has_diff +---------- + f + +-- buckets_per_batch => 0 is NOT incremental: the whole window is materialized +-- in a single pass. Under LOG this shows exactly one delete+insert pair, unlike +-- the multi-batch run above. The TRUNCATE invalidates the whole range so there +-- is data to re-materialize. +TRUNCATE conditions_by_day; +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 0}'::jsonb); +LOG: statement: CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 0}'::jsonb); +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 145 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +SELECT count(*) FROM conditions_by_day; + count +------- + 145 + +-- The continuous aggregate is now fully materialized with no pending +-- invalidations, so a normal refresh is a no-op (reports up-to-date). +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate('conditions_by_day', NULL, NULL); +LOG: statement: CALL refresh_continuous_aggregate('conditions_by_day', NULL, NULL); +NOTICE: continuous aggregate "conditions_by_day" is already up-to-date +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +-- Assert there really are no pending invalidations +SELECT + (SELECT count(*) + FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log + WHERE materialization_id = cagg.mat_hypertable_id + AND greatest_modified_value >= lowest_modified_value + AND lowest_modified_value != -9223372036854775808 + AND greatest_modified_value != 9223372036854775807) AS mat_invalidations, + (SELECT count(*) + FROM _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log + WHERE hypertable_id = cagg.raw_hypertable_id + AND greatest_modified_value >= lowest_modified_value) AS ht_invalidations +FROM _timescaledb_catalog.continuous_agg cagg +WHERE cagg.user_view_name = 'conditions_by_day'; + mat_invalidations | ht_invalidations +-------------------+------------------ + 0 | 0 + +-- A forced refresh is incremental but ignores the invalidation logs: +-- it re-materializes the entire window in batches (buckets_per_batch +-- defaults to 10) even though nothing is invalidated. +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate('conditions_by_day', NULL, NULL, force => true); +LOG: statement: CALL refresh_continuous_aggregate('conditions_by_day', NULL, NULL, force => true); +LOG: deleted 25 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 25 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: deleted 50 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 50 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: deleted 50 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 50 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: deleted 20 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 20 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +SELECT count(*) FROM conditions_by_day; + count +------- + 145 + +TRUNCATE conditions_by_day; +-- Run with max_batches_per_execution => 2. Manual refresh stops mid-window +-- after processing 2 batches, leaving the rest for subsequent calls. +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 10, "max_batches_per_execution": 2}'::jsonb); +LOG: statement: CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 10, "max_batches_per_execution": 2}'::jsonb); +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 25 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 50 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: reached maximum number of batches per execution (2), batches not processed (2) +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +SELECT count(*) FROM conditions_by_day; + count +------- + 75 + +SELECT count(*) FROM conditions_by_day_atomic_refresh; + count +------- + 145 + +-- Should have differences (partial materialization) +SELECT + count(*) > 0 AS has_diff +FROM + ((SELECT * FROM conditions_by_day_atomic_refresh ORDER BY 1, 2) + EXCEPT + (SELECT * FROM conditions_by_day ORDER BY 1, 2)) AS diff; + has_diff +---------- + t + +-- Run a second call (same options) to process more batches. +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 10, "max_batches_per_execution": 2}'::jsonb); +LOG: statement: CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 10, "max_batches_per_execution": 2}'::jsonb); +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 50 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 20 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +SELECT count(*) FROM conditions_by_day; + count +------- + 145 + +SELECT count(*) FROM conditions_by_day_atomic_refresh; + count +------- + 145 + +-- Should have no differences (all batches processed) +SELECT + count(*) > 0 AS has_diff +FROM + ((SELECT * FROM conditions_by_day_atomic_refresh ORDER BY 1, 2) + EXCEPT + (SELECT * FROM conditions_by_day ORDER BY 1, 2)) AS diff; + has_diff +---------- + f + +-- Set max_batches_per_execution to 10 (effectively unlimited for our window) +-- and insert data into the past so a new set of batches must be processed. +INSERT INTO conditions +SELECT + t, d, 10 +FROM + generate_series( + '2020-02-05 00:00:00+00', + '2020-03-05 00:00:00+00', + '1 hour'::interval) AS t, + generate_series(1,5) AS d; +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 10, "max_batches_per_execution": 10}'::jsonb); +LOG: statement: CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 10, "max_batches_per_execution": 10}'::jsonb); +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 50 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 50 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 50 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +SELECT count(*) FROM conditions_by_day; + count +------- + 295 + +SELECT count(*) FROM conditions_by_day_atomic_refresh; + count +------- + 145 + +CALL refresh_continuous_aggregate('conditions_by_day_atomic_refresh', NULL, NULL); +SELECT count(*) FROM conditions_by_day; + count +------- + 295 + +SELECT count(*) FROM conditions_by_day_atomic_refresh; + count +------- + 295 + +-- Should have no differences +SELECT + count(*) > 0 AS has_diff +FROM + ((SELECT * FROM conditions_by_day_atomic_refresh ORDER BY 1, 2) + EXCEPT + (SELECT * FROM conditions_by_day ORDER BY 1, 2)) AS diff; + has_diff +---------- + f + +-- Invalid configurations should be rejected +\set ON_ERROR_STOP 0 +\set VERBOSITY default +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"max_batches_per_execution": -1}'::jsonb); +ERROR: invalid max batches per execution +DETAIL: max_batches_per_execution: -1 +HINT: The max batches per execution should be greater than or equal to zero. +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": -1}'::jsonb); +ERROR: invalid buckets per batch +DETAIL: buckets_per_batch: -1 +HINT: The buckets per batch should be greater than or equal to zero. +\set VERBOSITY terse +\set ON_ERROR_STOP 1 +-- Truncate all data from the original hypertable. +TRUNCATE conditions; +-- Should fall back to single-batch processing because there's no data +-- to refresh on the source hypertable. +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 10}'::jsonb); +LOG: statement: CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 10}'::jsonb); +LOG: deleted 295 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 0 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +-- Should return zero rows +SELECT count(*) FROM conditions_by_day; + count +------- + 0 + +-- Insert 1 day of data +INSERT INTO conditions +SELECT + t, d, 10 +FROM + generate_series( + '2020-02-05 00:00:00+00', + '2020-02-06 00:00:00+00', + '1 hour'::interval) AS t, + generate_series(1,5) AS d; +-- Should fall back to single-batch processing because the refresh size +-- (1 day) is smaller than 10 buckets x 1 day. +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 10}'::jsonb); +LOG: statement: CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 10}'::jsonb); +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 10 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +-- Should return 10 rows because the bucket width is `1 day` and we +-- inserted across two boundary timestamps for 5 devices. +SELECT count(*) FROM conditions_by_day; + count +------- + 10 + +TRUNCATE conditions_by_day, conditions; +-- Less than 1 day of data (smaller than the bucket width) +INSERT INTO conditions +VALUES ('2020-02-05 00:00:00+00', 1, 10); +-- Should fall back to single-batch processing because the refresh size +-- is smaller than the bucket width. +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 10}'::jsonb); +LOG: statement: CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 10}'::jsonb); +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +-- Should return 1 row +SELECT count(*) FROM conditions_by_day; + count +------- + 1 + +-- Re-test with an explicit refresh_newest_first => true (default behavior). +TRUNCATE conditions_by_day, conditions_by_day_atomic_refresh, conditions; +INSERT INTO conditions +SELECT + t, d, 10 +FROM + generate_series( + '2025-03-11 00:00:00+00'::timestamptz - INTERVAL '30 days', + '2025-03-11 00:00:00+00'::timestamptz, + '1 hour'::interval) AS t, + generate_series(1,5) AS d; +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_day', + '2025-03-11 00:00:00+00'::timestamptz - INTERVAL '15 days', + NULL, + options => '{"buckets_per_batch": 5, "refresh_newest_first": true}'::jsonb); +LOG: statement: CALL refresh_continuous_aggregate( + 'conditions_by_day', + '2025-03-11 00:00:00+00'::timestamptz - INTERVAL '15 days', + NULL, + options => '{"buckets_per_batch": 5, "refresh_newest_first": true}'::jsonb); +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 5 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 25 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 25 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 25 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +CALL refresh_continuous_aggregate( + 'conditions_by_day_atomic_refresh', + '2025-03-11 00:00:00+00'::timestamptz - INTERVAL '15 days', + NULL); +-- Both continuous aggregates should have the same data +SELECT count(*) FROM conditions_by_day; + count +------- + 80 + +SELECT count(*) FROM conditions_by_day_atomic_refresh; + count +------- + 80 + +-- Should have no differences +SELECT + count(*) > 0 AS has_diff +FROM + ((SELECT * FROM conditions_by_day_atomic_refresh ORDER BY 1, 2) + EXCEPT + (SELECT * FROM conditions_by_day ORDER BY 1, 2)) AS diff; + has_diff +---------- + f + +-- refresh_newest_first => false (process from oldest to newest) +TRUNCATE conditions_by_day; +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_day', + '2025-03-11 00:00:00+00'::timestamptz - INTERVAL '15 days', + NULL, + options => '{"buckets_per_batch": 5, "refresh_newest_first": false}'::jsonb); +LOG: statement: CALL refresh_continuous_aggregate( + 'conditions_by_day', + '2025-03-11 00:00:00+00'::timestamptz - INTERVAL '15 days', + NULL, + options => '{"buckets_per_batch": 5, "refresh_newest_first": false}'::jsonb); +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 25 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 25 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 25 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 5 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +-- Both continuous aggregates should have the same data +SELECT count(*) FROM conditions_by_day; + count +------- + 80 + +SELECT count(*) FROM conditions_by_day_atomic_refresh; + count +------- + 80 + +-- Should have no differences +SELECT + count(*) > 0 AS has_diff +FROM + ((SELECT * FROM conditions_by_day_atomic_refresh ORDER BY 1, 2) + EXCEPT + (SELECT * FROM conditions_by_day ORDER BY 1, 2)) AS diff; + has_diff +---------- + f + +-- Tests with variable-sized bucket (monthly) +TRUNCATE conditions; +INSERT INTO conditions +SELECT + t, d, 10 +FROM + generate_series( + '2025-01-01 00:00:00+00', + '2025-10-08 00:00:00+00', + '1 hour'::interval) AS t, + generate_series(1,5) AS d; +CREATE MATERIALIZED VIEW conditions_by_month +WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS +SELECT + time_bucket('1 month', time), + device_id, + count(*), + min(temperature), + max(temperature), + avg(temperature), + sum(temperature) +FROM + conditions +GROUP BY + 1, 2 +WITH NO DATA; +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_month', + '2025-03-11 00:00:00+00'::timestamptz - INTERVAL '600 days', + '2025-03-11 00:00:00+00'::timestamptz - INTERVAL '7 days', + options => '{"refresh_newest_first": false}'::jsonb); +LOG: statement: CALL refresh_continuous_aggregate( + 'conditions_by_month', + '2025-03-11 00:00:00+00'::timestamptz - INTERVAL '600 days', + '2025-03-11 00:00:00+00'::timestamptz - INTERVAL '7 days', + options => '{"refresh_newest_first": false}'::jsonb); +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_4" +LOG: inserted 10 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_4" +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +SELECT count(*) FROM conditions_by_month; + count +------- + 10 + +------------------------------------------------------------------------------------------ +-- Test that batched refresh with variable-length buckets doesn't leave remainders +------------------------------------------------------------------------------------------ +CREATE TABLE test_data ( + time TIMESTAMPTZ NOT NULL, + value INT +); +SELECT public.create_hypertable( + relation => 'test_data', + time_column_name => 'time', + chunk_time_interval => interval '1 months' +); + create_hypertable +------------------------ + (5,public,test_data,t) + +-- Insert initial data +INSERT INTO test_data +SELECT time, 1 +FROM generate_series('2024-01-01'::timestamptz, '2024-12-31'::timestamptz, '1 day'::interval) time; +-- Create continuous aggregate with monthly buckets (variable-length) +CREATE MATERIALIZED VIEW batch_test_cagg +WITH (timescaledb.continuous) AS +SELECT + time_bucket('1 month'::interval, time) AS bucket, + count(*) as count +FROM test_data +GROUP BY bucket +WITH NO DATA; +-- Run incremental manual refresh, 1 bucket per batch +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'batch_test_cagg', NULL, '2024-12-01'::timestamptz, + options => '{"buckets_per_batch": 1}'::jsonb); +LOG: statement: CALL refresh_continuous_aggregate( + 'batch_test_cagg', NULL, '2024-12-01'::timestamptz, + options => '{"buckets_per_batch": 1}'::jsonb); +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_6" +LOG: inserted 0 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_6" +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +-- Verify that the materialization invalidation log has no entries other than +-- the boundary -/+ infinity rows. +SELECT materialization_id, + _timescaledb_functions.to_timestamp(lowest_modified_value) as low, + _timescaledb_functions.to_timestamp(greatest_modified_value) as high +FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log +WHERE materialization_id IN + (SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg + WHERE user_view_name = 'batch_test_cagg') + AND lowest_modified_value != -9223372036854775808 + AND greatest_modified_value != 9223372036854775807 +ORDER BY low; + materialization_id | low | high +--------------------+-----+------ + +-- Running again should be a no-op +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'batch_test_cagg', NULL, '2024-12-01'::timestamptz, + options => '{"buckets_per_batch": 1}'::jsonb); +LOG: statement: CALL refresh_continuous_aggregate( + 'batch_test_cagg', NULL, '2024-12-01'::timestamptz, + options => '{"buckets_per_batch": 1}'::jsonb); +NOTICE: continuous aggregate "batch_test_cagg" is already up-to-date +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +DROP TABLE test_data CASCADE; +NOTICE: drop cascades to 2 other objects +NOTICE: drop cascades to 2 other objects +------------------------------------------------------------------------------------------ +-- Test incremental manual refresh crashing between batches +------------------------------------------------------------------------------------------ +-- Rows processed by a batch should be visible immediately after it finishes. +-- Inject an error after batch 1 completes and observe the cagg state. +\c :TEST_DBNAME :ROLE_SUPERUSER +TRUNCATE _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log, _timescaledb_catalog.continuous_aggs_materialization_invalidation_log; +\c :TEST_DBNAME test_cagg_refresh_manual_user +SET timezone TO 'UTC'; +TRUNCATE conditions, conditions_by_day, conditions_by_day_atomic_refresh; +INSERT INTO conditions +SELECT + t, d, 10 +FROM + generate_series( + '2025-02-05 00:00:00+00', + '2025-03-05 00:00:00+00', + '1 hour'::interval) AS t, + generate_series(1,5) AS d; +-- Crash after batch 1 finishes +SELECT debug_waitpoint_enable('cagg_policy_batch_1_after_refresh'); + debug_waitpoint_enable +------------------------ + + +-- Cagg state before refresh starts +SELECT min(time_bucket), max(time_bucket) FROM conditions_by_day; + min | max +-----+----- + | + +SET client_min_messages TO LOG; +\set ON_ERROR_STOP 0 +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, '1 hour'::interval, + options => '{"buckets_per_batch": 10}'::jsonb); +LOG: statement: CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, '1 hour'::interval, + options => '{"buckets_per_batch": 10}'::jsonb); +LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" +LOG: inserted 25 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" +ERROR: error injected at debug point 'cagg_policy_batch_1_after_refresh' +\set ON_ERROR_STOP 1 +RESET client_min_messages; +LOG: statement: RESET client_min_messages; +-- Rows processed by batch 1 should be materialized +SELECT count(*) AS rows_after_refresh FROM conditions_by_day; + rows_after_refresh +-------------------- + 25 + +SELECT min(time_bucket), max(time_bucket) FROM conditions_by_day; + min | max +------------------------------+------------------------------ + Sat Mar 01 00:00:00 2025 UTC | Wed Mar 05 00:00:00 2025 UTC + +-- Registered ranges should be cleaned up after the crash +SELECT count(*) AS registered_ranges FROM _timescaledb_catalog.continuous_aggs_jobs_refresh_ranges; + registered_ranges +------------------- + 0 + +SELECT debug_waitpoint_release('cagg_policy_batch_1_after_refresh'); + debug_waitpoint_release +------------------------- + + +-- Process remaining invalidations and verify +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, '1 hour'::interval, + options => '{"buckets_per_batch": 10}'::jsonb); +-- Verify against an atomic refresh +CALL refresh_continuous_aggregate('conditions_by_day_atomic_refresh', NULL, '1 hour'::interval); +SELECT + count(*) > 0 AS has_diff +FROM + ((SELECT * FROM conditions_by_day_atomic_refresh ORDER BY 1, 2) + EXCEPT + (SELECT * FROM conditions_by_day ORDER BY 1, 2)) AS diff; + has_diff +---------- + f + +\c :TEST_DBNAME :ROLE_SUPERUSER +REASSIGN OWNED BY test_cagg_refresh_manual_user TO :ROLE_SUPERUSER; +REVOKE ALL ON SCHEMA public FROM test_cagg_refresh_manual_user; +DROP ROLE test_cagg_refresh_manual_user; diff --git a/tsl/test/expected/cagg_refresh_using_merge.out b/tsl/test/expected/cagg_refresh_using_merge.out index b9a5e2d76cf..67533c5912d 100644 --- a/tsl/test/expected/cagg_refresh_using_merge.out +++ b/tsl/test/expected/cagg_refresh_using_merge.out @@ -813,7 +813,8 @@ WITH NO DATA; SET client_min_messages TO LOG; CALL refresh_continuous_aggregate('conditions_daily', NULL, '2018-11-01 23:59:59-08'); LOG: statement: CALL refresh_continuous_aggregate('conditions_daily', NULL, '2018-11-01 23:59:59-08'); -LOG: inserted 5 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_22" +LOG: inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_22" +LOG: inserted 4 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_22" SELECT * FROM conditions_daily ORDER BY 1, 2, 3 NULLS LAST, 4 NULLS LAST, 5 NULLS LAST; LOG: statement: SELECT * FROM conditions_daily ORDER BY 1, 2, 3 NULLS LAST, 4 NULLS LAST, 5 NULLS LAST; bucket | location | avg | max | min @@ -843,7 +844,7 @@ LOG: statement: SELECT * FROM conditions_daily ORDER BY 1, 2, 3 NULLS LAST, 4 N -- All data should be in the materialization hypertable CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL); LOG: statement: CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL); -NOTICE: continuous aggregate "conditions_daily" is already up-to-date +LOG: inserted 0 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_22" SELECT * FROM conditions_daily ORDER BY 1, 2, 3 NULLS LAST, 4 NULLS LAST, 5 NULLS LAST; LOG: statement: SELECT * FROM conditions_daily ORDER BY 1, 2, 3 NULLS LAST, 4 NULLS LAST, 5 NULLS LAST; bucket | location | avg | max | min diff --git a/tsl/test/expected/cagg_watermark.out b/tsl/test/expected/cagg_watermark.out index c2ee59c29c2..6935a5b984c 100644 --- a/tsl/test/expected/cagg_watermark.out +++ b/tsl/test/expected/cagg_watermark.out @@ -479,9 +479,9 @@ CALL refresh_continuous_aggregate('chunks_1h', '2000-01-01', '2021-06-01'); Append (actual rows=3.00 loops=1) -> Append (actual rows=3.00 loops=1) -> Seq Scan on _hyper_8_17_chunk (actual rows=1.00 loops=1) - -> Seq Scan on _hyper_8_20_chunk (actual rows=1.00 loops=1) - -> Index Scan using _hyper_8_21_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_21_chunk (actual rows=1.00 loops=1) + -> Index Scan using _hyper_8_20_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_20_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < 'Wed Jul 31 18:00:00 2002 PDT'::timestamp with time zone) + -> Seq Scan on _hyper_8_21_chunk (actual rows=1.00 loops=1) -> HashAggregate (actual rows=0.00 loops=1) Group Key: time_bucket('@ 1 hour'::interval, _hyper_7_19_chunk."time"), _hyper_7_19_chunk.device -> Result (actual rows=0.00 loops=1) @@ -493,9 +493,9 @@ CALL refresh_continuous_aggregate('chunks_1h', '2000-01-01', '2021-06-01'); Append (actual rows=3.00 loops=1) -> Append (actual rows=3.00 loops=1) -> Seq Scan on _hyper_8_17_chunk (actual rows=1.00 loops=1) - -> Seq Scan on _hyper_8_20_chunk (actual rows=1.00 loops=1) - -> Index Scan using _hyper_8_21_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_21_chunk (actual rows=1.00 loops=1) + -> Index Scan using _hyper_8_20_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_20_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < 'Wed Jul 31 18:00:00 2002 PDT'::timestamp with time zone) + -> Seq Scan on _hyper_8_21_chunk (actual rows=1.00 loops=1) -> HashAggregate (actual rows=0.00 loops=1) Group Key: time_bucket('@ 1 hour'::interval, _hyper_7_19_chunk."time"), _hyper_7_19_chunk.device -> Result (actual rows=0.00 loops=1) @@ -512,9 +512,9 @@ SET timescaledb.enable_constraint_aware_append = OFF; Append (actual rows=3.00 loops=1) -> Append (actual rows=3.00 loops=1) -> Seq Scan on _hyper_8_17_chunk (actual rows=1.00 loops=1) - -> Seq Scan on _hyper_8_20_chunk (actual rows=1.00 loops=1) - -> Index Scan using _hyper_8_21_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_21_chunk (actual rows=1.00 loops=1) + -> Index Scan using _hyper_8_20_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_20_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < 'Wed Jul 31 18:00:00 2002 PDT'::timestamp with time zone) + -> Seq Scan on _hyper_8_21_chunk (actual rows=1.00 loops=1) -> HashAggregate (actual rows=0.00 loops=1) Group Key: time_bucket('@ 1 hour'::interval, _hyper_7_19_chunk."time"), _hyper_7_19_chunk.device -> Result (actual rows=0.00 loops=1) @@ -551,16 +551,16 @@ EXECUTE cagg_scan_1h; bucket | device | max ------------------------------+--------+----- Mon Jul 31 17:00:00 2000 PDT | 1 | 2 - Tue Jul 31 17:00:00 2001 PDT | 1 | 2 Wed Jul 31 17:00:00 2002 PDT | 1 | 2 + Tue Jul 31 17:00:00 2001 PDT | 1 | 2 Thu Jul 31 17:00:00 2003 PDT | 1 | 2 SELECT * FROM chunks_1h; bucket | device | max ------------------------------+--------+----- Mon Jul 31 17:00:00 2000 PDT | 1 | 2 - Tue Jul 31 17:00:00 2001 PDT | 1 | 2 Wed Jul 31 17:00:00 2002 PDT | 1 | 2 + Tue Jul 31 17:00:00 2001 PDT | 1 | 2 Thu Jul 31 17:00:00 2003 PDT | 1 | 2 INSERT INTO chunks VALUES ('2004-08-01 01:01:01+01', 1, 2); @@ -677,8 +677,8 @@ EXECUTE cagg_scan_1h; bucket | device | max ------------------------------+--------+----- Mon Jul 31 17:00:00 2000 PDT | 1 | 2 - Tue Jul 31 17:00:00 2001 PDT | 1 | 2 Wed Jul 31 17:00:00 2002 PDT | 1 | 2 + Tue Jul 31 17:00:00 2001 PDT | 1 | 2 Thu Jul 31 17:00:00 2003 PDT | 1 | 2 Sat Jul 31 17:00:00 2004 PDT | 1 | 2 Sun Jul 31 17:00:00 2005 PDT | 1 | 2 @@ -689,8 +689,8 @@ SELECT * FROM chunks_1h; bucket | device | max ------------------------------+--------+----- Mon Jul 31 17:00:00 2000 PDT | 1 | 2 - Tue Jul 31 17:00:00 2001 PDT | 1 | 2 Wed Jul 31 17:00:00 2002 PDT | 1 | 2 + Tue Jul 31 17:00:00 2001 PDT | 1 | 2 Thu Jul 31 17:00:00 2003 PDT | 1 | 2 Sat Jul 31 17:00:00 2004 PDT | 1 | 2 Sun Jul 31 17:00:00 2005 PDT | 1 | 2 @@ -704,8 +704,8 @@ EXECUTE cagg_scan_1h; bucket | device | max ------------------------------+--------+----- Mon Jul 31 17:00:00 2000 PDT | 1 | 2 - Tue Jul 31 17:00:00 2001 PDT | 1 | 2 Wed Jul 31 17:00:00 2002 PDT | 1 | 2 + Tue Jul 31 17:00:00 2001 PDT | 1 | 2 Thu Jul 31 17:00:00 2003 PDT | 1 | 2 Sat Jul 31 17:00:00 2004 PDT | 1 | 2 Sun Jul 31 17:00:00 2005 PDT | 1 | 2 @@ -716,8 +716,8 @@ SELECT * FROM chunks_1h; bucket | device | max ------------------------------+--------+----- Mon Jul 31 17:00:00 2000 PDT | 1 | 2 - Tue Jul 31 17:00:00 2001 PDT | 1 | 2 Wed Jul 31 17:00:00 2002 PDT | 1 | 2 + Tue Jul 31 17:00:00 2001 PDT | 1 | 2 Thu Jul 31 17:00:00 2003 PDT | 1 | 2 Sat Jul 31 17:00:00 2004 PDT | 1 | 2 Sun Jul 31 17:00:00 2005 PDT | 1 | 2 @@ -963,8 +963,8 @@ INSERT INTO chunks VALUES ('2014-01-01 01:01:01+01', 1, 2); CALL refresh_continuous_aggregate('chunks_1h', '2000-01-01', '2021-06-01'); :EXPLAIN_ANALYZE EXECUTE ht_scan_realtime_1h; --- QUERY PLAN --- - Append (actual rows=2.00 loops=1) - -> Custom Scan (ChunkAppend) on _materialized_hypertable_8 (actual rows=2.00 loops=1) + Append (actual rows=8.00 loops=1) + -> Custom Scan (ChunkAppend) on _materialized_hypertable_8 (actual rows=8.00 loops=1) Chunks excluded during startup: 0 -> Index Scan using _hyper_8_17_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_17_chunk (actual rows=0.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) @@ -982,17 +982,17 @@ CALL refresh_continuous_aggregate('chunks_1h', '2000-01-01', '2021-06-01'); Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) -> Index Scan using _hyper_8_31_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_31_chunk (actual rows=0.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_33_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_33_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_33_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_33_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_36_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_36_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_36_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_36_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_39_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_39_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_39_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_39_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_42_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_42_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_42_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_42_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_44_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_44_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_44_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_44_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_46_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_46_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_46_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_46_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) -> Index Scan using _hyper_8_48_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_48_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) @@ -1013,10 +1013,10 @@ INSERT INTO chunks VALUES ('2015-01-01 01:01:01+01', 1, 2); CALL refresh_continuous_aggregate('chunks_1h', '2000-01-01', '2021-06-01'); :EXPLAIN_ANALYZE SELECT device FROM chunks_1h; --- QUERY PLAN --- - Append (actual rows=3.00 loops=1) - -> Subquery Scan on "*SELECT* 1" (actual rows=3.00 loops=1) - -> Result (actual rows=3.00 loops=1) - -> Append (actual rows=3.00 loops=1) + Append (actual rows=9.00 loops=1) + -> Subquery Scan on "*SELECT* 1" (actual rows=9.00 loops=1) + -> Result (actual rows=9.00 loops=1) + -> Append (actual rows=9.00 loops=1) -> Seq Scan on _hyper_8_17_chunk (actual rows=0.00 loops=1) -> Seq Scan on _hyper_8_20_chunk (actual rows=0.00 loops=1) -> Seq Scan on _hyper_8_21_chunk (actual rows=0.00 loops=1) @@ -1025,12 +1025,12 @@ CALL refresh_continuous_aggregate('chunks_1h', '2000-01-01', '2021-06-01'); -> Seq Scan on _hyper_8_27_chunk (actual rows=0.00 loops=1) -> Seq Scan on _hyper_8_29_chunk (actual rows=0.00 loops=1) -> Seq Scan on _hyper_8_31_chunk (actual rows=0.00 loops=1) - -> Seq Scan on _hyper_8_33_chunk (actual rows=0.00 loops=1) - -> Seq Scan on _hyper_8_36_chunk (actual rows=0.00 loops=1) - -> Seq Scan on _hyper_8_39_chunk (actual rows=0.00 loops=1) - -> Seq Scan on _hyper_8_42_chunk (actual rows=0.00 loops=1) - -> Seq Scan on _hyper_8_44_chunk (actual rows=0.00 loops=1) - -> Seq Scan on _hyper_8_46_chunk (actual rows=0.00 loops=1) + -> Seq Scan on _hyper_8_33_chunk (actual rows=1.00 loops=1) + -> Seq Scan on _hyper_8_36_chunk (actual rows=1.00 loops=1) + -> Seq Scan on _hyper_8_39_chunk (actual rows=1.00 loops=1) + -> Seq Scan on _hyper_8_42_chunk (actual rows=1.00 loops=1) + -> Seq Scan on _hyper_8_44_chunk (actual rows=1.00 loops=1) + -> Seq Scan on _hyper_8_46_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_8_48_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_8_50_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_8_52_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_52_chunk (actual rows=1.00 loops=1) @@ -1055,8 +1055,8 @@ CREATE TABLE continuous_agg_test(time int, data int); -- Query without COALESCE - should not be optimized :EXPLAIN_ANALYZE (SELECT * FROM chunks_1h AS t1) UNION ALL (SELECT * from chunks_1h AS t2 WHERE _timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(:MAT_HT_ID_1H)) IS NOT NULL); --- QUERY PLAN --- - Append (actual rows=6.00 loops=1) - -> Custom Scan (ChunkAppend) on _materialized_hypertable_8 (actual rows=3.00 loops=1) + Append (actual rows=18.00 loops=1) + -> Custom Scan (ChunkAppend) on _materialized_hypertable_8 (actual rows=9.00 loops=1) Chunks excluded during startup: 0 -> Index Scan using _hyper_8_17_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_17_chunk (actual rows=0.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) @@ -1074,17 +1074,17 @@ CREATE TABLE continuous_agg_test(time int, data int); Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) -> Index Scan using _hyper_8_31_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_31_chunk (actual rows=0.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_33_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_33_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_33_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_33_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_36_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_36_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_36_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_36_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_39_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_39_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_39_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_39_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_42_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_42_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_42_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_42_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_44_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_44_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_44_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_44_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_46_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_46_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_46_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_46_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) -> Index Scan using _hyper_8_48_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_48_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) @@ -1100,10 +1100,10 @@ CREATE TABLE continuous_agg_test(time int, data int); Group Key: time_bucket('@ 1 hour'::interval, _hyper_7_51_chunk."time"), _hyper_7_51_chunk.device -> Index Scan using _hyper_7_51_chunk_chunks_time_idx on _hyper_7_51_chunk (actual rows=0.00 loops=1) Index Cond: ("time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Append (actual rows=3.00 loops=1) - -> Result (actual rows=3.00 loops=1) + -> Append (actual rows=9.00 loops=1) + -> Result (actual rows=9.00 loops=1) One-Time Filter: (_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)) IS NOT NULL) - -> Custom Scan (ChunkAppend) on _materialized_hypertable_8 _materialized_hypertable_8_1 (actual rows=3.00 loops=1) + -> Custom Scan (ChunkAppend) on _materialized_hypertable_8 _materialized_hypertable_8_1 (actual rows=9.00 loops=1) Chunks excluded during startup: 0 -> Index Scan using _hyper_8_17_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_17_chunk _hyper_8_17_chunk_1 (actual rows=0.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) @@ -1121,17 +1121,17 @@ CREATE TABLE continuous_agg_test(time int, data int); Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) -> Index Scan using _hyper_8_31_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_31_chunk _hyper_8_31_chunk_1 (actual rows=0.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_33_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_33_chunk _hyper_8_33_chunk_1 (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_33_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_33_chunk _hyper_8_33_chunk_1 (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_36_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_36_chunk _hyper_8_36_chunk_1 (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_36_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_36_chunk _hyper_8_36_chunk_1 (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_39_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_39_chunk _hyper_8_39_chunk_1 (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_39_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_39_chunk _hyper_8_39_chunk_1 (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_42_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_42_chunk _hyper_8_42_chunk_1 (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_42_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_42_chunk _hyper_8_42_chunk_1 (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_44_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_44_chunk _hyper_8_44_chunk_1 (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_44_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_44_chunk _hyper_8_44_chunk_1 (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_46_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_46_chunk _hyper_8_46_chunk_1 (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_46_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_46_chunk _hyper_8_46_chunk_1 (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) -> Index Scan using _hyper_8_48_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_48_chunk _hyper_8_48_chunk_1 (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) @@ -1154,10 +1154,10 @@ CREATE TABLE continuous_agg_test(time int, data int); :EXPLAIN_ANALYZE SELECT max(device) from chunks_1h; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) - -> Append (actual rows=3.00 loops=1) - -> Subquery Scan on "*SELECT* 1" (actual rows=3.00 loops=1) - -> Result (actual rows=3.00 loops=1) - -> Append (actual rows=3.00 loops=1) + -> Append (actual rows=9.00 loops=1) + -> Subquery Scan on "*SELECT* 1" (actual rows=9.00 loops=1) + -> Result (actual rows=9.00 loops=1) + -> Append (actual rows=9.00 loops=1) -> Seq Scan on _hyper_8_17_chunk (actual rows=0.00 loops=1) -> Seq Scan on _hyper_8_20_chunk (actual rows=0.00 loops=1) -> Seq Scan on _hyper_8_21_chunk (actual rows=0.00 loops=1) @@ -1166,12 +1166,12 @@ CREATE TABLE continuous_agg_test(time int, data int); -> Seq Scan on _hyper_8_27_chunk (actual rows=0.00 loops=1) -> Seq Scan on _hyper_8_29_chunk (actual rows=0.00 loops=1) -> Seq Scan on _hyper_8_31_chunk (actual rows=0.00 loops=1) - -> Seq Scan on _hyper_8_33_chunk (actual rows=0.00 loops=1) - -> Seq Scan on _hyper_8_36_chunk (actual rows=0.00 loops=1) - -> Seq Scan on _hyper_8_39_chunk (actual rows=0.00 loops=1) - -> Seq Scan on _hyper_8_42_chunk (actual rows=0.00 loops=1) - -> Seq Scan on _hyper_8_44_chunk (actual rows=0.00 loops=1) - -> Seq Scan on _hyper_8_46_chunk (actual rows=0.00 loops=1) + -> Seq Scan on _hyper_8_33_chunk (actual rows=1.00 loops=1) + -> Seq Scan on _hyper_8_36_chunk (actual rows=1.00 loops=1) + -> Seq Scan on _hyper_8_39_chunk (actual rows=1.00 loops=1) + -> Seq Scan on _hyper_8_42_chunk (actual rows=1.00 loops=1) + -> Seq Scan on _hyper_8_44_chunk (actual rows=1.00 loops=1) + -> Seq Scan on _hyper_8_46_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_8_48_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_8_50_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_8_52_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_52_chunk (actual rows=1.00 loops=1) @@ -1364,8 +1364,8 @@ UNION ALL WHERE chunks_1h.bucket >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(COALESCE(:MAT_HT_ID_1H, :MAT_HT_ID_1H))), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 day'::interval, chunks_1h.bucket)), chunks_1h.device; --- QUERY PLAN --- - Append (actual rows=3.00 loops=1) - -> Custom Scan (ChunkAppend) on _materialized_hypertable_8 (actual rows=3.00 loops=1) + Append (actual rows=9.00 loops=1) + -> Custom Scan (ChunkAppend) on _materialized_hypertable_8 (actual rows=9.00 loops=1) Chunks excluded during startup: 0 -> Index Scan using _hyper_8_17_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_17_chunk (actual rows=0.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) @@ -1383,17 +1383,17 @@ UNION ALL Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) -> Index Scan using _hyper_8_31_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_31_chunk (actual rows=0.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_33_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_33_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_33_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_33_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_36_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_36_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_36_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_36_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_39_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_39_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_39_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_39_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_42_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_42_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_42_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_42_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_44_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_44_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_44_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_44_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_46_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_46_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_46_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_46_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) -> Index Scan using _hyper_8_48_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_48_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) @@ -1431,8 +1431,8 @@ UNION ALL WHERE chunks_1h.bucket >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(COALESCE(:MAT_HT_ID_1H, :MAT_HT_ID_1H))), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 day'::interval, chunks_1h.bucket)), chunks_1h.device; --- QUERY PLAN --- - Append (actual rows=3.00 loops=1) - -> Custom Scan (ChunkAppend) on _materialized_hypertable_8 (actual rows=3.00 loops=1) + Append (actual rows=9.00 loops=1) + -> Custom Scan (ChunkAppend) on _materialized_hypertable_8 (actual rows=9.00 loops=1) Chunks excluded during startup: 0 -> Index Scan using _hyper_8_17_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_17_chunk (actual rows=0.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) @@ -1450,17 +1450,17 @@ UNION ALL Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) -> Index Scan using _hyper_8_31_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_31_chunk (actual rows=0.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_33_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_33_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_33_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_33_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_36_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_36_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_36_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_36_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_39_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_39_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_39_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_39_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_42_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_42_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_42_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_42_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_44_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_44_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_44_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_44_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) - -> Index Scan using _hyper_8_46_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_46_chunk (actual rows=0.00 loops=1) + -> Index Scan using _hyper_8_46_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_46_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) -> Index Scan using _hyper_8_48_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_48_chunk (actual rows=1.00 loops=1) Index Cond: (bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone)) @@ -1498,8 +1498,8 @@ UNION ALL WHERE chunks_1h.bucket >= COALESCE(COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(:MAT_HT_ID_1H)), '-infinity'::timestamp with time zone), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 day'::interval, chunks_1h.bucket)), chunks_1h.device; --- QUERY PLAN --- - Append (actual rows=3.00 loops=1) - -> Append (actual rows=3.00 loops=1) + Append (actual rows=9.00 loops=1) + -> Append (actual rows=9.00 loops=1) -> Seq Scan on _hyper_8_17_chunk (actual rows=0.00 loops=1) -> Seq Scan on _hyper_8_20_chunk (actual rows=0.00 loops=1) -> Seq Scan on _hyper_8_21_chunk (actual rows=0.00 loops=1) @@ -1508,12 +1508,12 @@ UNION ALL -> Seq Scan on _hyper_8_27_chunk (actual rows=0.00 loops=1) -> Seq Scan on _hyper_8_29_chunk (actual rows=0.00 loops=1) -> Seq Scan on _hyper_8_31_chunk (actual rows=0.00 loops=1) - -> Seq Scan on _hyper_8_33_chunk (actual rows=0.00 loops=1) - -> Seq Scan on _hyper_8_36_chunk (actual rows=0.00 loops=1) - -> Seq Scan on _hyper_8_39_chunk (actual rows=0.00 loops=1) - -> Seq Scan on _hyper_8_42_chunk (actual rows=0.00 loops=1) - -> Seq Scan on _hyper_8_44_chunk (actual rows=0.00 loops=1) - -> Seq Scan on _hyper_8_46_chunk (actual rows=0.00 loops=1) + -> Seq Scan on _hyper_8_33_chunk (actual rows=1.00 loops=1) + -> Seq Scan on _hyper_8_36_chunk (actual rows=1.00 loops=1) + -> Seq Scan on _hyper_8_39_chunk (actual rows=1.00 loops=1) + -> Seq Scan on _hyper_8_42_chunk (actual rows=1.00 loops=1) + -> Seq Scan on _hyper_8_44_chunk (actual rows=1.00 loops=1) + -> Seq Scan on _hyper_8_46_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_8_48_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_8_50_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_8_52_chunk__materialized_hypertable_8_bucket_idx on _hyper_8_52_chunk (actual rows=1.00 loops=1) diff --git a/tsl/test/expected/telemetry_stats.out b/tsl/test/expected/telemetry_stats.out index be01aef86c9..2be2efb3c2f 100644 --- a/tsl/test/expected/telemetry_stats.out +++ b/tsl/test/expected/telemetry_stats.out @@ -155,7 +155,7 @@ INSERT INTO hyper SELECT * FROM normal; INSERT INTO part SELECT * FROM normal; -CALL refresh_continuous_aggregate('contagg', NULL, NULL); +CALL refresh_continuous_aggregate('contagg', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); -- Reindex to avoid the dependency on the way the index is built (e.g. the caggs -- might get their rows inserted in different order during the refresh based on -- the underlying aggregation plan, and the index will be built differently, diff --git a/tsl/test/isolation/expected/cagg_cancel_kill_refresh.out b/tsl/test/isolation/expected/cagg_cancel_kill_refresh.out index d4a2c039c9b..54d8c9c8093 100644 --- a/tsl/test/isolation/expected/cagg_cancel_kill_refresh.out +++ b/tsl/test/isolation/expected/cagg_cancel_kill_refresh.out @@ -49,7 +49,7 @@ cagg_name|start_range|end_range starting permutation: wp1_enable r1_register_pid r1_refresh s1_registered_ranges k1_cancel wp1_release s1_registered_ranges step wp1_enable: - SELECT debug_waitpoint_enable('cagg_policy_batch_0_after_txn_1_wait'); + SELECT debug_waitpoint_enable('cagg_policy_batch_1_after_txn_1_wait'); debug_waitpoint_enable ---------------------- @@ -77,7 +77,7 @@ step k1_cancel: step r1_refresh: <... completed> ERROR: canceling statement due to user request step wp1_release: - SELECT debug_waitpoint_release('cagg_policy_batch_0_after_txn_1_wait'); + SELECT debug_waitpoint_release('cagg_policy_batch_1_after_txn_1_wait'); debug_waitpoint_release ----------------------- @@ -254,7 +254,7 @@ cagg_name|start_range|end_range starting permutation: wp1_enable tr2_register_pid tr2_refresh s1_registered_ranges t1_terminate wp1_release s1_registered_ranges r2_refresh s1_registered_ranges step wp1_enable: - SELECT debug_waitpoint_enable('cagg_policy_batch_0_after_txn_1_wait'); + SELECT debug_waitpoint_enable('cagg_policy_batch_1_after_txn_1_wait'); debug_waitpoint_enable ---------------------- @@ -286,7 +286,7 @@ server closed the connection unexpectedly before or while processing the request. step wp1_release: - SELECT debug_waitpoint_release('cagg_policy_batch_0_after_txn_1_wait'); + SELECT debug_waitpoint_release('cagg_policy_batch_1_after_txn_1_wait'); debug_waitpoint_release ----------------------- diff --git a/tsl/test/isolation/expected/cagg_concurrent_register.out b/tsl/test/isolation/expected/cagg_concurrent_register.out index 534aec5e3a1..c36e526728b 100644 --- a/tsl/test/isolation/expected/cagg_concurrent_register.out +++ b/tsl/test/isolation/expected/cagg_concurrent_register.out @@ -39,8 +39,8 @@ step s5_show_running_jobs: cagg_name| start_range| end_range|start_ts_utc |end_ts_utc ---------+----------------+----------------+------------------------+------------------------ -cagg_1 |1577836800000000|1578009600000000|Wed Jan 01 00:00:00 2020|Fri Jan 03 00:00:00 2020 -cagg_2 |1578009600000000|1578182400000000|Fri Jan 03 00:00:00 2020|Sun Jan 05 00:00:00 2020 +cagg_1 |1577980800000000|1578009600000000|Thu Jan 02 16:00:00 2020|Fri Jan 03 00:00:00 2020 +cagg_2 |1578153600000000|1578182400000000|Sat Jan 04 16:00:00 2020|Sun Jan 05 00:00:00 2020 step s4_release_before_process_cagg_invalidations: SELECT debug_waitpoint_release('before_process_cagg_invalidations_for_refresh_lock'); @@ -81,8 +81,6 @@ step s3_release_after_register: -- release lock on jobs_refresh_ranges table ROLLBACK; -step s2_run_cagg2_overlap_refresh: <... completed> -ERROR: could not refresh continuous aggregate "cagg_2" due to a concurrent refresh step s5_show_running_jobs: SELECT ca.user_view_name AS cagg_name, r.start_range, r.end_range, to_timestamp(r.start_range / 1000000) AT TIME ZONE 'UTC' AS start_ts_utc, @@ -93,7 +91,8 @@ step s5_show_running_jobs: cagg_name| start_range| end_range|start_ts_utc |end_ts_utc ---------+----------------+----------------+------------------------+------------------------ -cagg_2 |1577836800000000|1578355200000000|Wed Jan 01 00:00:00 2020|Tue Jan 07 00:00:00 2020 +cagg_2 |1578268800000000|1578355200000000|Mon Jan 06 00:00:00 2020|Tue Jan 07 00:00:00 2020 +cagg_2 |1578153600000000|1578182400000000|Sat Jan 04 16:00:00 2020|Sun Jan 05 00:00:00 2020 step s4_release_before_process_cagg_invalidations: SELECT debug_waitpoint_release('before_process_cagg_invalidations_for_refresh_lock'); @@ -103,6 +102,8 @@ debug_waitpoint_release step s1_run_cagg2_overlap_refresh: <... completed> +ERROR: could not refresh continuous aggregate "cagg_2" due to a concurrent refresh +step s2_run_cagg2_overlap_refresh: <... completed> starting permutation: s2_insert_new_data_2020 s3_lock_before_register s1_run_cagg2_nonoverlap_refresh s2_run_cagg2_overlap_refresh s4_enable_before_process_cagg_invalidations s3_release_after_register s5_show_running_jobs s4_release_before_process_cagg_invalidations step s2_insert_new_data_2020: @@ -144,7 +145,7 @@ step s5_show_running_jobs: cagg_name| start_range| end_range|start_ts_utc |end_ts_utc ---------+----------------+----------------+------------------------+------------------------ cagg_2 |1577836800000000|1577923200000000|Wed Jan 01 00:00:00 2020|Thu Jan 02 00:00:00 2020 -cagg_2 |1578009600000000|1578182400000000|Fri Jan 03 00:00:00 2020|Sun Jan 05 00:00:00 2020 +cagg_2 |1578153600000000|1578182400000000|Sat Jan 04 16:00:00 2020|Sun Jan 05 00:00:00 2020 step s4_release_before_process_cagg_invalidations: SELECT debug_waitpoint_release('before_process_cagg_invalidations_for_refresh_lock'); diff --git a/tsl/test/isolation/expected/cagg_hierarchical_concurrent_refresh.out b/tsl/test/isolation/expected/cagg_hierarchical_concurrent_refresh.out index 2e0ef6d71fd..df0af1d9921 100644 --- a/tsl/test/isolation/expected/cagg_hierarchical_concurrent_refresh.out +++ b/tsl/test/isolation/expected/cagg_hierarchical_concurrent_refresh.out @@ -509,7 +509,8 @@ step chk_hyper_invals: source |lowest |greatest -------+----------------------------+---------------------------- -cagg_6h|Thu Jan 01 00:00:00 2026 UTC|Sat Jan 03 18:00:00 2026 UTC +cagg_6h|Thu Jan 01 00:00:00 2026 UTC|Sat Jan 03 06:00:00 2026 UTC +cagg_6h|Sat Jan 03 12:00:00 2026 UTC|Sat Jan 03 18:00:00 2026 UTC step lock_L2_source: BEGIN; @@ -631,7 +632,8 @@ step chk_hyper_invals: source |lowest |greatest -------+----------------------------+---------------------------- -cagg_6h|Thu Jan 01 00:00:00 2026 UTC|Sat Jan 03 18:00:00 2026 UTC +cagg_6h|Thu Jan 01 00:00:00 2026 UTC|Sat Jan 03 06:00:00 2026 UTC +cagg_6h|Sat Jan 03 12:00:00 2026 UTC|Sat Jan 03 18:00:00 2026 UTC step lock_L2_source: BEGIN; @@ -908,7 +910,8 @@ step chk_hyper_invals: source |lowest |greatest -------+----------------------------+---------------------------- -cagg_6h|Thu Jan 01 00:00:00 2026 UTC|Sat Jan 03 18:00:00 2026 UTC +cagg_6h|Thu Jan 01 00:00:00 2026 UTC|Sat Jan 03 06:00:00 2026 UTC +cagg_6h|Sat Jan 03 12:00:00 2026 UTC|Sat Jan 03 18:00:00 2026 UTC step lock_L2_source: BEGIN; diff --git a/tsl/test/isolation/expected/cagg_incremental_concurrent.out b/tsl/test/isolation/expected/cagg_incremental_concurrent.out index eb9f25bae95..e4acd8bb438 100644 --- a/tsl/test/isolation/expected/cagg_incremental_concurrent.out +++ b/tsl/test/isolation/expected/cagg_incremental_concurrent.out @@ -1,4 +1,4 @@ -Parsed test spec with 7 sessions +Parsed test spec with 8 sessions starting permutation: wp_enable r1_run s1_refresh_ranges s1_count r2_refresh wp_release s1_count s1_check_duplicates step wp_enable: @@ -113,6 +113,11 @@ R3: LOG: statement: SET SESSION client_min_messages = 'LOG'; SET timescaledb.current_timestamp_mock TO '2026-04-01 00:30:00+00'; +R4: LOG: statement: + SET SESSION lock_timeout = '500ms'; + SET SESSION deadlock_timeout = '500ms'; + SET SESSION client_min_messages = 'LOG'; + step wp_enable: SELECT debug_waitpoint_enable('cagg_policy_batch_2_after_txn_1_wait'); @@ -241,6 +246,11 @@ R3: LOG: statement: SET SESSION client_min_messages = 'LOG'; SET timescaledb.current_timestamp_mock TO '2026-04-01 00:30:00+00'; +R4: LOG: statement: + SET SESSION lock_timeout = '500ms'; + SET SESSION deadlock_timeout = '500ms'; + SET SESSION client_min_messages = 'LOG'; + step wp_enable: SELECT debug_waitpoint_enable('cagg_policy_batch_2_after_txn_1_wait'); @@ -357,6 +367,11 @@ R3: LOG: statement: SET SESSION client_min_messages = 'LOG'; SET timescaledb.current_timestamp_mock TO '2026-04-01 00:30:00+00'; +R4: LOG: statement: + SET SESSION lock_timeout = '500ms'; + SET SESSION deadlock_timeout = '500ms'; + SET SESSION client_min_messages = 'LOG'; + step wp_enable: SELECT debug_waitpoint_enable('cagg_policy_batch_2_after_txn_1_wait'); @@ -448,6 +463,11 @@ R3: LOG: statement: SET SESSION client_min_messages = 'LOG'; SET timescaledb.current_timestamp_mock TO '2026-04-01 00:30:00+00'; +R4: LOG: statement: + SET SESSION lock_timeout = '500ms'; + SET SESSION deadlock_timeout = '500ms'; + SET SESSION client_min_messages = 'LOG'; + step wp_enable: SELECT debug_waitpoint_enable('cagg_policy_batch_2_after_txn_1_wait'); @@ -521,3 +541,203 @@ R3: LOG: continuous aggregate refresh (individual invalidation) on "sensor_hour R3: LOG: deleted 2 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_X" R3: LOG: inserted 2 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_X" step r3_run_oldest_first: <... completed> + +starting permutation: wp_enable r4_run s1_refresh_ranges s1_count r2_refresh wp_release s1_count s1_check_duplicates +R1: LOG: statement: + SET SESSION lock_timeout = '500ms'; + SET SESSION deadlock_timeout = '500ms'; + SET SESSION client_min_messages = 'LOG'; + +R2: LOG: statement: + SET SESSION lock_timeout = '500ms'; + SET SESSION deadlock_timeout = '500ms'; + SET SESSION client_min_messages = 'LOG'; + +R3: LOG: statement: + SET SESSION lock_timeout = '500ms'; + SET SESSION deadlock_timeout = '500ms'; + SET SESSION client_min_messages = 'LOG'; + SET timescaledb.current_timestamp_mock TO '2026-04-01 00:30:00+00'; + +R4: LOG: statement: + SET SESSION lock_timeout = '500ms'; + SET SESSION deadlock_timeout = '500ms'; + SET SESSION client_min_messages = 'LOG'; + +step wp_enable: + SELECT debug_waitpoint_enable('cagg_policy_batch_2_after_txn_1_wait'); + +debug_waitpoint_enable +---------------------- + + +R4: LOG: statement: + CALL refresh_continuous_aggregate( + 'cond_10', NULL, NULL, + options => jsonb_build_object('buckets_per_batch', 1)); + +R4: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_X" +R4: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_X" +step r4_run: + CALL refresh_continuous_aggregate( + 'cond_10', NULL, NULL, + options => jsonb_build_object('buckets_per_batch', 1)); + +step s1_refresh_ranges: + SELECT ca.user_view_name AS cagg_name, r.start_range, r.end_range + FROM _timescaledb_catalog.continuous_aggs_jobs_refresh_ranges r + JOIN _timescaledb_catalog.continuous_agg ca ON r.materialization_id = ca.mat_hypertable_id + ORDER BY ca.user_view_name; + +cagg_name|start_range|end_range +---------+-----------+--------- +cond_10 | 50| 60 + +step s1_count: + SELECT count(*) AS row_count FROM cond_10; + +row_count +--------- + 18 + +R2: LOG: statement: + -- Non-overlapping with R1's registered batch range [50, 60) + CALL refresh_continuous_aggregate('cond_10', 1, 50); + +R2: LOG: deleted 12 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_X" +R2: LOG: inserted 12 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_X" +step r2_refresh: + -- Non-overlapping with R1's registered batch range [50, 60) + CALL refresh_continuous_aggregate('cond_10', 1, 50); + +step wp_release: + SELECT debug_waitpoint_release('cagg_policy_batch_2_after_txn_1_wait'); + +debug_waitpoint_release +----------------------- + + +R4: LOG: deleted 3 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_X" +R4: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_X" +R4: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_X" +R4: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_X" +step r4_run: <... completed> +step s1_count: + SELECT count(*) AS row_count FROM cond_10; + +row_count +--------- + 21 + +step s1_check_duplicates: + -- Check for duplicate (bucket, device_id) rows in the materialization hypertable. + -- If duplicates exist, the same range was materialized twice without proper cleanup. + SELECT bucket, device_id, count(*) AS copies + FROM cond_10 + GROUP BY bucket, device_id + HAVING count(*) > 1 + ORDER BY bucket, device_id; + +bucket|device_id|copies +------+---------+------ + + +starting permutation: wp_enable r4_run s1_refresh_ranges r2_refresh_overlap wp_release s1_count s1_check_duplicates +R1: LOG: statement: + SET SESSION lock_timeout = '500ms'; + SET SESSION deadlock_timeout = '500ms'; + SET SESSION client_min_messages = 'LOG'; + +R2: LOG: statement: + SET SESSION lock_timeout = '500ms'; + SET SESSION deadlock_timeout = '500ms'; + SET SESSION client_min_messages = 'LOG'; + +R3: LOG: statement: + SET SESSION lock_timeout = '500ms'; + SET SESSION deadlock_timeout = '500ms'; + SET SESSION client_min_messages = 'LOG'; + SET timescaledb.current_timestamp_mock TO '2026-04-01 00:30:00+00'; + +R4: LOG: statement: + SET SESSION lock_timeout = '500ms'; + SET SESSION deadlock_timeout = '500ms'; + SET SESSION client_min_messages = 'LOG'; + +step wp_enable: + SELECT debug_waitpoint_enable('cagg_policy_batch_2_after_txn_1_wait'); + +debug_waitpoint_enable +---------------------- + + +R4: LOG: statement: + CALL refresh_continuous_aggregate( + 'cond_10', NULL, NULL, + options => jsonb_build_object('buckets_per_batch', 1)); + +R4: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_X" +R4: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_X" +step r4_run: + CALL refresh_continuous_aggregate( + 'cond_10', NULL, NULL, + options => jsonb_build_object('buckets_per_batch', 1)); + +step s1_refresh_ranges: + SELECT ca.user_view_name AS cagg_name, r.start_range, r.end_range + FROM _timescaledb_catalog.continuous_aggs_jobs_refresh_ranges r + JOIN _timescaledb_catalog.continuous_agg ca ON r.materialization_id = ca.mat_hypertable_id + ORDER BY ca.user_view_name; + +cagg_name|start_range|end_range +---------+-----------+--------- +cond_10 | 50| 60 + +R2: LOG: statement: + -- Overlapping with R1's registered batch range [50, 60) + CALL refresh_continuous_aggregate('cond_10', 40, 60); + +step r2_refresh_overlap: + -- Overlapping with R1's registered batch range [50, 60) + CALL refresh_continuous_aggregate('cond_10', 40, 60); + +ERROR: could not refresh continuous aggregate "cond_10" due to a concurrent refresh +step wp_release: + SELECT debug_waitpoint_release('cagg_policy_batch_2_after_txn_1_wait'); + +debug_waitpoint_release +----------------------- + + +R4: LOG: deleted 3 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_X" +R4: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_X" +R4: LOG: deleted 3 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_X" +R4: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_X" +R4: LOG: deleted 3 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_X" +R4: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_X" +R4: LOG: deleted 3 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_X" +R4: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_X" +R4: LOG: deleted 3 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_X" +R4: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_X" +R4: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_X" +R4: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_X" +step r4_run: <... completed> +step s1_count: + SELECT count(*) AS row_count FROM cond_10; + +row_count +--------- + 21 + +step s1_check_duplicates: + -- Check for duplicate (bucket, device_id) rows in the materialization hypertable. + -- If duplicates exist, the same range was materialized twice without proper cleanup. + SELECT bucket, device_id, count(*) AS copies + FROM cond_10 + GROUP BY bucket, device_id + HAVING count(*) > 1 + ORDER BY bucket, device_id; + +bucket|device_id|copies +------+---------+------ + diff --git a/tsl/test/isolation/expected/cagg_refresh_cleanup_register.out b/tsl/test/isolation/expected/cagg_refresh_cleanup_register.out index 19102a98690..4c39ffdf81f 100644 --- a/tsl/test/isolation/expected/cagg_refresh_cleanup_register.out +++ b/tsl/test/isolation/expected/cagg_refresh_cleanup_register.out @@ -1,6 +1,6 @@ Parsed test spec with 9 sessions -starting permutation: WP_mat_enable R2_refresh L1_lock WP_mat_disable R3_refresh R4_refresh check_locks check_jobs L1_unlock check_locks check_jobs +starting permutation: WP_mat_enable R2_refresh L1_lock WP_mat_disable WP_enable_after_refresh R3_refresh R4_refresh check_locks check_jobs L1_unlock WP_disable_after_refresh check_locks check_jobs step WP_mat_enable: SELECT debug_waitpoint_enable('after_process_cagg_materializations'); debug_waitpoint_enable ---------------------- @@ -19,6 +19,11 @@ debug_waitpoint_release ----------------------- +step WP_enable_after_refresh: SELECT debug_waitpoint_enable('after_cagg_refresh_window'); +debug_waitpoint_enable +---------------------- + + step R3_refresh: CALL refresh_continuous_aggregate('cond_daily', '2026-02-15', '2026-03-15'); @@ -50,11 +55,16 @@ step check_jobs: user_view_name|start_time |end_time --------------+----------------------------+---------------------------- -cond_daily |Mon Jan 05 16:00:00 2026 PST|Sat Feb 14 16:00:00 2026 PST +cond_daily |Sun Feb 08 16:00:00 2026 PST|Sat Feb 14 16:00:00 2026 PST step L1_unlock: COMMIT; +step WP_disable_after_refresh: SELECT debug_waitpoint_release('after_cagg_refresh_window'); +debug_waitpoint_release +----------------------- + + step R2_refresh: <... completed> step R3_refresh: <... completed> step R4_refresh: <... completed> @@ -81,7 +91,7 @@ user_view_name|start_time|end_time --------------+----------+-------- -starting permutation: WP_mat_enable R2_refresh L1_lock WP_mat_disable R3_refresh R4_overlapping_refresh check_locks check_jobs L1_unlock check_locks check_jobs +starting permutation: WP_mat_enable R2_refresh L1_lock WP_mat_disable WP_after_register_enable R3_refresh R4_overlapping_refresh check_locks check_jobs L1_unlock WP_after_register_disable check_locks check_jobs step WP_mat_enable: SELECT debug_waitpoint_enable('after_process_cagg_materializations'); debug_waitpoint_enable ---------------------- @@ -100,11 +110,16 @@ debug_waitpoint_release ----------------------- +step WP_after_register_enable: SELECT debug_waitpoint_enable('cagg_refresh_after_register'); +debug_waitpoint_enable +---------------------- + + step R3_refresh: CALL refresh_continuous_aggregate('cond_daily', '2026-02-15', '2026-03-15'); step R4_overlapping_refresh: - CALL refresh_continuous_aggregate('cond_daily', '2026-02-15', '2026-03-05'); + CALL refresh_continuous_aggregate('cond_daily', '2026-03-01', '2026-03-15'); step check_locks: SELECT l.mode, l.granted @@ -131,11 +146,16 @@ step check_jobs: user_view_name|start_time |end_time --------------+----------------------------+---------------------------- -cond_daily |Mon Jan 05 16:00:00 2026 PST|Sat Feb 14 16:00:00 2026 PST +cond_daily |Sun Feb 08 16:00:00 2026 PST|Sat Feb 14 16:00:00 2026 PST step L1_unlock: COMMIT; +step WP_after_register_disable: SELECT debug_waitpoint_release('cagg_refresh_after_register'); +debug_waitpoint_release +----------------------- + + step R2_refresh: <... completed> step R3_refresh: <... completed> step R4_overlapping_refresh: <... completed> @@ -183,7 +203,7 @@ step check_jobs: user_view_name|start_time |end_time --------------+----------------------------+---------------------------- -cond_daily |Sun Feb 15 16:00:00 2026 PST|Sat Mar 14 17:00:00 2026 PDT +cond_daily |Sat Mar 07 16:00:00 2026 PST|Sat Mar 14 17:00:00 2026 PDT step A1_revoke_perm: REVOKE SELECT on conditions FROM cagg_user; @@ -228,7 +248,7 @@ step check_jobs: user_view_name|start_time |end_time --------------+----------------------------+---------------------------- -cond_daily |Sun Feb 15 16:00:00 2026 PST|Sat Mar 14 17:00:00 2026 PDT +cond_daily |Sat Mar 07 16:00:00 2026 PST|Sat Mar 14 17:00:00 2026 PDT step A1_revoke_mat_perm: DO $$ @@ -363,7 +383,7 @@ step check_jobs: user_view_name|start_time |end_time --------------+----------------------------+---------------------------- -cond_daily |Mon Jan 05 16:00:00 2026 PST|Sat Mar 14 17:00:00 2026 PDT +cond_daily |Tue Mar 10 17:00:00 2026 PDT|Sat Mar 14 17:00:00 2026 PDT step K1_terminate: DO $$ diff --git a/tsl/test/isolation/specs/cagg_cancel_kill_refresh.spec b/tsl/test/isolation/specs/cagg_cancel_kill_refresh.spec index 601519fcadb..8da8d00d03b 100644 --- a/tsl/test/isolation/specs/cagg_cancel_kill_refresh.spec +++ b/tsl/test/isolation/specs/cagg_cancel_kill_refresh.spec @@ -101,11 +101,11 @@ step "wp0_release" session "WP1" step "wp1_enable" { - SELECT debug_waitpoint_enable('cagg_policy_batch_0_after_txn_1_wait'); + SELECT debug_waitpoint_enable('cagg_policy_batch_1_after_txn_1_wait'); } step "wp1_release" { - SELECT debug_waitpoint_release('cagg_policy_batch_0_after_txn_1_wait'); + SELECT debug_waitpoint_release('cagg_policy_batch_1_after_txn_1_wait'); } # Waitpoint after txn 2 (cagg invalidations processed) diff --git a/tsl/test/isolation/specs/cagg_incremental_concurrent.spec b/tsl/test/isolation/specs/cagg_incremental_concurrent.spec index 98f3e451577..e48b7c7a89a 100644 --- a/tsl/test/isolation/specs/cagg_incremental_concurrent.spec +++ b/tsl/test/isolation/specs/cagg_incremental_concurrent.spec @@ -295,6 +295,23 @@ step "r3_run_oldest_first" $$; } +# Session for batched manual refresh via JSONB options. Goes through the same +# continuous_agg_refresh_single_window path as the policy, so it pauses at the +# same waitpoint. +session "R4" +setup +{ + SET SESSION lock_timeout = '500ms'; + SET SESSION deadlock_timeout = '500ms'; + SET SESSION client_min_messages = 'LOG'; +} +step "r4_run" +{ + CALL refresh_continuous_aggregate( + 'cond_10', NULL, NULL, + options => jsonb_build_object('buckets_per_batch', 1)); +} + #insert data to create invalidations when refresh is stopped after batch 1 session "I2" step "i2_insert" @@ -325,4 +342,14 @@ permutation "wp_enable" "r1_run"("wp_enable") "s1_refresh_ranges" "r2_refresh_ov permutation "wp_enable" "r3_run"("wp_enable") "i2_insert" "wp_release" # Test 5: Same as test 5, but the policy refreshes oldest batch first -permutation "wp_enable" "r3_run_oldest_first"("wp_enable") "i2_insert" "wp_release" \ No newline at end of file +permutation "wp_enable" "r3_run_oldest_first"("wp_enable") "i2_insert" "wp_release" + +# Test 6: Manual incremental refresh pauses after batch 1. +# A non-overlapping atomic manual refresh proceeds successfully +# Mirrors test 1 but with batched manual refresh on the paused side instead of policy. +permutation "wp_enable" "r4_run"("wp_enable") "s1_refresh_ranges" "s1_count" "r2_refresh" "wp_release" "s1_count" "s1_check_duplicates" + +# Test 7: Manual incremental refresh pauses after batch 1. +# An overlapping atomic manual refresh must error with "concurrent refresh". +# Mirrors test 3 with batched manual refresh on the paused side. +permutation "wp_enable" "r4_run"("wp_enable") "s1_refresh_ranges" "r2_refresh_overlap" "wp_release" "s1_count" "s1_check_duplicates" \ No newline at end of file diff --git a/tsl/test/isolation/specs/cagg_refresh_cleanup_register.spec b/tsl/test/isolation/specs/cagg_refresh_cleanup_register.spec index 92f235dbaf7..c49c1c6087e 100644 --- a/tsl/test/isolation/specs/cagg_refresh_cleanup_register.spec +++ b/tsl/test/isolation/specs/cagg_refresh_cleanup_register.spec @@ -74,6 +74,8 @@ step "WP_before_txn2_start_enable" { SELECT debug_waitpoint_enable('cagg_refres step "WP_before_txn2_start_disable" { SELECT debug_waitpoint_release('cagg_refresh_after_register'); } step "WP_after_register_enable" { SELECT debug_waitpoint_enable('cagg_refresh_after_register'); } step "WP_after_register_disable" { SELECT debug_waitpoint_release('cagg_refresh_after_register'); } +step "WP_enable_after_refresh" { SELECT debug_waitpoint_enable('after_cagg_refresh_window'); } +step "WP_disable_after_refresh" { SELECT debug_waitpoint_release('after_cagg_refresh_window'); } # Session K1: terminate R1's backend so its PID becomes dead in the # registration table, then wait until the process is gone. @@ -123,7 +125,7 @@ step "R4_refresh" { CALL refresh_continuous_aggregate('cond_daily', '2026-03-15', '2026-03-30'); } step "R4_overlapping_refresh" { - CALL refresh_continuous_aggregate('cond_daily', '2026-02-15', '2026-03-05'); + CALL refresh_continuous_aggregate('cond_daily', '2026-03-01', '2026-03-15'); } session "A1" @@ -220,11 +222,11 @@ step "P1_run_policy" { # Two refreshes wait for registration, one waits for cleanup before exiting. All blocked on an AccessExclusiveLock on continuous_aggs_jobs_refresh_ranges. # None of those refreshes overlaps, so all should succeed. -permutation "WP_mat_enable" "R2_refresh" "L1_lock" "WP_mat_disable" "R3_refresh" "R4_refresh" "check_locks" "check_jobs" "L1_unlock" "check_locks" "check_jobs" +permutation "WP_mat_enable" "R2_refresh" "L1_lock" "WP_mat_disable" "WP_enable_after_refresh" "R3_refresh"("R2_refresh") "R4_refresh"("R3_refresh") "check_locks" "check_jobs" "L1_unlock" "WP_disable_after_refresh" "check_locks" "check_jobs" # Two refreshes wait for registration, one waits for cleanup before exiting. All blocked on an AccessExclusiveLock on continuous_aggs_jobs_refresh_ranges. # Refreshes waiting for registration overlap with each other, so one should fail. -permutation "WP_mat_enable" "R2_refresh" "L1_lock" "WP_mat_disable" "R3_refresh" "R4_overlapping_refresh" "check_locks" "check_jobs" "L1_unlock" "check_locks" "check_jobs" +permutation "WP_mat_enable" "R2_refresh" "L1_lock" "WP_mat_disable" "WP_after_register_enable" "R3_refresh"("R2_refresh") "R4_overlapping_refresh"("R3_refresh") "check_locks" "check_jobs" "L1_unlock" "WP_after_register_disable" "check_locks" "check_jobs" ## Refresh registers . But fails in txn2. Gets into catch block. Cleanup should succeed permutation "WP_before_txn2_start_enable" "R3_refresh" "check_jobs" "A1_revoke_perm" "WP_before_txn2_start_disable"("A1_revoke_perm") "check_jobs" diff --git a/tsl/test/shared/expected/constify_now-15.out b/tsl/test/shared/expected/constify_now-15.out index 7e4f202db01..0725ea404aa 100644 --- a/tsl/test/shared/expected/constify_now-15.out +++ b/tsl/test/shared/expected/constify_now-15.out @@ -588,11 +588,11 @@ CALL refresh_continuous_aggregate('cagg_now_view', NULL, '2004-01-01'); --- QUERY PLAN --- Append -> Append + -> Index Scan using _hyper_X_X_chunk__materialized_hypertable_X_bucket_idx on _hyper_X_X_chunk + Index Cond: (bucket < 'Wed Jan 01 01:00:00 2003 UTC'::timestamp with time zone) -> Seq Scan on _hyper_X_X_chunk -> Seq Scan on _hyper_X_X_chunk -> Seq Scan on _hyper_X_X_chunk - -> Index Scan using _hyper_X_X_chunk__materialized_hypertable_X_bucket_idx on _hyper_X_X_chunk - Index Cond: (bucket < 'Wed Jan 01 01:00:00 2003 UTC'::timestamp with time zone) -> Finalize HashAggregate Group Key: (time_bucket('@ 1 day'::interval, cagg_now_test."time")), cagg_now_test.device -> Append @@ -713,10 +713,10 @@ CALL refresh_continuous_aggregate('cagg_hierarch_l2', NULL, '2004-01-01'); --- QUERY PLAN --- Append -> Append - -> Index Scan using _hyper_X_X_chunk__materialized_hypertable_X_bucket_idx on _hyper_X_X_chunk - Index Cond: ((bucket < 'Wed Jan 01 01:00:00 2003 UTC'::timestamp with time zone) AND (bucket >= (ts_now_mock() - '@ 1 year'::interval))) -> Index Scan using _hyper_X_X_chunk__materialized_hypertable_X_bucket_idx on _hyper_X_X_chunk Index Cond: (bucket >= (ts_now_mock() - '@ 1 year'::interval)) + -> Index Scan using _hyper_X_X_chunk__materialized_hypertable_X_bucket_idx on _hyper_X_X_chunk + Index Cond: ((bucket < 'Wed Jan 01 01:00:00 2003 UTC'::timestamp with time zone) AND (bucket >= (ts_now_mock() - '@ 1 year'::interval))) -> HashAggregate Group Key: time_bucket('@ 1 mon'::interval, (time_bucket('@ 1 day'::interval, cagg_hierarch_test."time"))), cagg_hierarch_test.device -> Result diff --git a/tsl/test/shared/expected/constify_now-16.out b/tsl/test/shared/expected/constify_now-16.out index 7e4f202db01..0725ea404aa 100644 --- a/tsl/test/shared/expected/constify_now-16.out +++ b/tsl/test/shared/expected/constify_now-16.out @@ -588,11 +588,11 @@ CALL refresh_continuous_aggregate('cagg_now_view', NULL, '2004-01-01'); --- QUERY PLAN --- Append -> Append + -> Index Scan using _hyper_X_X_chunk__materialized_hypertable_X_bucket_idx on _hyper_X_X_chunk + Index Cond: (bucket < 'Wed Jan 01 01:00:00 2003 UTC'::timestamp with time zone) -> Seq Scan on _hyper_X_X_chunk -> Seq Scan on _hyper_X_X_chunk -> Seq Scan on _hyper_X_X_chunk - -> Index Scan using _hyper_X_X_chunk__materialized_hypertable_X_bucket_idx on _hyper_X_X_chunk - Index Cond: (bucket < 'Wed Jan 01 01:00:00 2003 UTC'::timestamp with time zone) -> Finalize HashAggregate Group Key: (time_bucket('@ 1 day'::interval, cagg_now_test."time")), cagg_now_test.device -> Append @@ -713,10 +713,10 @@ CALL refresh_continuous_aggregate('cagg_hierarch_l2', NULL, '2004-01-01'); --- QUERY PLAN --- Append -> Append - -> Index Scan using _hyper_X_X_chunk__materialized_hypertable_X_bucket_idx on _hyper_X_X_chunk - Index Cond: ((bucket < 'Wed Jan 01 01:00:00 2003 UTC'::timestamp with time zone) AND (bucket >= (ts_now_mock() - '@ 1 year'::interval))) -> Index Scan using _hyper_X_X_chunk__materialized_hypertable_X_bucket_idx on _hyper_X_X_chunk Index Cond: (bucket >= (ts_now_mock() - '@ 1 year'::interval)) + -> Index Scan using _hyper_X_X_chunk__materialized_hypertable_X_bucket_idx on _hyper_X_X_chunk + Index Cond: ((bucket < 'Wed Jan 01 01:00:00 2003 UTC'::timestamp with time zone) AND (bucket >= (ts_now_mock() - '@ 1 year'::interval))) -> HashAggregate Group Key: time_bucket('@ 1 mon'::interval, (time_bucket('@ 1 day'::interval, cagg_hierarch_test."time"))), cagg_hierarch_test.device -> Result diff --git a/tsl/test/shared/expected/constify_now-17.out b/tsl/test/shared/expected/constify_now-17.out index 7e4f202db01..0725ea404aa 100644 --- a/tsl/test/shared/expected/constify_now-17.out +++ b/tsl/test/shared/expected/constify_now-17.out @@ -588,11 +588,11 @@ CALL refresh_continuous_aggregate('cagg_now_view', NULL, '2004-01-01'); --- QUERY PLAN --- Append -> Append + -> Index Scan using _hyper_X_X_chunk__materialized_hypertable_X_bucket_idx on _hyper_X_X_chunk + Index Cond: (bucket < 'Wed Jan 01 01:00:00 2003 UTC'::timestamp with time zone) -> Seq Scan on _hyper_X_X_chunk -> Seq Scan on _hyper_X_X_chunk -> Seq Scan on _hyper_X_X_chunk - -> Index Scan using _hyper_X_X_chunk__materialized_hypertable_X_bucket_idx on _hyper_X_X_chunk - Index Cond: (bucket < 'Wed Jan 01 01:00:00 2003 UTC'::timestamp with time zone) -> Finalize HashAggregate Group Key: (time_bucket('@ 1 day'::interval, cagg_now_test."time")), cagg_now_test.device -> Append @@ -713,10 +713,10 @@ CALL refresh_continuous_aggregate('cagg_hierarch_l2', NULL, '2004-01-01'); --- QUERY PLAN --- Append -> Append - -> Index Scan using _hyper_X_X_chunk__materialized_hypertable_X_bucket_idx on _hyper_X_X_chunk - Index Cond: ((bucket < 'Wed Jan 01 01:00:00 2003 UTC'::timestamp with time zone) AND (bucket >= (ts_now_mock() - '@ 1 year'::interval))) -> Index Scan using _hyper_X_X_chunk__materialized_hypertable_X_bucket_idx on _hyper_X_X_chunk Index Cond: (bucket >= (ts_now_mock() - '@ 1 year'::interval)) + -> Index Scan using _hyper_X_X_chunk__materialized_hypertable_X_bucket_idx on _hyper_X_X_chunk + Index Cond: ((bucket < 'Wed Jan 01 01:00:00 2003 UTC'::timestamp with time zone) AND (bucket >= (ts_now_mock() - '@ 1 year'::interval))) -> HashAggregate Group Key: time_bucket('@ 1 mon'::interval, (time_bucket('@ 1 day'::interval, cagg_hierarch_test."time"))), cagg_hierarch_test.device -> Result diff --git a/tsl/test/shared/expected/constify_now-18.out b/tsl/test/shared/expected/constify_now-18.out index 6f193e5c590..25fb86957c0 100644 --- a/tsl/test/shared/expected/constify_now-18.out +++ b/tsl/test/shared/expected/constify_now-18.out @@ -578,11 +578,11 @@ CALL refresh_continuous_aggregate('cagg_now_view', NULL, '2004-01-01'); --- QUERY PLAN --- Append -> Append + -> Index Scan using _hyper_X_X_chunk__materialized_hypertable_X_bucket_idx on _hyper_X_X_chunk + Index Cond: (bucket < 'Wed Jan 01 01:00:00 2003 UTC'::timestamp with time zone) -> Seq Scan on _hyper_X_X_chunk -> Seq Scan on _hyper_X_X_chunk -> Seq Scan on _hyper_X_X_chunk - -> Index Scan using _hyper_X_X_chunk__materialized_hypertable_X_bucket_idx on _hyper_X_X_chunk - Index Cond: (bucket < 'Wed Jan 01 01:00:00 2003 UTC'::timestamp with time zone) -> Finalize HashAggregate Group Key: (time_bucket('@ 1 day'::interval, cagg_now_test."time")), cagg_now_test.device -> Append @@ -703,10 +703,10 @@ CALL refresh_continuous_aggregate('cagg_hierarch_l2', NULL, '2004-01-01'); --- QUERY PLAN --- Append -> Append - -> Index Scan using _hyper_X_X_chunk__materialized_hypertable_X_bucket_idx on _hyper_X_X_chunk - Index Cond: ((bucket < 'Wed Jan 01 01:00:00 2003 UTC'::timestamp with time zone) AND (bucket >= (ts_now_mock() - '@ 1 year'::interval))) -> Index Scan using _hyper_X_X_chunk__materialized_hypertable_X_bucket_idx on _hyper_X_X_chunk Index Cond: (bucket >= (ts_now_mock() - '@ 1 year'::interval)) + -> Index Scan using _hyper_X_X_chunk__materialized_hypertable_X_bucket_idx on _hyper_X_X_chunk + Index Cond: ((bucket < 'Wed Jan 01 01:00:00 2003 UTC'::timestamp with time zone) AND (bucket >= (ts_now_mock() - '@ 1 year'::interval))) -> HashAggregate Group Key: time_bucket('@ 1 mon'::interval, (time_bucket('@ 1 day'::interval, cagg_hierarch_test."time"))), cagg_hierarch_test.device -> Result diff --git a/tsl/test/sql/CMakeLists.txt b/tsl/test/sql/CMakeLists.txt index d1f31fc94dd..a5893f9dad9 100644 --- a/tsl/test/sql/CMakeLists.txt +++ b/tsl/test/sql/CMakeLists.txt @@ -136,6 +136,7 @@ if(CMAKE_BUILD_TYPE MATCHES Debug) cagg_policy_concurrent.sql cagg_policy_incremental.sql cagg_refresh_cleanup.sql + cagg_refresh_incremental.sql chunk_column_stats.sql compress_bgw_reorder_drop_chunks.sql compress_bloom_legacy_v1.sql diff --git a/tsl/test/sql/cagg_direct_compress.sql b/tsl/test/sql/cagg_direct_compress.sql index acc2d122993..a6d918bd366 100644 --- a/tsl/test/sql/cagg_direct_compress.sql +++ b/tsl/test/sql/cagg_direct_compress.sql @@ -30,7 +30,8 @@ FROM conditions GROUP BY 1, 2 WITH NO DATA; -CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL); +-- Setting buckets_per_batch to a high value to bypass "disabling direct compress because of too small batch size" situation +CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_hourly') chunk; -- Enable columnstore @@ -39,14 +40,14 @@ ALTER MATERIALIZED VIEW conditions_hourly SET (timescaledb.compress); -- Enable direct compress on cagg refresh SET timescaledb.enable_direct_compress_on_cagg_refresh TO on; -CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL); +CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_hourly') chunk; -- Backfill data and refresh again WITHOUT direct compress INSERT INTO conditions SELECT t, d::text, 1, 1 FROM generate_series('2025-12-15 00:00:00+00'::timestamptz - interval '1 year', '2025-12-15 00:00:00+00'::timestamptz, interval '1 hour') AS t, generate_series(1, 10) AS d; SET timescaledb.enable_direct_compress_on_cagg_refresh TO off; -CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL); +CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_hourly') chunk; -- Recompress all uncompressed chunks @@ -57,7 +58,7 @@ SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks INSERT INTO conditions SELECT t, d::text, 1, 1 FROM generate_series('2025-12-15 00:00:00+00'::timestamptz - interval '1 year', '2025-12-15 00:00:00+00'::timestamptz, interval '1 hour') AS t, generate_series(1, 10) AS d; SET timescaledb.enable_direct_compress_on_cagg_refresh TO on; -CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL); +CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_hourly') chunk; -- Cleanup @@ -85,11 +86,11 @@ SELECT t, d::text, 1, 1 FROM generate_series('2025-12-15 00:00:00+00'::timestamp SET timescaledb.enable_direct_compress_on_cagg_refresh TO on; -- Refresh the base CAgg -CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL); +CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_hourly') chunk; -- Refresh the hierarchical CAgg -CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL); +CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_daily') chunk; -- Produce some invalidations for the base CAgg @@ -97,18 +98,18 @@ INSERT INTO conditions SELECT t, d::text, 1, 1 FROM generate_series('2025-12-15 00:00:00+00'::timestamptz - interval '1 year', '2025-12-15 00:00:00+00'::timestamptz, interval '1 hour') AS t, generate_series(1, 10) AS d; -- Refresh the base CAgg -CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL); +CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_hourly') chunk; -- Refreshing again the base CAgg is a no-op since everything is up to date -CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL); +CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); -- Refresh the hierarchical CAgg with invalidations procuded by the base CAgg -CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL); +CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_daily') chunk; -- Refreshing again the hierarchical CAgg is a no-op since everything is up to date -CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL); +CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); -- Tests with custom segmentby and orderby CREATE MATERIALIZED VIEW conditions_weekly @@ -126,7 +127,7 @@ WITH NO DATA; ALTER MATERIALIZED VIEW conditions_weekly SET (timescaledb.compress_segmentby = 'device_id, location_id', timescaledb.compress_orderby = 'max, min, bucket DESC'); -CALL refresh_continuous_aggregate('conditions_weekly', NULL, NULL); +CALL refresh_continuous_aggregate('conditions_weekly', NULL, NULL, options => '{"buckets_per_batch": 10000}'::jsonb); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_weekly') chunk; -- Test GROUP BY ROLLUP on compressed continuous aggregate (issue #9520) diff --git a/tsl/test/sql/cagg_invalidation.sql b/tsl/test/sql/cagg_invalidation.sql index 84cbf14ef63..f543798c0f6 100644 --- a/tsl/test/sql/cagg_invalidation.sql +++ b/tsl/test/sql/cagg_invalidation.sql @@ -1017,6 +1017,13 @@ SET timezone = 'UTC'; call refresh_continuous_aggregate ('test_cagg','2023-12-29 15:00:00', '2026-01-28 15:00:00'); call refresh_continuous_aggregate ('test_cagg_1d_offset','2023-12-29 18:00:00', '2024-01-01 18:00:00'); +-- test_cagg uses 1-month buckets with buckets_per_batch=10. The inscribed window +-- [2024-01-01, 2026-01-01) spans 24 months, producing two batches: +-- [2024-01-01, 2024-11-01) and [2024-11-01, 2025-09-01). +-- Data only exists through Dec 2024, so the split function finds no chunks past +-- Dec 2024 and stops at Sep 2025 (end of batch 2). The range [2025-09-01, 2026-01-01) +-- has no data and is left unprocessed, so the upper residual in the mat_inval_log +-- is 2025-09-01 rather than 2026-01-01 as it would be with single-pass refresh. SELECT materialization_id, _timescaledb_functions.to_timestamp(lowest_modified_value) as low, _timescaledb_functions.to_timestamp(greatest_modified_value) as high @@ -1043,9 +1050,10 @@ WHERE materialization_id = ( ) ORDER BY lowest_modified_value, greatest_modified_value; - ---now do the same refresh again, it should say the cagg is already up to date +-- Refresh the same range again, [2025-09-01 ─ 2026-01-01) was left. +-- This time refresh will go into the single batch path, so it will process the whole range. CALL refresh_continuous_aggregate ('test_cagg','2023-12-29 15:00:00', '2026-01-28 15:00:00'); +--Do the same refresh once again, it should say the cagg is already up to date CALL refresh_continuous_aggregate ('test_cagg','2023-12-29 15:00:00', '2026-01-28 15:00:00'); @@ -1095,7 +1103,8 @@ WHERE hypertable_id IN ( SELECT raw_hypertable_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'test_cagg_1d_offset'); INSERT INTO test_data values ('2026-01-05 00:00:00', 1); -CALL refresh_continuous_aggregate ('test_cagg_1d_offset','2023-12-29 15:00:00', NULL); +-- Setting buckets_per_batch to 0 as the intention is to test the capping when windew end is set to NULL, no need for batching. +CALL refresh_continuous_aggregate ('test_cagg_1d_offset','2023-12-29 15:00:00', NULL, options => '{"buckets_per_batch": 0}'::jsonb); --should be at the 18th hour SELECT _timescaledb_functions.to_timestamp(watermark) as invalidation_threshold @@ -1158,9 +1167,9 @@ WHERE hypertable_id = ( -- Now refresh cagg_4hrs with NULL,NULL. -- cagg_4hrs computes its own threshold = 04:00, but stored threshold = 06:00 > 04:00, -- so the stored (misaligned) value is used but capped to the start of the current bucket of cagg_4hrs, --- which is 2020-01-01 04:00 UTC. +-- which is 2020-01-01 04:00 UTC. Disabling incremental refresh to preseve the isolate the test intention. SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_4hrs', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4hrs', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); RESET client_min_messages; -- Show invalidations left in cagg_4hrs's invalidation log, diff --git a/tsl/test/sql/cagg_invalidation_variable_bucket.sql b/tsl/test/sql/cagg_invalidation_variable_bucket.sql index 038eb48d27e..0b272e2396b 100644 --- a/tsl/test/sql/cagg_invalidation_variable_bucket.sql +++ b/tsl/test/sql/cagg_invalidation_variable_bucket.sql @@ -217,7 +217,8 @@ FROM generate_series('2025-03-30 00:00:00'::timestamptz, '2025-03-31 23:59:59.999999'::timestamptz, '1 hour'::interval) ts; -CALL refresh_continuous_aggregate('cagg_dst_daily', '2025-03-01 00:00:00', '2025-05-01 00:00:00'); +-- Disabling incremental refresh as it's not the focus of the test and it's easier to read the test output without it. +CALL refresh_continuous_aggregate('cagg_dst_daily', '2025-03-01 00:00:00', '2025-05-01 00:00:00', options => '{"buckets_per_batch": 0}'::jsonb); -- March 30 should have 23 hours SELECT bucket, cnt FROM cagg_dst_daily ORDER BY bucket; @@ -234,7 +235,7 @@ FROM generate_series('2025-10-26 00:00:00'::timestamptz, '1 hour'::interval) ts; -- Wide window to cover all DST-shifted buckets -CALL refresh_continuous_aggregate('cagg_dst_daily', '2025-10-01 00:00:00', '2026-12-01 00:00:00'); +CALL refresh_continuous_aggregate('cagg_dst_daily', '2025-10-01 00:00:00', '2026-12-01 00:00:00', options => '{"buckets_per_batch": 0}'::jsonb); -- October bucket should have extra hour (25-hour day on Oct 26) SELECT bucket, cnt FROM cagg_dst_daily @@ -245,7 +246,7 @@ ORDER BY bucket; INSERT INTO dst_data VALUES ('2025-10-26 01:00:00', 888.0); -- 2:00 AM Europe/Berlin (after fall-back) INSERT INTO dst_data VALUES ('2025-10-26 00:30:00', 777.0); -- 2:30 AM Europe/Berlin (before fall-back) -CALL refresh_continuous_aggregate('cagg_dst_daily', '2025-09-01 00:00:00', '2026-02-01 00:00:00'); +CALL refresh_continuous_aggregate('cagg_dst_daily', '2025-09-01 00:00:00', '2026-02-01 00:00:00', options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_inval_log WHERE cagg_name = 'cagg_dst_daily'; SET timezone TO 'UTC'; diff --git a/tsl/test/sql/cagg_refresh_incremental.sql b/tsl/test/sql/cagg_refresh_incremental.sql new file mode 100644 index 00000000000..74d56014516 --- /dev/null +++ b/tsl/test/sql/cagg_refresh_incremental.sql @@ -0,0 +1,491 @@ +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license. + +\c :TEST_DBNAME :ROLE_SUPERUSER + +-- Create a user with specific timezone for deterministic output +CREATE ROLE test_cagg_refresh_manual_user WITH LOGIN; +ALTER ROLE test_cagg_refresh_manual_user SET timezone TO 'UTC'; +GRANT ALL ON SCHEMA public TO test_cagg_refresh_manual_user; + +\c :TEST_DBNAME test_cagg_refresh_manual_user +SET timezone TO 'UTC'; + +CREATE TABLE conditions ( + time TIMESTAMP WITH TIME ZONE NOT NULL, + device_id INTEGER, + temperature NUMERIC +); + +SELECT FROM create_hypertable('conditions', by_range('time')); + +INSERT INTO conditions +SELECT + t, d, 10 +FROM + generate_series( + '2025-02-05 00:00:00+00', + '2025-03-05 00:00:00+00', + '1 hour'::interval) AS t, + generate_series(1,5) AS d; + +CREATE MATERIALIZED VIEW conditions_by_day +WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS +SELECT + time_bucket('1 day', time), + device_id, + count(*), + min(temperature), + max(temperature), + avg(temperature), + sum(temperature) +FROM + conditions +GROUP BY + 1, 2 +WITH NO DATA; + +-- Issue an incremental manual refresh using JSONB options: +-- buckets_per_batch => 10. This is the manual equivalent of a +-- policy with the same setting. +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 10}'::jsonb); +RESET client_min_messages; + +SELECT count(*) FROM conditions_by_day; + +CREATE MATERIALIZED VIEW conditions_by_day_atomic_refresh +WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS +SELECT + time_bucket('1 day', time), + device_id, + count(*), + min(temperature), + max(temperature), + avg(temperature), + sum(temperature) +FROM + conditions +GROUP BY + 1, 2 +WITH NO DATA; + +CALL refresh_continuous_aggregate('conditions_by_day_atomic_refresh', NULL, NULL); + +SELECT count(*) FROM conditions_by_day; +SELECT count(*) FROM conditions_by_day_atomic_refresh; + +-- Should have no differences +SELECT + count(*) > 0 AS has_diff +FROM + ((SELECT * FROM conditions_by_day_atomic_refresh ORDER BY 1, 2) + EXCEPT + (SELECT * FROM conditions_by_day ORDER BY 1, 2)) AS diff; + +-- buckets_per_batch => 0 is NOT incremental: the whole window is materialized +-- in a single pass. Under LOG this shows exactly one delete+insert pair, unlike +-- the multi-batch run above. The TRUNCATE invalidates the whole range so there +-- is data to re-materialize. +TRUNCATE conditions_by_day; +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 0}'::jsonb); +RESET client_min_messages; +SELECT count(*) FROM conditions_by_day; + +-- The continuous aggregate is now fully materialized with no pending +-- invalidations, so a normal refresh is a no-op (reports up-to-date). +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate('conditions_by_day', NULL, NULL); +RESET client_min_messages; + +-- Assert there really are no pending invalidations +SELECT + (SELECT count(*) + FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log + WHERE materialization_id = cagg.mat_hypertable_id + AND greatest_modified_value >= lowest_modified_value + AND lowest_modified_value != -9223372036854775808 + AND greatest_modified_value != 9223372036854775807) AS mat_invalidations, + (SELECT count(*) + FROM _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log + WHERE hypertable_id = cagg.raw_hypertable_id + AND greatest_modified_value >= lowest_modified_value) AS ht_invalidations +FROM _timescaledb_catalog.continuous_agg cagg +WHERE cagg.user_view_name = 'conditions_by_day'; + +-- A forced refresh is incremental but ignores the invalidation logs: +-- it re-materializes the entire window in batches (buckets_per_batch +-- defaults to 10) even though nothing is invalidated. +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate('conditions_by_day', NULL, NULL, force => true); +RESET client_min_messages; +SELECT count(*) FROM conditions_by_day; + +TRUNCATE conditions_by_day; + +-- Run with max_batches_per_execution => 2. Manual refresh stops mid-window +-- after processing 2 batches, leaving the rest for subsequent calls. +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 10, "max_batches_per_execution": 2}'::jsonb); +RESET client_min_messages; + +SELECT count(*) FROM conditions_by_day; +SELECT count(*) FROM conditions_by_day_atomic_refresh; + +-- Should have differences (partial materialization) +SELECT + count(*) > 0 AS has_diff +FROM + ((SELECT * FROM conditions_by_day_atomic_refresh ORDER BY 1, 2) + EXCEPT + (SELECT * FROM conditions_by_day ORDER BY 1, 2)) AS diff; + +-- Run a second call (same options) to process more batches. +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 10, "max_batches_per_execution": 2}'::jsonb); +RESET client_min_messages; + +SELECT count(*) FROM conditions_by_day; +SELECT count(*) FROM conditions_by_day_atomic_refresh; + +-- Should have no differences (all batches processed) +SELECT + count(*) > 0 AS has_diff +FROM + ((SELECT * FROM conditions_by_day_atomic_refresh ORDER BY 1, 2) + EXCEPT + (SELECT * FROM conditions_by_day ORDER BY 1, 2)) AS diff; + +-- Set max_batches_per_execution to 10 (effectively unlimited for our window) +-- and insert data into the past so a new set of batches must be processed. +INSERT INTO conditions +SELECT + t, d, 10 +FROM + generate_series( + '2020-02-05 00:00:00+00', + '2020-03-05 00:00:00+00', + '1 hour'::interval) AS t, + generate_series(1,5) AS d; + +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 10, "max_batches_per_execution": 10}'::jsonb); +RESET client_min_messages; + +SELECT count(*) FROM conditions_by_day; +SELECT count(*) FROM conditions_by_day_atomic_refresh; + +CALL refresh_continuous_aggregate('conditions_by_day_atomic_refresh', NULL, NULL); + +SELECT count(*) FROM conditions_by_day; +SELECT count(*) FROM conditions_by_day_atomic_refresh; + +-- Should have no differences +SELECT + count(*) > 0 AS has_diff +FROM + ((SELECT * FROM conditions_by_day_atomic_refresh ORDER BY 1, 2) + EXCEPT + (SELECT * FROM conditions_by_day ORDER BY 1, 2)) AS diff; + +-- Invalid configurations should be rejected +\set ON_ERROR_STOP 0 +\set VERBOSITY default +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"max_batches_per_execution": -1}'::jsonb); +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": -1}'::jsonb); +\set VERBOSITY terse +\set ON_ERROR_STOP 1 + +-- Truncate all data from the original hypertable. +TRUNCATE conditions; + +-- Should fall back to single-batch processing because there's no data +-- to refresh on the source hypertable. +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 10}'::jsonb); +RESET client_min_messages; + +-- Should return zero rows +SELECT count(*) FROM conditions_by_day; + +-- Insert 1 day of data +INSERT INTO conditions +SELECT + t, d, 10 +FROM + generate_series( + '2020-02-05 00:00:00+00', + '2020-02-06 00:00:00+00', + '1 hour'::interval) AS t, + generate_series(1,5) AS d; + +-- Should fall back to single-batch processing because the refresh size +-- (1 day) is smaller than 10 buckets x 1 day. +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 10}'::jsonb); +RESET client_min_messages; + +-- Should return 10 rows because the bucket width is `1 day` and we +-- inserted across two boundary timestamps for 5 devices. +SELECT count(*) FROM conditions_by_day; + +TRUNCATE conditions_by_day, conditions; + +-- Less than 1 day of data (smaller than the bucket width) +INSERT INTO conditions +VALUES ('2020-02-05 00:00:00+00', 1, 10); + +-- Should fall back to single-batch processing because the refresh size +-- is smaller than the bucket width. +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, NULL, + options => '{"buckets_per_batch": 10}'::jsonb); +RESET client_min_messages; + +-- Should return 1 row +SELECT count(*) FROM conditions_by_day; + +-- Re-test with an explicit refresh_newest_first => true (default behavior). +TRUNCATE conditions_by_day, conditions_by_day_atomic_refresh, conditions; + +INSERT INTO conditions +SELECT + t, d, 10 +FROM + generate_series( + '2025-03-11 00:00:00+00'::timestamptz - INTERVAL '30 days', + '2025-03-11 00:00:00+00'::timestamptz, + '1 hour'::interval) AS t, + generate_series(1,5) AS d; + +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_day', + '2025-03-11 00:00:00+00'::timestamptz - INTERVAL '15 days', + NULL, + options => '{"buckets_per_batch": 5, "refresh_newest_first": true}'::jsonb); +RESET client_min_messages; + +CALL refresh_continuous_aggregate( + 'conditions_by_day_atomic_refresh', + '2025-03-11 00:00:00+00'::timestamptz - INTERVAL '15 days', + NULL); + +-- Both continuous aggregates should have the same data +SELECT count(*) FROM conditions_by_day; +SELECT count(*) FROM conditions_by_day_atomic_refresh; + +-- Should have no differences +SELECT + count(*) > 0 AS has_diff +FROM + ((SELECT * FROM conditions_by_day_atomic_refresh ORDER BY 1, 2) + EXCEPT + (SELECT * FROM conditions_by_day ORDER BY 1, 2)) AS diff; + +-- refresh_newest_first => false (process from oldest to newest) +TRUNCATE conditions_by_day; + +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_day', + '2025-03-11 00:00:00+00'::timestamptz - INTERVAL '15 days', + NULL, + options => '{"buckets_per_batch": 5, "refresh_newest_first": false}'::jsonb); +RESET client_min_messages; + +-- Both continuous aggregates should have the same data +SELECT count(*) FROM conditions_by_day; +SELECT count(*) FROM conditions_by_day_atomic_refresh; + +-- Should have no differences +SELECT + count(*) > 0 AS has_diff +FROM + ((SELECT * FROM conditions_by_day_atomic_refresh ORDER BY 1, 2) + EXCEPT + (SELECT * FROM conditions_by_day ORDER BY 1, 2)) AS diff; + +-- Tests with variable-sized bucket (monthly) +TRUNCATE conditions; + +INSERT INTO conditions +SELECT + t, d, 10 +FROM + generate_series( + '2025-01-01 00:00:00+00', + '2025-10-08 00:00:00+00', + '1 hour'::interval) AS t, + generate_series(1,5) AS d; + +CREATE MATERIALIZED VIEW conditions_by_month +WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS +SELECT + time_bucket('1 month', time), + device_id, + count(*), + min(temperature), + max(temperature), + avg(temperature), + sum(temperature) +FROM + conditions +GROUP BY + 1, 2 +WITH NO DATA; + +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'conditions_by_month', + '2025-03-11 00:00:00+00'::timestamptz - INTERVAL '600 days', + '2025-03-11 00:00:00+00'::timestamptz - INTERVAL '7 days', + options => '{"refresh_newest_first": false}'::jsonb); +RESET client_min_messages; + +SELECT count(*) FROM conditions_by_month; + +------------------------------------------------------------------------------------------ +-- Test that batched refresh with variable-length buckets doesn't leave remainders +------------------------------------------------------------------------------------------ +CREATE TABLE test_data ( + time TIMESTAMPTZ NOT NULL, + value INT +); + +SELECT public.create_hypertable( + relation => 'test_data', + time_column_name => 'time', + chunk_time_interval => interval '1 months' +); +-- Insert initial data +INSERT INTO test_data +SELECT time, 1 +FROM generate_series('2024-01-01'::timestamptz, '2024-12-31'::timestamptz, '1 day'::interval) time; + +-- Create continuous aggregate with monthly buckets (variable-length) +CREATE MATERIALIZED VIEW batch_test_cagg +WITH (timescaledb.continuous) AS +SELECT + time_bucket('1 month'::interval, time) AS bucket, + count(*) as count +FROM test_data +GROUP BY bucket +WITH NO DATA; + +-- Run incremental manual refresh, 1 bucket per batch +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'batch_test_cagg', NULL, '2024-12-01'::timestamptz, + options => '{"buckets_per_batch": 1}'::jsonb); +RESET client_min_messages; + +-- Verify that the materialization invalidation log has no entries other than +-- the boundary -/+ infinity rows. +SELECT materialization_id, + _timescaledb_functions.to_timestamp(lowest_modified_value) as low, + _timescaledb_functions.to_timestamp(greatest_modified_value) as high +FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log +WHERE materialization_id IN + (SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg + WHERE user_view_name = 'batch_test_cagg') + AND lowest_modified_value != -9223372036854775808 + AND greatest_modified_value != 9223372036854775807 +ORDER BY low; + +-- Running again should be a no-op +SET client_min_messages TO LOG; +CALL refresh_continuous_aggregate( + 'batch_test_cagg', NULL, '2024-12-01'::timestamptz, + options => '{"buckets_per_batch": 1}'::jsonb); +RESET client_min_messages; + +DROP TABLE test_data CASCADE; + +------------------------------------------------------------------------------------------ +-- Test incremental manual refresh crashing between batches +------------------------------------------------------------------------------------------ + +-- Rows processed by a batch should be visible immediately after it finishes. +-- Inject an error after batch 1 completes and observe the cagg state. +\c :TEST_DBNAME :ROLE_SUPERUSER +TRUNCATE _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log, _timescaledb_catalog.continuous_aggs_materialization_invalidation_log; +\c :TEST_DBNAME test_cagg_refresh_manual_user +SET timezone TO 'UTC'; + +TRUNCATE conditions, conditions_by_day, conditions_by_day_atomic_refresh; + +INSERT INTO conditions +SELECT + t, d, 10 +FROM + generate_series( + '2025-02-05 00:00:00+00', + '2025-03-05 00:00:00+00', + '1 hour'::interval) AS t, + generate_series(1,5) AS d; + +-- Crash after batch 1 finishes +SELECT debug_waitpoint_enable('cagg_policy_batch_1_after_refresh'); + +-- Cagg state before refresh starts +SELECT min(time_bucket), max(time_bucket) FROM conditions_by_day; + +SET client_min_messages TO LOG; +\set ON_ERROR_STOP 0 +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, '1 hour'::interval, + options => '{"buckets_per_batch": 10}'::jsonb); +\set ON_ERROR_STOP 1 +RESET client_min_messages; + +-- Rows processed by batch 1 should be materialized +SELECT count(*) AS rows_after_refresh FROM conditions_by_day; +SELECT min(time_bucket), max(time_bucket) FROM conditions_by_day; + +-- Registered ranges should be cleaned up after the crash +SELECT count(*) AS registered_ranges FROM _timescaledb_catalog.continuous_aggs_jobs_refresh_ranges; + +SELECT debug_waitpoint_release('cagg_policy_batch_1_after_refresh'); + +-- Process remaining invalidations and verify +CALL refresh_continuous_aggregate( + 'conditions_by_day', NULL, '1 hour'::interval, + options => '{"buckets_per_batch": 10}'::jsonb); + +-- Verify against an atomic refresh +CALL refresh_continuous_aggregate('conditions_by_day_atomic_refresh', NULL, '1 hour'::interval); + +SELECT + count(*) > 0 AS has_diff +FROM + ((SELECT * FROM conditions_by_day_atomic_refresh ORDER BY 1, 2) + EXCEPT + (SELECT * FROM conditions_by_day ORDER BY 1, 2)) AS diff; + +\c :TEST_DBNAME :ROLE_SUPERUSER +REASSIGN OWNED BY test_cagg_refresh_manual_user TO :ROLE_SUPERUSER; +REVOKE ALL ON SCHEMA public FROM test_cagg_refresh_manual_user; + +DROP ROLE test_cagg_refresh_manual_user; diff --git a/tsl/test/sql/include/cagg_query_common.sql b/tsl/test/sql/include/cagg_query_common.sql index efdceb6b548..7dd4660dc72 100644 --- a/tsl/test/sql/include/cagg_query_common.sql +++ b/tsl/test/sql/include/cagg_query_common.sql @@ -54,7 +54,7 @@ group by time_bucket('1day', timec), location WITH NO DATA; --compute time_bucketted max+bucket_width for the materialized view SELECT time_bucket('1day' , q.timeval+ '1day'::interval) FROM ( select max(timec)as timeval from conditions ) as q; -CALL refresh_continuous_aggregate('mat_m1', NULL, NULL); +CALL refresh_continuous_aggregate('mat_m1', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); --test first/last create materialized view mat_m2(location, timec, firsth, lasth, maxtemp, mintemp) @@ -66,7 +66,7 @@ group by time_bucket('1day', timec), location WITH NO DATA; --time that refresh assumes as now() for repeatability SELECT time_bucket('1day' , q.timeval+ '1day'::interval) FROM ( select max(timec)as timeval from conditions ) as q; -CALL refresh_continuous_aggregate('mat_m2', NULL, NULL); +CALL refresh_continuous_aggregate('mat_m2', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); --normal view -- create or replace view regview( location, timec, minl, sumt , sumh) @@ -563,7 +563,7 @@ SET ROLE :ROLE_SUPERUSER; BEGIN; UPDATE _timescaledb_catalog.continuous_aggs_bucket_function SET bucket_func = 'func_does_not_exist()'; -- should error because function does not exist -CALL refresh_continuous_aggregate('cagg_bigint_offset2', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_bigint_offset2', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); ROLLBACK; \set ON_ERROR_STOP 1 SET ROLE :ROLE_DEFAULT_PERM_USER; @@ -644,9 +644,9 @@ SELECT * FROM cagg_4_hours_origin; -- Update the last bucket and re-materialize INSERT INTO temperature values('2020-01-01 23:55:00 PST', 10); -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_4_hours; SELECT * FROM cagg_4_hours_offset; @@ -680,9 +680,9 @@ SELECT * FROM cagg_4_hours_origin; -- Update materialized data SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); RESET client_min_messages; -- Query the CAggs and check that all buckets are materialized @@ -703,9 +703,9 @@ SELECT time_bucket('4 hour', time, '2000-01-01 01:00:00 PST'::timestamptz), max( -- Test invalidations TRUNCATE temperature; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); INSERT INTO temperature SELECT time, 5 @@ -722,9 +722,9 @@ INSERT INTO temperature values('2020-01-02 01:35:00+01', 5555); INSERT INTO temperature values('2020-01-02 05:05:00+01', 8888); SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); RESET client_min_messages; ALTER MATERIALIZED VIEW cagg_4_hours SET (timescaledb.materialized_only=true); @@ -780,7 +780,7 @@ WHERE materialization_id IN (SELECT mat_hypertable_id FROM _timescaledb_catalog. WHERE user_view_name = 'cagg_4_hours') ORDER BY 1, 2, 3; -CALL refresh_continuous_aggregate('cagg_4_hours', '2000-01-01 00:00:00'::timestamptz, '2020-12-31 23:59:59'::timestamptz); +CALL refresh_continuous_aggregate('cagg_4_hours', '2000-01-01 00:00:00'::timestamptz, '2020-12-31 23:59:59'::timestamptz, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log WHERE materialization_id IN (SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg @@ -823,12 +823,12 @@ INSERT INTO table_int VALUES(100, 555); -- Compare bucketing results SELECT time_bucket('10', time), SUM(data) FROM table_int GROUP BY 1 ORDER BY 1; SELECT * FROM cagg_int; -CALL refresh_continuous_aggregate('cagg_int', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_int', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_int; SELECT time_bucket('10', time, "offset"=>5), SUM(data) FROM table_int GROUP BY 1 ORDER BY 1; SELECT * FROM cagg_int_offset; -- the value 100 is part of the already serialized bucket, so it should not be visible -CALL refresh_continuous_aggregate('cagg_int_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_int_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); SELECT * FROM cagg_int_offset; -- Ensure everything was materialized @@ -842,7 +842,7 @@ SELECT * FROM cagg_int_offset; INSERT INTO table_int VALUES(114, 0); SET client_min_messages TO DEBUG1; -CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130); +CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130, options => '{"buckets_per_batch": 0}'::jsonb); RESET client_min_messages; SELECT * FROM cagg_int_offset; @@ -982,8 +982,8 @@ SELECT * FROM cagg_1_week_offset; SELECT time_bucket('1 week', time, origin=>'2000-01-02 01:00:00 PST'::timestamptz), max(value) FROM temperature GROUP BY 1 ORDER BY 1; -- Test refresh -CALL refresh_continuous_aggregate('cagg_1_hour_offset', NULL, NULL); -CALL refresh_continuous_aggregate('cagg_1_week_offset', NULL, NULL); +CALL refresh_continuous_aggregate('cagg_1_hour_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); +CALL refresh_continuous_aggregate('cagg_1_week_offset', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); -- Everything should be now materailized ALTER MATERIALIZED VIEW cagg_1_hour_offset SET (timescaledb.materialized_only=false); diff --git a/tsl/test/sql/telemetry_stats.sql b/tsl/test/sql/telemetry_stats.sql index 080f4343058..245fb1927a8 100644 --- a/tsl/test/sql/telemetry_stats.sql +++ b/tsl/test/sql/telemetry_stats.sql @@ -77,7 +77,7 @@ SELECT * FROM normal; INSERT INTO part SELECT * FROM normal; -CALL refresh_continuous_aggregate('contagg', NULL, NULL); +CALL refresh_continuous_aggregate('contagg', NULL, NULL, options => '{"buckets_per_batch": 0}'::jsonb); -- Reindex to avoid the dependency on the way the index is built (e.g. the caggs -- might get their rows inserted in different order during the refresh based on