diff --git a/.github/workflows/linux-32bit-build-and-test.yaml b/.github/workflows/linux-32bit-build-and-test.yaml index 571cca97ff5..f083efaeee4 100644 --- a/.github/workflows/linux-32bit-build-and-test.yaml +++ b/.github/workflows/linux-32bit-build-and-test.yaml @@ -70,6 +70,7 @@ jobs: compress_bloom_sparse_debug compress_compbloom_hash_pushdown compress_composite_bloom_debug + compress_observ-* compress_qualpushdown_saop compress_sort_transform compression_allocation diff --git a/.github/workflows/windows-build-and-test.yaml b/.github/workflows/windows-build-and-test.yaml index fb9396e426c..102e071d7a8 100644 --- a/.github/workflows/windows-build-and-test.yaml +++ b/.github/workflows/windows-build-and-test.yaml @@ -89,6 +89,7 @@ jobs: compress_bloom_sparse_debug compress_compbloom_hash_pushdown compress_composite_bloom_debug + compress_observ-* compress_qualpushdown_saop compress_sort_transform compression_algos diff --git a/.unreleased/pr_9730 b/.unreleased/pr_9730 new file mode 100644 index 00000000000..b183b7a8174 --- /dev/null +++ b/.unreleased/pr_9730 @@ -0,0 +1 @@ +Implements: #9730 In memory observability for compressed chunks diff --git a/cmake/ScriptFiles.cmake b/cmake/ScriptFiles.cmake index 06b20556a34..d8df1359fee 100644 --- a/cmake/ScriptFiles.cmake +++ b/cmake/ScriptFiles.cmake @@ -42,6 +42,7 @@ set(SOURCE_FILES bgw_scheduler.sql metadata.sql uuidv7.sql + ts_stats.sql views.sql views_experimental.sql gapfill.sql diff --git a/sql/ts_stats.sql b/sql/ts_stats.sql new file mode 100644 index 00000000000..7e7410404e3 --- /dev/null +++ b/sql/ts_stats.sql @@ -0,0 +1,71 @@ +-- This file and its contents are licensed under the Apache License 2.0. +-- Please see the included NOTICE for copyright information and +-- LICENSE-APACHE for a copy of the license. + +-- Chunk statistics. This function takes three optional parameters: +-- - compressed_relid: if provided, returns stats for the specified compressed chunk only; otherwise, returns stats for all chunks. +-- - uncompressed_relid: if provided, returns stats for the specified user chunk only; if compressed_relid is also provided, both must agree. +-- - since: if provided, returns stats for operations that occurred since the specified timestamp; otherwise, returns all stats. +-- +CREATE OR REPLACE FUNCTION _timescaledb_functions.chunk_statistics( + compressed_relid regclass DEFAULT NULL, + uncompressed_relid regclass DEFAULT NULL, + since timestamptz DEFAULT NULL +) +RETURNS TABLE ( + compressed_relid oid, + uncompressed_relid oid, + -- Compression + compressed_batch_count bigint, + compressed_block_count bigint, + compressed_batch_rows_min bigint, + compressed_batch_rows_max bigint, + compressed_batch_rows_sum bigint, + compressed_batch_rows_sqsum double precision, + compressed_batch_bytes_min bigint, + compressed_batch_bytes_max bigint, + compressed_batch_bytes_sum bigint, + compressed_batch_bytes_sqsum double precision, + compressed_block_bytes_min bigint, + compressed_block_bytes_max bigint, + compressed_block_bytes_sum bigint, + compressed_block_bytes_sqsum double precision, + -- DML totals (cumulative, from cmd_totals) + total_batches_deleted bigint, + total_batches_decompressed bigint, + total_tuples_decompressed bigint, + total_batches_scanned bigint, + total_batches_checked_by_bloom bigint, + total_batches_pruned_by_bloom bigint, + total_batches_without_bloom bigint, + total_batches_bloom_false_positives bigint, + total_batches_filtered_compressed bigint, + total_batches_filtered_decompressed bigint, + -- DML last operation snapshot (from cmd_last_op) + last_op_batches_deleted bigint, + last_op_batches_decompressed bigint, + last_op_tuples_decompressed bigint, + last_op_batches_scanned bigint, + last_op_batches_checked_by_bloom bigint, + last_op_batches_pruned_by_bloom bigint, + last_op_batches_without_bloom bigint, + last_op_batches_bloom_false_positives bigint, + last_op_batches_filtered_compressed bigint, + last_op_batches_filtered_decompressed bigint, + -- Operation counts + n_selects bigint, + n_inserts bigint, + n_updates bigint, + n_deletes bigint, + -- Timestamps + first_update timestamptz, + last_update timestamptz +) +AS '@MODULE_PATHNAME@', 'ts_stats_chunks' +LANGUAGE C STABLE; + +-- Logical reset: clears all cached chunks under the segment lock. +CREATE OR REPLACE FUNCTION _timescaledb_functions.chunk_statistics_reset() +RETURNS VOID +AS '@MODULE_PATHNAME@', 'ts_stats_reset' +LANGUAGE C VOLATILE; diff --git a/sql/updates/reverse-dev.sql b/sql/updates/reverse-dev.sql index 3ad68d7015c..ff5bccc0788 100644 --- a/sql/updates/reverse-dev.sql +++ b/sql/updates/reverse-dev.sql @@ -10,6 +10,11 @@ JOIN pg_constraint con WHERE c.osm_chunk ON CONFLICT DO NOTHING; +-- Drop chunk stats related objects +DROP VIEW IF EXISTS timescaledb_information.stat_chunk_activity; +DROP FUNCTION IF EXISTS _timescaledb_functions.chunk_statistics(regclass, regclass, timestamptz); +DROP FUNCTION IF EXISTS _timescaledb_functions.chunk_statistics_reset(); + -- Restore chunk_constraint rows for outbound FKs by matching chunk-side -- FKs to their hypertable-side counterpart by name. INSERT INTO _timescaledb_catalog.chunk_constraint diff --git a/sql/views.sql b/sql/views.sql index 3fbcaec8cf7..7fe64322f3b 100644 --- a/sql/views.sql +++ b/sql/views.sql @@ -430,5 +430,74 @@ SELECT * FROM timescaledb_information.chunk_compression_settings; CREATE OR REPLACE VIEW _timescaledb_config.bgw_job AS SELECT * from _timescaledb_catalog.bgw_job; +-- chunk statistics view +CREATE OR REPLACE VIEW timescaledb_information.stat_chunk_activity AS +SELECT + o.uncompressed_relid::regclass AS chunk, + o.compressed_relid::regclass AS compressed_chunk, + c.id AS chunk_id, + c.hypertable_id, + h.table_name AS hypertable, + -- Compression + o.compressed_batch_count, + o.compressed_block_count, + o.compressed_batch_rows_min, + o.compressed_batch_rows_max, + o.compressed_batch_rows_sum / NULLIF(o.compressed_batch_count, 0) + AS compressed_batch_rows_avg, + sqrt(o.compressed_batch_rows_sqsum / NULLIF(o.compressed_batch_count, 0) + - power(o.compressed_batch_rows_sum::double precision + / NULLIF(o.compressed_batch_count, 0), 2)) + AS compressed_batch_rows_stddev, + o.compressed_batch_bytes_min, + o.compressed_batch_bytes_max, + o.compressed_batch_bytes_sum / NULLIF(o.compressed_batch_count, 0) + AS compressed_batch_bytes_avg, + sqrt(o.compressed_batch_bytes_sqsum / NULLIF(o.compressed_batch_count, 0) + - power(o.compressed_batch_bytes_sum::double precision + / NULLIF(o.compressed_batch_count, 0), 2)) + AS compressed_batch_bytes_stddev, + o.compressed_block_bytes_min, + o.compressed_block_bytes_max, + o.compressed_block_bytes_sum / NULLIF(o.compressed_block_count, 0) + AS compressed_block_bytes_avg, + sqrt(o.compressed_block_bytes_sqsum / NULLIF(o.compressed_block_count, 0) + - power(o.compressed_block_bytes_sum::double precision + / NULLIF(o.compressed_block_count, 0), 2)) + AS compressed_block_bytes_stddev, + -- CMD totals + o.total_batches_deleted, + o.total_batches_decompressed, + o.total_tuples_decompressed, + o.total_batches_scanned, + o.total_batches_checked_by_bloom, + o.total_batches_pruned_by_bloom, + o.total_batches_without_bloom, + o.total_batches_bloom_false_positives, + o.total_batches_filtered_compressed, + o.total_batches_filtered_decompressed, + -- last CMD operation + o.last_op_batches_deleted, + o.last_op_batches_decompressed, + o.last_op_tuples_decompressed, + o.last_op_batches_scanned, + o.last_op_batches_checked_by_bloom, + o.last_op_batches_pruned_by_bloom, + o.last_op_batches_without_bloom, + o.last_op_batches_bloom_false_positives, + o.last_op_batches_filtered_compressed, + o.last_op_batches_filtered_decompressed, + o.n_selects, + o.n_inserts, + o.n_updates, + o.n_deletes, + o.first_update, + o.last_update +FROM _timescaledb_functions.chunk_statistics() o +LEFT JOIN _timescaledb_catalog.chunk c + ON format('%I.%I', c.schema_name, c.table_name)::regclass = o.uncompressed_relid +LEFT JOIN _timescaledb_catalog.hypertable h ON h.id = c.hypertable_id; + + GRANT SELECT ON ALL TABLES IN SCHEMA _timescaledb_config TO PUBLIC; GRANT SELECT ON ALL TABLES IN SCHEMA timescaledb_information TO PUBLIC; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 7f1640f1db9..29e03ff8287 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -125,6 +125,7 @@ add_subdirectory(ts_catalog) add_subdirectory(nodes) add_subdirectory(planner) add_subdirectory(with_clause) +add_subdirectory(ts_stats) # Don't run clang-tidy on the files we copied from Postgres. We don't want to # introduce changes there unless absolutely necessary. CMake can only access the diff --git a/src/chunk.c b/src/chunk.c index 7ba519e68cb..b794c75f37d 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include #include @@ -4074,6 +4075,9 @@ ts_chunk_drop(const Chunk *chunk, DropBehavior behavior, int32 log_level) /* Drop the table */ performDeletion(&objaddr, behavior, 0); + + /* Evict the chunk stats from the shared memory */ + ts_stats_chunk_evict(chunk->table_id); } static void diff --git a/src/guc.c b/src/guc.c index 8f23e2d1814..e315b269a3d 100644 --- a/src/guc.c +++ b/src/guc.c @@ -154,6 +154,8 @@ TSDLLEXPORT bool ts_guc_enable_delete_after_compression = false; TSDLLEXPORT bool ts_guc_enable_merge_on_cagg_refresh = false; bool ts_guc_enable_partitioned_hypertables = false; +TSDLLEXPORT int ts_guc_stats_max_chunks = TS_STATS_MAX_CHUNKS_DEFAULT; + #if PG16_GE TSDLLEXPORT bool ts_guc_enable_cagg_rewrites = false; TSDLLEXPORT bool ts_guc_cagg_rewrites_debug_info = false; @@ -494,6 +496,46 @@ assign_default_chunk_time_interval(const char *newval, void *extra) default_chunk_time_interval = extra; } +/* + * check_hook: accept 0 (feature disabled) or a power of two in [MIN, MAX]. + * Non-power-of-two values are rejected with a hint listing valid sizes. + */ +static bool +stats_max_chunks_check_hook(int *newval, void **extra, GucSource source) +{ + int v = *newval; + + if (v == 0) + { + return true; /* 0 = disabled, explicitly allowed */ + } + + if (v < TS_STATS_MAX_CHUNKS_MIN) + { + GUC_check_errdetail("Minimum cache capacity is %d chunks. " + "Set to 0 to disable the observability feature.", + TS_STATS_MAX_CHUNKS_MIN); + return false; + } + + if (v > TS_STATS_MAX_CHUNKS_MAX) + { + GUC_check_errdetail("Maximum cache capacity is %d chunks " + "(largest power of 2 within the slot index space).", + TS_STATS_MAX_CHUNKS_MAX); + return false; + } + + if ((v & (v - 1)) != 0) + { + GUC_check_errdetail("timescaledb.stats_max_chunks must be 0 (disabled) or a power of 2."); + GUC_check_errhint("Valid values: 0, 256, 512, 1024, 2048, " + "4096, 8192, 16384, 32768, 65536, 131072, 262144."); + return false; + } + return true; +} + void _guc_init(void) { @@ -1443,6 +1485,21 @@ _guc_init(void) NULL); #endif + DefineCustomIntVariable(MAKE_EXTOPTION("stats_max_chunks"), + "Per-database statistics cache capacity, " + "in chunks. 0 disables the feature.", + "Must be 0 or a power of 2. " + "Takes effect only after server restart.", + &ts_guc_stats_max_chunks, + TS_STATS_MAX_CHUNKS_DEFAULT, /* default */ + 0, /* min: 0 = disabled */ + TS_STATS_MAX_CHUNKS_MAX, /* max: 2^18 = 262144 chunks */ + PGC_SIGHUP, + 0, + stats_max_chunks_check_hook, + NULL, + NULL); + #ifdef USE_TELEMETRY DefineCustomEnumVariable(MAKE_EXTOPTION("telemetry_level"), "Telemetry settings level", diff --git a/src/guc.h b/src/guc.h index ce6f4c9bdd1..a843a201817 100644 --- a/src/guc.h +++ b/src/guc.h @@ -161,6 +161,13 @@ extern TSDLLEXPORT bool ts_guc_enable_rowlevel_compression_locking; extern TSDLLEXPORT DebugRequireOption ts_guc_debug_require_batch_sorted_merge; extern bool ts_guc_enable_partitioned_hypertables; +extern TSDLLEXPORT int ts_guc_stats_max_chunks; + +#define TS_STATS_MAX_CHUNKS_DEFAULT 1024 +#define TS_STATS_MAX_CHUNKS_MIN 256 +#define TS_STATS_MAX_CHUNKS_MAX 262144 + +#define IS_STATS_CHUNKS_ENABLED() (ts_guc_stats_max_chunks > 0) void _guc_init(void); diff --git a/src/loader/CMakeLists.txt b/src/loader/CMakeLists.txt index aad5071edc3..713870d3198 100644 --- a/src/loader/CMakeLists.txt +++ b/src/loader/CMakeLists.txt @@ -5,6 +5,7 @@ set(SOURCES bgw_launcher.c bgw_interface.c function_telemetry.c + ts_stats_handles.c lwlocks.c) set(TEST_SOURCES ${PROJECT_SOURCE_DIR}/test/src/symbol_conflict.c) diff --git a/src/loader/loader.c b/src/loader/loader.c index b7ef2f87420..f0b117429a4 100644 --- a/src/loader/loader.c +++ b/src/loader/loader.c @@ -41,6 +41,7 @@ #include "loader/function_telemetry.h" #include "loader/loader.h" #include "loader/lwlocks.h" +#include "loader/ts_stats_handles.h" /* * Loading process: @@ -623,6 +624,9 @@ timescaledb_shmem_startup_hook(void) ts_bgw_message_queue_shmem_startup(); ts_lwlocks_shmem_startup(); ts_function_telemetry_shmem_startup(); +#if PG17_LT + ts_stats_shmem_startup(); +#endif } /* @@ -642,6 +646,9 @@ timescaledb_shmem_request_hook(void) ts_bgw_message_queue_alloc(); ts_lwlocks_shmem_alloc(); ts_function_telemetry_shmem_alloc(); +#if PG17_LT + ts_stats_shmem_request(); +#endif } static void diff --git a/src/loader/ts_stats_handles.c b/src/loader/ts_stats_handles.c new file mode 100644 index 00000000000..9035da338a8 --- /dev/null +++ b/src/loader/ts_stats_handles.c @@ -0,0 +1,65 @@ +/* + * This file and its contents are licensed under the Apache License 2.0. + * Please see the included NOTICE for copyright information and + * LICENSE-APACHE for a copy of the license. + */ + +#include "ts_stats_handles.h" + +#include +#include +#include +#include +#include + +/* + * This is the fallback implementation for PG 15-16 of the per-database shared memory + * segment for chunk observability. In PG 17+ we can use the DSM registry and avoid this complexity. + * The fallback implementation uses a rendezvous-based registry of DSM segments keyed by database + * OID. + */ + +#if PG17_LT + +static Size +ts_stats_handle_table_size(void) +{ + return MAXALIGN(sizeof(TsObservHandleTable)); +} + +void +ts_stats_shmem_request(void) +{ + RequestAddinShmemSpace(ts_stats_handle_table_size()); +} + +void +ts_stats_shmem_startup(void) +{ + bool found; + + LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE); + + TsObservHandleTable *tbl = + ShmemInitStruct("ts_stats_handles", ts_stats_handle_table_size(), &found); + if (!found) + { + memset(tbl, 0, ts_stats_handle_table_size()); + LWLockInitialize(&tbl->lock, LWLockNewTrancheId()); + LWLockRegisterTranche(tbl->lock.tranche, "ts_stats_handles"); + tbl->max_entries = TS_STATS_MAX_DATABASES; + for (int i = 0; i < TS_STATS_MAX_DATABASES; i++) + { + tbl->entries[i].dboid = InvalidOid; + } + } + + LWLockRelease(AddinShmemInitLock); + + /* Publish via rendezvous for the versioned extension to find. */ + TsObservHandleTable **rv = + (TsObservHandleTable **) find_rendezvous_variable("ts_stats_handles"); + *rv = tbl; +} + +#endif /* PG17_LT */ diff --git a/src/loader/ts_stats_handles.h b/src/loader/ts_stats_handles.h new file mode 100644 index 00000000000..5fcabf42300 --- /dev/null +++ b/src/loader/ts_stats_handles.h @@ -0,0 +1,41 @@ +/* + * This file and its contents are licensed under the Apache License 2.0. + * Please see the included NOTICE for copyright information and + * LICENSE-APACHE for a copy of the license. + */ + +#pragma once + +#include +#include +#include +#include + +/* + * See more details in ts_stats_segment.c these declaration are only for the fallback + * implementation of the per-database DSM segment for PG 15-16. + */ + +#if PG17_LT +#define TS_STATS_MAX_DATABASES 1024 + +/* Segment types */ +#define TS_STATS_TYPE_CHUNK_SEGMENT 1 + +typedef struct TsObservHandleEntry +{ + Oid dboid; + int segment_type; + dsm_handle handle; +} TsObservHandleEntry; + +typedef struct TsObservHandleTable +{ + LWLock lock; + uint32 max_entries; + TsObservHandleEntry entries[TS_STATS_MAX_DATABASES]; +} TsObservHandleTable; + +void ts_stats_shmem_request(void); +void ts_stats_shmem_startup(void); +#endif /* PG17_LT */ diff --git a/src/ts_stats/CMakeLists.txt b/src/ts_stats/CMakeLists.txt new file mode 100644 index 00000000000..51aa44c5a96 --- /dev/null +++ b/src/ts_stats/CMakeLists.txt @@ -0,0 +1,5 @@ +set(SOURCES + ${CMAKE_CURRENT_SOURCE_DIR}/ts_stats_record.c + ${CMAKE_CURRENT_SOURCE_DIR}/ts_stats_segment.c + ${CMAKE_CURRENT_SOURCE_DIR}/ts_stats_srf.c) +target_sources(${PROJECT_NAME} PRIVATE ${SOURCES}) diff --git a/src/ts_stats/ts_stats_defs.h b/src/ts_stats/ts_stats_defs.h new file mode 100644 index 00000000000..f11f3461a6f --- /dev/null +++ b/src/ts_stats/ts_stats_defs.h @@ -0,0 +1,370 @@ +/* + * This file and its contents are licensed under the Apache License 2.0. + * Please see the included NOTICE for copyright information and + * LICENSE-APACHE for a copy of the license. + */ +#pragma once + +#include +#include +#include "export.h" +#include +#include +#include +#include +#include +#include + +#include + +#define TS_STATS_SHMEM_MAGIC 0x54534F42 /* "TSOB" — initialized-segment latch */ + +#define TS_STATS_BUCKETS 8 +#define TS_STATS_PROBE_WIDTH 8 +#define TS_STATS_CACHE_LINE_SIZE 64 +#define TS_STATS_INVALID_IDX (-1) + +/* note that the code expects TS_STATS_BUCKETS and TS_STATS_PROBE_WIDTH + * to be small, < 256 numbers and uses uint8 at multiple places to store them. + */ +StaticAssertDecl(TS_STATS_BUCKETS < 256 && TS_STATS_PROBE_WIDTH < 256, + "TS_STATS_BUCKETS and TS_STATS_PROBE_WIDTH must be less than 256"); + +/* + * Chunk information is stored in a shared-memory segment, one per database. + * + * The information is organized into two parallel arrays (they logically + * belong together), one stores metadata about each chunk such as its ID and + * a sequence number, and the other stores the actual statistics. The actual + * stats are being read/written after the lookup on the metadata is done, so + * the metadata is held together for better cache locality. + * + * The table is organised into TS_STATS_BUCKETS buckets, and we keep track of + * the last update sequence number for these buckets. In each bucket the chunk id + * (compressed_relid) is hashed to find the 'home location' within the bucket. + * If the bucket is occupied by another chunk, we do a linear probe of up to + * TS_STATS_PROBE_WIDTH slots within the bucket. If the chunk is not found + * we proceed to the next bucket. We traverse the buckets in recency order. + * If we still can't find the chunk after traversing all buckets, it doesn't + * exist. On upsert, we check the entries in the last (the oldest bucket) + * within the search window (TS_STATS_PROBE_WIDTH slots) for eviction + * candidates. If we find an empty slot, we use it. If not we evict the oldest + * entry in the search window of the last bucket. + * + * Once we updated an entry in a bucket we update the last update sequence + * number for the bucket, so the next upsert will traverse the buckets in + * the order of recency. + */ + +typedef struct TsStatsRelids +{ + Oid compressed_relid; + Oid uncompressed_relid; +} TsStatsRelids; + +/* + * As explained above, TsStatsChunkMetadata is split off from TsStatsChunk + * because these are needed more frequently than the rest of the structure, + * because during traversal we read these fields first to find the entry. + * Keeping them together allows better cache locality for the common case. + * + * The 'state' field is composed of two parts: the lower two bits are flags, + * and the rest is a version counter. The valid flag is set when the slot is + * initialized with valid data, and being cleared at eviction. The version + * counter is incremented at every update, and readers can use it to order + * updates. + */ +typedef struct TsStatsChunkMetadata +{ + /* state layout: + * bit 0: is valid (valid when set) + * bit 1: in progress (being updated, not safe to read or evict) + * bits 2-63: seqno, incremented at every update for recency tracking + */ + pg_atomic_uint64 state; + /* Chunks are identified by relids.compressed_relid */ + TsStatsRelids relids; +} TsStatsChunkMetadata; + +/* Bit flag values as per the above: */ +#define TS_STATS_CHUNK_METADATA_VALID_FLAG ((uint64) 1) +#define TS_STATS_CHUNK_METADATA_IN_PROGRESS_FLAG ((uint64) 2) +/* Bit mask helpers */ +#define TS_STATS_CHUNK_METADATA_SEQNO_SHIFT 2 +#define TS_STATS_CHUNK_METADATA_SEQNO_MASK (~((1ULL << TS_STATS_CHUNK_METADATA_SEQNO_SHIFT) - 1)) +#define TS_STATS_CHUNK_METADATA_GET_SEQNO(state) \ + ((state & TS_STATS_CHUNK_METADATA_SEQNO_MASK) >> TS_STATS_CHUNK_METADATA_SEQNO_SHIFT) +#define TS_STATS_CHUNK_METADATA_IS_VALID(state) ((state & TS_STATS_CHUNK_METADATA_VALID_FLAG) != 0) +#define TS_STATS_CHUNK_METADATA_IS_IN_PROGRESS(state) \ + ((state & TS_STATS_CHUNK_METADATA_IN_PROGRESS_FLAG) != 0) + +/* + * MinMaxSumDataWA uses atomics, but updates to this whole structure are not atomic. + * WA means 'With Atomics'. The potential inconsistency is acceptable for our use + * case, and it allows us to update the stats without acquiring an exclusive + * lock, which reduces contention and improves performance. + */ +typedef struct MinMaxSumDataWA +{ + /* + * The sqsum_val is a sum of squares, which is calculated as + * a double value and then cast to uint64 for atomic operations. + */ + pg_atomic_uint64 sqsum_val; + pg_atomic_uint64 sum_val; + pg_atomic_uint32 min_val; + pg_atomic_uint32 max_val; +} MinMaxSumDataWA; + +typedef struct MinMaxSumData +{ + double sqsum_val; + uint64 sum_val; + uint32 min_val; + uint32 max_val; +} MinMaxSumData; + +/* + * SharedCountersWA uses atomics, and mirrors the structure of SharedCounters. + * Updates to this structure are not atomic, WA means 'With Atomics'. The + * potential inconsistency is acceptable for our use case, and it allows us to + * update the stats without acquiring an exclusive lock. + */ +typedef struct SharedCountersWA +{ + pg_atomic_uint64 batches_deleted; + pg_atomic_uint64 batches_decompressed; + pg_atomic_uint64 tuples_decompressed; + pg_atomic_uint64 batches_scanned; + pg_atomic_uint64 batches_checked_by_bloom; + pg_atomic_uint64 batches_pruned_by_bloom; + pg_atomic_uint64 batches_without_bloom; + pg_atomic_uint64 batches_bloom_false_positives; + pg_atomic_uint64 batches_filtered_compressed; + pg_atomic_uint64 batches_filtered_decompressed; +} SharedCountersWA; + +typedef struct TsStatsChunk +{ + /* + * Microseconds since segment->base_timestamp. These are + * not used in CAS, only EQ and Assign, we can afford + * these to be non atomic. + */ + uint64 first_update_us; + uint64 last_update_us; + + /* Chunk stats at compression, totals: */ + pg_atomic_uint32 compressed_batch_count; + pg_atomic_uint32 compressed_block_count; + + /* byte stat across all batches and columns: */ + MinMaxSumDataWA compressed_block_bytes; + + /* sum bytes for each column, across batches: */ + MinMaxSumDataWA compressed_batch_bytes; + + /* rows across batches: */ + MinMaxSumDataWA compressed_batch_rows; + + /* Aggregated Cmd stats */ + SharedCountersWA cmd_totals; + + /* Cmd stats for the last operation */ + SharedCountersWA cmd_last_op; + + /* Operation counts */ + pg_atomic_uint32 n_selects; + pg_atomic_uint32 n_inserts; + pg_atomic_uint32 n_updates; + pg_atomic_uint32 n_deletes; + +} TsStatsChunk; + +typedef struct TsStatsChunkBucketMetadata +{ + pg_atomic_uint64 last_update_seqno; + /* padding to avoid false sharing on 64 byte cache lines */ + uint8 padding[56]; +} TsStatsChunkBucketMetadata; + +typedef struct TsStatsChunkSegment +{ + /* Magic set once at init, never written again*/ + uint32 magic; /* TS_STATS_SHMEM_MAGIC when initialized */ + Oid dboid; /* database that owns this segment */ + + /* Only used during whole segment operations, such as initialization. */ + LWLock lock; + + /* base timestamp for *_update_us fields */ + TimestampTz base_timestamp; + + /* Number of slots */ + uint32 num_slots; + + /* Sequence number shared across all chunk slots */ + pg_atomic_uint64 update_seqno; + + /* + * Trailing arrays — dynamically sized and aligned to cache lines. + * + * The logical layout is: + * + * TsStatsChunkMetadata meta[num_slots]; + * TsStatsChunk slots[num_slots]; + * TsStatsChunkBucketMetadata buckets[TS_STATS_BUCKETS]; + * + */ +} TsStatsChunkSegment; + +#define TS_STATS_CACHE_PADDED(S) (TYPEALIGN((TS_STATS_CACHE_LINE_SIZE), (S))) + +static inline Size +ts_stats_chunk_data_size(uint32 num_slots) +{ + Size size = 0; + size += TS_STATS_CACHE_PADDED(num_slots * sizeof(TsStatsChunkMetadata)); + size += TS_STATS_CACHE_PADDED(num_slots * sizeof(TsStatsChunk)); + size += TS_STATS_CACHE_PADDED(TS_STATS_BUCKETS * sizeof(TsStatsChunkBucketMetadata)); + return size; +} + +static inline Size +ts_stats_chunk_segment_size(uint32 num_slots) +{ + Size size = TS_STATS_CACHE_PADDED(sizeof(TsStatsChunkSegment)); + return add_size(size, ts_stats_chunk_data_size(num_slots)); +} + +static inline TsStatsChunkMetadata * +ts_stats_chunk_metadata(TsStatsChunkSegment *seg) +{ + return (TsStatsChunkMetadata *) ((char *) seg + + TS_STATS_CACHE_PADDED(sizeof(TsStatsChunkSegment))); +} + +static inline TsStatsChunk * +ts_stats_chunk_slots(TsStatsChunkSegment *seg) +{ + return (TsStatsChunk *) ((char *) seg + TS_STATS_CACHE_PADDED(sizeof(TsStatsChunkSegment)) + + TS_STATS_CACHE_PADDED(seg->num_slots * sizeof(TsStatsChunkMetadata))); +} + +static inline TsStatsChunkBucketMetadata * +ts_stats_chunk_bucket_metadata(TsStatsChunkSegment *seg) +{ + return (TsStatsChunkBucketMetadata *) ((char *) seg + + TS_STATS_CACHE_PADDED(sizeof(TsStatsChunkSegment)) + + TS_STATS_CACHE_PADDED(seg->num_slots * + sizeof(TsStatsChunkMetadata)) + + TS_STATS_CACHE_PADDED(seg->num_slots * + sizeof(TsStatsChunk))); +} + +#undef TS_STATS_CACHE_PADDED + +static inline void +ts_stats_chunk_init_slot(TsStatsChunk *slot) +{ + /* memset all to zero, except the min values */ + memset(slot, 0, sizeof(TsStatsChunk)); + pg_atomic_write_u32(&slot->compressed_block_bytes.min_val, UINT32_MAX); + pg_atomic_write_u32(&slot->compressed_batch_bytes.min_val, UINT32_MAX); + pg_atomic_write_u32(&slot->compressed_batch_rows.min_val, UINT32_MAX); +} + +static inline void +init_min_max_sum(MinMaxSumData *mmsd) +{ + mmsd->sqsum_val = 0; + mmsd->sum_val = 0; + mmsd->min_val = UINT32_MAX; + mmsd->max_val = 0; +} + +static inline void +mms_observe(MinMaxSumData *m, uint32 value) +{ + if (value < m->min_val) + { + m->min_val = value; + } + if (value > m->max_val) + { + m->max_val = value; + } + m->sum_val += value; + + double square_value = (double) value * (double) value; + m->sqsum_val += square_value; +} + +#define TS_STATS_MMS_CAS_MAX_RETRIES 5 + +static inline void +mms_merge(MinMaxSumDataWA *dst, const MinMaxSumData *src) +{ + /* + * this function does a bounded CAS loop so it won't spin indefinitely and does a best + * effort update of the min, max, sum, and sqsum values. + */ + uint32 cur_min = pg_atomic_read_u32(&dst->min_val); + for (int r = 0; src->min_val < cur_min && r < TS_STATS_MMS_CAS_MAX_RETRIES; r++) + { + if (pg_atomic_compare_exchange_u32(&dst->min_val, &cur_min, src->min_val)) + { + break; + } + /* CAS failed, stay with the old value */ + } + + uint32 cur_max = pg_atomic_read_u32(&dst->max_val); + for (int r = 0; src->max_val > cur_max && r < TS_STATS_MMS_CAS_MAX_RETRIES; r++) + { + if (pg_atomic_compare_exchange_u32(&dst->max_val, &cur_max, src->max_val)) + { + break; + } + /* CAS failed, stay with the old value */ + } + + pg_atomic_add_fetch_u64(&dst->sum_val, src->sum_val); + + uint64 cur_sqsum = pg_atomic_read_u64(&dst->sqsum_val); + for (int r = 0; r < TS_STATS_MMS_CAS_MAX_RETRIES; r++) + { + double new_d; + memcpy(&new_d, &cur_sqsum, sizeof(double)); + new_d += src->sqsum_val; + + uint64 desired; + memcpy(&desired, &new_d, sizeof(uint64)); + + if (pg_atomic_compare_exchange_u64(&dst->sqsum_val, &cur_sqsum, desired)) + { + break; + } + /* CAS failed, stay with the old value */ + } +} + +#undef TS_STATS_MMS_CAS_MAX_RETRIES + +static inline void +mms_update(MinMaxSumDataWA *dst, const MinMaxSumData *src) +{ + pg_atomic_write_u32(&dst->min_val, src->min_val); + pg_atomic_write_u32(&dst->max_val, src->max_val); + pg_atomic_write_u64(&dst->sum_val, src->sum_val); + + /* sqsum is calculated as a double and then copied to uint64 for atomic operations */ + pg_atomic_write_u64(&dst->sqsum_val, *((uint64 *) &src->sqsum_val)); +} + +static inline double +mms_sqsum(MinMaxSumDataWA *m) +{ + uint64 sqsum_val = pg_atomic_read_u64(&m->sqsum_val); + return *((double *) &sqsum_val); +} diff --git a/src/ts_stats/ts_stats_record.c b/src/ts_stats/ts_stats_record.c new file mode 100644 index 00000000000..930f5e48efb --- /dev/null +++ b/src/ts_stats/ts_stats_record.c @@ -0,0 +1,338 @@ +/* + * This file and its contents are licensed under the Apache License 2.0. + * Please see the included NOTICE for copyright information and + * LICENSE-APACHE for a copy of the license. + */ + +#include "ts_stats_record.h" +#include "ts_stats/ts_stats_defs.h" +#include "ts_stats_segment.h" + +#include +#include +#include +#include +#include + +static inline uint64 +now_us_offset(TsStatsChunkSegment *seg) +{ + return (uint64) (GetCurrentTimestamp() - seg->base_timestamp); +} + +static inline void +slot_touch_timestamps(TsStatsChunk *slot, uint64 now_us) +{ + /* There is very slight chance that this function is being called, just in the + * same microsecond as the base timestamp. */ + if (now_us == 0) + { + now_us = 1; + } + + if (slot->first_update_us == 0) + { + slot->first_update_us = now_us; + } + slot->last_update_us = now_us; +} + +void +ts_stats_chunk_record_compression(TsStatsRelids relids, const CompressionStatsAccumulator *a) +{ + if (!OidIsValid(relids.compressed_relid) || !OidIsValid(relids.uncompressed_relid) || + a->batch_count == 0) + { + return; + } + + TsStatsChunkSegment *seg = ts_get_stats_chunk_segment(); + if (seg == NULL) + { + return; + } + + /* get the slot_index for the upsert. this sets the in-progress flag and updates the bucket's + * metadata if successful. + */ + int32 slot_idx = ts_stats_chunk_segment_prepare_upsert(seg, relids); + if (slot_idx == TS_STATS_INVALID_IDX) + { + /* if we failed to find or insert a slot for this compressed_relid, we just give up this + * time */ + return; + } + Assert(slot_idx >= 0 && slot_idx < (int32) seg->num_slots); + + /* get the timestamp outside the lock scope */ + uint64 now_us = now_us_offset(seg); + + /* get the base pointers for the chunk */ + TsStatsChunk *slot_base = ts_stats_chunk_slots(seg); + + /* since we have a valid slot_idx, it means the admin stuff is done with the slot, and we only + * need to update the actual statistics. the in-progress flag is set for the slot which we + * will need to clear once the update is done. + */ + TsStatsChunk *slot = &slot_base[slot_idx]; + pg_atomic_add_fetch_u32(&slot->compressed_batch_count, a->batch_count); + pg_atomic_add_fetch_u32(&slot->compressed_block_count, a->block_count); + + /* Fold the three accumulator triples into the matching slot triples. */ + mms_merge(&slot->compressed_block_bytes, &a->block_size); + mms_merge(&slot->compressed_batch_bytes, &a->batch_size); + mms_merge(&slot->compressed_batch_rows, &a->batch_rows); + + slot_touch_timestamps(slot, now_us); + + /* clear the in-progress flag */ + ts_stats_chunk_segment_finish_upsert(seg, slot_idx, relids.compressed_relid); +} + +void +ts_stats_chunk_record_cmd(TsStatsRelids relids, CmdType cmd, const SharedCounters *c) +{ + if (!OidIsValid(relids.compressed_relid) || !OidIsValid(relids.uncompressed_relid) || c == NULL) + { + return; + } + if (cmd != CMD_SELECT && cmd != CMD_INSERT && cmd != CMD_UPDATE && cmd != CMD_DELETE) + { + return; + } + + TsStatsChunkSegment *seg = ts_get_stats_chunk_segment(); + if (seg == NULL) + { + return; + } + + /* get the slot_index for the upsert. this sets the in-progress flag and updates the bucket's + * metadata if successful. + */ + int32 slot_idx = ts_stats_chunk_segment_prepare_upsert(seg, relids); + if (slot_idx == TS_STATS_INVALID_IDX) + { + /* if we failed to find or insert a slot for this compressed_relid, we just give up this + * time */ + return; + } + Assert(slot_idx >= 0 && slot_idx < (int32) seg->num_slots); + + /* get the timestamp outside the lock scope */ + uint64 now_us = now_us_offset(seg); + + /* get the base pointers for the chunk */ + TsStatsChunk *slot_base = ts_stats_chunk_slots(seg); + + /* since we have a valid slot_idx, it means the admin stuff is done with the slot, and we only + * need to update the actual statistics. the in-progress flag is set for the slot which we + * will need to clear once the update is done. + */ + TsStatsChunk *slot = &slot_base[slot_idx]; + + switch (cmd) + { + case CMD_SELECT: + pg_atomic_add_fetch_u32(&slot->n_selects, 1); + break; + case CMD_INSERT: + pg_atomic_add_fetch_u32(&slot->n_inserts, 1); + break; + case CMD_UPDATE: + pg_atomic_add_fetch_u32(&slot->n_updates, 1); + break; + case CMD_DELETE: + pg_atomic_add_fetch_u32(&slot->n_deletes, 1); + break; + default: + break; + } + + /* update totals */ + pg_atomic_add_fetch_u64(&slot->cmd_totals.batches_deleted, c->batches_deleted); + pg_atomic_add_fetch_u64(&slot->cmd_totals.batches_decompressed, c->batches_decompressed); + pg_atomic_add_fetch_u64(&slot->cmd_totals.tuples_decompressed, c->tuples_decompressed); + pg_atomic_add_fetch_u64(&slot->cmd_totals.batches_scanned, c->batches_scanned); + pg_atomic_add_fetch_u64(&slot->cmd_totals.batches_checked_by_bloom, + c->batches_checked_by_bloom); + pg_atomic_add_fetch_u64(&slot->cmd_totals.batches_pruned_by_bloom, c->batches_pruned_by_bloom); + pg_atomic_add_fetch_u64(&slot->cmd_totals.batches_without_bloom, c->batches_without_bloom); + pg_atomic_add_fetch_u64(&slot->cmd_totals.batches_bloom_false_positives, + c->batches_bloom_false_positives); + pg_atomic_add_fetch_u64(&slot->cmd_totals.batches_filtered_compressed, + c->batches_filtered_compressed); + pg_atomic_add_fetch_u64(&slot->cmd_totals.batches_filtered_decompressed, + c->batches_filtered_decompressed); + + /* update last op */ + pg_atomic_write_u64(&slot->cmd_last_op.batches_deleted, c->batches_deleted); + pg_atomic_write_u64(&slot->cmd_last_op.batches_decompressed, c->batches_decompressed); + pg_atomic_write_u64(&slot->cmd_last_op.tuples_decompressed, c->tuples_decompressed); + pg_atomic_write_u64(&slot->cmd_last_op.batches_scanned, c->batches_scanned); + pg_atomic_write_u64(&slot->cmd_last_op.batches_checked_by_bloom, c->batches_checked_by_bloom); + pg_atomic_write_u64(&slot->cmd_last_op.batches_pruned_by_bloom, c->batches_pruned_by_bloom); + pg_atomic_write_u64(&slot->cmd_last_op.batches_without_bloom, c->batches_without_bloom); + pg_atomic_write_u64(&slot->cmd_last_op.batches_bloom_false_positives, + c->batches_bloom_false_positives); + pg_atomic_write_u64(&slot->cmd_last_op.batches_filtered_compressed, + c->batches_filtered_compressed); + pg_atomic_write_u64(&slot->cmd_last_op.batches_filtered_decompressed, + c->batches_filtered_decompressed); + + /* update timestamps */ + slot_touch_timestamps(slot, now_us); + + /* clear the in-progress flag */ + ts_stats_chunk_segment_finish_upsert(seg, slot_idx, relids.compressed_relid); +} + +void +ts_stats_chunk_record_decompression(TsStatsRelids relids, const CompressionStatsAccumulator *a) +{ + if (!OidIsValid(relids.compressed_relid) || !OidIsValid(relids.uncompressed_relid)) + { + return; + } + if (a->batch_count == 0 && a->block_count == 0) + { + return; + } + + TsStatsChunkSegment *seg = ts_get_stats_chunk_segment(); + if (seg == NULL) + { + return; + } + + /* get the slot_index for the upsert. this sets the in-progress flag and updates the bucket's + * metadata if successful. + */ + int32 slot_idx = ts_stats_chunk_segment_prepare_upsert(seg, relids); + if (slot_idx == TS_STATS_INVALID_IDX) + { + /* if we failed to find or insert a slot for this compressed_relid, we just give up this + * time */ + return; + } + Assert(slot_idx >= 0 && slot_idx < (int32) seg->num_slots); + + /* get the timestamp outside the lock scope */ + uint64 now_us = now_us_offset(seg); + + /* get the base pointers for the chunk */ + TsStatsChunk *slot_base = ts_stats_chunk_slots(seg); + + /* since we have a valid slot_idx, it means the admin stuff is done with the slot, and we only + * need to update the actual statistics. the in-progress flag is set for the slot which we + * will need to clear once the update is done. + */ + TsStatsChunk *slot = &slot_base[slot_idx]; + + /* Batch group being updated if the current information is more complete. */ + if (a->batch_count > pg_atomic_read_u32(&slot->compressed_batch_count) && + a->batch_rows.sum_val > pg_atomic_read_u64(&slot->compressed_batch_rows.sum_val) && + a->batch_rows.sqsum_val > mms_sqsum(&slot->compressed_batch_rows) && + a->batch_size.sum_val > pg_atomic_read_u64(&slot->compressed_batch_bytes.sum_val) && + a->batch_size.sqsum_val > mms_sqsum(&slot->compressed_batch_bytes)) + { + pg_atomic_write_u32(&slot->compressed_batch_count, a->batch_count); + mms_update(&slot->compressed_batch_rows, &a->batch_rows); + mms_update(&slot->compressed_batch_bytes, &a->batch_size); + } + + /* Column group being updated if the current information is more complete. */ + if (a->block_count > pg_atomic_read_u32(&slot->compressed_block_count) && + a->block_size.sum_val > pg_atomic_read_u64(&slot->compressed_block_bytes.sum_val) && + a->block_size.sqsum_val > mms_sqsum(&slot->compressed_block_bytes)) + { + pg_atomic_write_u32(&slot->compressed_block_count, a->block_count); + mms_update(&slot->compressed_block_bytes, &a->block_size); + } + + /* update timestamps */ + slot_touch_timestamps(slot, now_us); + + /* clear the in-progress flag */ + ts_stats_chunk_segment_finish_upsert(seg, slot_idx, relids.compressed_relid); +} + +bool +ts_stats_chunk_lookup(Oid compressed_relid, TsStatsChunk *out) +{ + if (!OidIsValid(compressed_relid) || out == NULL) + { + return false; + } + + TsStatsChunkSegment *seg = ts_get_stats_chunk_segment(); + if (seg == NULL) + { + return false; + } + + int32 slot_idx = ts_stats_chunk_segment_lookup(seg, compressed_relid); + + if (slot_idx == TS_STATS_INVALID_IDX) + { + return false; + } + Assert(slot_idx >= 0 && slot_idx < (int32) seg->num_slots); + + /* get the base pointers for the chunk */ + TsStatsChunk *slot_base = ts_stats_chunk_slots(seg); + TsStatsChunk *slot = &slot_base[slot_idx]; + + /* dumb memcpy, hoping the data is still consistent */ + memcpy(out, slot, sizeof(TsStatsChunk)); + return true; +} + +void +ts_stats_chunk_evict(Oid compressed_relid) +{ + if (!OidIsValid(compressed_relid)) + { + return; + } + TsStatsChunkSegment *seg = ts_get_stats_chunk_segment(); + if (seg == NULL) + { + return; + } + + int32 slot_idx = ts_stats_chunk_segment_lookup(seg, compressed_relid); + + if (slot_idx == TS_STATS_INVALID_IDX) + { + return; + } + Assert(slot_idx >= 0 && slot_idx < (int32) seg->num_slots); + + TsStatsChunkMetadata *meta_base = ts_stats_chunk_metadata(seg); + TsStatsChunkMetadata *meta = &meta_base[slot_idx]; + + /* limited CAS loop to ensure we don't evict a chunk that is currently being updated, the danger + * is that the one that we are evicting is already being overwritten by a newer/other chunk. + */ + uint64 state = pg_atomic_read_u64(&meta->state); + uint64 new_seqno = pg_atomic_add_fetch_u64(&seg->update_seqno, 1); + uint64 new_state = (new_seqno << TS_STATS_CHUNK_METADATA_SEQNO_SHIFT); + for (int i = 0; + i < 5 && + /* the slot still matches the compressed_relid */ + meta->relids.compressed_relid == compressed_relid && + /* the slot is still valid and not in progress */ + TS_STATS_CHUNK_METADATA_IS_VALID(state) && + !(TS_STATS_CHUNK_METADATA_IS_IN_PROGRESS(state)) && + /* the slot being updated is older than the sequence number of the current update */ + TS_STATS_CHUNK_METADATA_GET_SEQNO(state) < new_seqno; + ++i) + { + if (pg_atomic_compare_exchange_u64(&meta->state, &state, new_state)) + { + meta->relids.compressed_relid = InvalidOid; + break; + } + } +} diff --git a/src/ts_stats/ts_stats_record.h b/src/ts_stats/ts_stats_record.h new file mode 100644 index 00000000000..1239d8a8e1e --- /dev/null +++ b/src/ts_stats/ts_stats_record.h @@ -0,0 +1,78 @@ +/* + * This file and its contents are licensed under the Apache License 2.0. + * Please see the included NOTICE for copyright information and + * LICENSE-APACHE for a copy of the license. + */ +#pragma once + +#include "ts_stats_defs.h" +#include /* CmdType */ + +typedef struct CompressionStatsAccumulator +{ + /* Per-column: updated on every column write. */ + uint32 block_count; + MinMaxSumData block_size; + + /* Per-compressed-row: updated on every batch flush. */ + uint32 batch_count; + MinMaxSumData batch_size; + MinMaxSumData batch_rows; + + /* Reused between the per-column and per-comp-row. */ + uint32 pending_batch_bytes; + +} CompressionStatsAccumulator; + +static inline void +ts_stats_compression_acc_init(CompressionStatsAccumulator *a) +{ + memset(a, 0, sizeof(*a)); + init_min_max_sum(&a->block_size); + init_min_max_sum(&a->batch_size); + init_min_max_sum(&a->batch_rows); +} + +static inline void +ts_stats_compression_acc_column(CompressionStatsAccumulator *a, uint32 block_bytes) +{ + a->block_count++; + mms_observe(&a->block_size, block_bytes); + a->pending_batch_bytes += block_bytes; +} + +static inline void +ts_stats_compression_acc_batch(CompressionStatsAccumulator *a, uint32 batch_rows) +{ + a->batch_count++; + mms_observe(&a->batch_size, a->pending_batch_bytes); + mms_observe(&a->batch_rows, batch_rows); + a->pending_batch_bytes = 0; +} + +/* Update shared memory with new statistics information. */ + +/* Compression — commits the accumulator into the chunk's slot. */ +extern TSDLLEXPORT void ts_stats_chunk_record_compression(TsStatsRelids relids, + const CompressionStatsAccumulator *a); + +/* Cmd stats on compressed data. cmd is one of CMD_INSERT / CMD_UPDATE / CMD_DELETE / CMD_SELECT */ +extern TSDLLEXPORT void ts_stats_chunk_record_cmd(TsStatsRelids relids, CmdType cmd, + const SharedCounters *c); + +/* For backfilling compressed data during decompression. + * This function will update the compression stats for the chunk when it finds more complete + * information about the compressed data than the one we have in memory. The reason we + * need this is because the in-memory entries may miss the compression event, either because + * of a restart or because the chunk information has been evicted from memory due to lack of use. + */ +extern TSDLLEXPORT void ts_stats_chunk_record_decompression(TsStatsRelids relids, + const CompressionStatsAccumulator *a); + +/* Lookup TsStatsChunk based on the compressed relid. It returns true if the chunk is found, false + * otherwise. */ +extern TSDLLEXPORT bool ts_stats_chunk_lookup(Oid compressed_relid, TsStatsChunk *out); + +/* Evict the chunk information for a given compressed relid. This function is called when we + * drop a compressed chunk. */ +extern TSDLLEXPORT void ts_stats_chunk_evict(Oid compressed_relid); diff --git a/src/ts_stats/ts_stats_segment.c b/src/ts_stats/ts_stats_segment.c new file mode 100644 index 00000000000..338d85bece2 --- /dev/null +++ b/src/ts_stats/ts_stats_segment.c @@ -0,0 +1,685 @@ +/* + * This file and its contents are licensed under the Apache License 2.0. + * Please see the included NOTICE for copyright information and + * LICENSE-APACHE for a copy of the license. + */ + +#include "ts_stats_segment.h" +#include "guc.h" /* ts_guc_stats_max_chunks */ + +#include +#include "utils/syscache.h" +#include +#include +#include + +#if PG17_GE +#include +#else +#include "../loader/ts_stats_handles.h" +#include +#include +#endif + +/* + * One-time init. + */ +static void +ts_stats_chunk_init_segment(void *ptr) +{ + TsStatsChunkSegment *seg = (TsStatsChunkSegment *) ptr; + uint32 num_slots = (uint32) ts_guc_stats_max_chunks; + + /* must be powers of two */ + Assert((num_slots & (num_slots - 1)) == 0); + + /* zeros are valid for all variables, in particular this clears + * the validity flag part of the TsStatsChunkMetadata.state field */ + memset(seg, 0, ts_stats_chunk_segment_size(num_slots)); + + seg->magic = TS_STATS_SHMEM_MAGIC; + seg->dboid = MyDatabaseId; + LWLockInitialize(&seg->lock, LWLockNewTrancheId()); + LWLockRegisterTranche(seg->lock.tranche, "ts_stats_chunk"); + seg->base_timestamp = GetCurrentTimestamp(); + seg->num_slots = num_slots; + + /* sequence number starts at 1 */ + pg_atomic_write_u64(&seg->update_seqno, 1); +} + +void +ts_stats_chunk_segment_reset(TsStatsChunkSegment *seg) +{ + /* blindly overwrite all dynamic state, no locking and no checks */ + void *meta_base = ts_stats_chunk_metadata(seg); + Size data_size = ts_stats_chunk_data_size(seg->num_slots); + memset(meta_base, 0, data_size); +} + +/* + * Backend-local cache. MyDatabaseId is fixed for a backend's lifetime, so + * cached_segment is safe to reuse across calls. + */ +static TsStatsChunkSegment *cached_segment = NULL; + +/* GetNamedDSMSegment was introduced in PG 17, so we need separate + * logic for older versions. */ +#if PG17_GE + +TsStatsChunkSegment * +ts_get_stats_chunk_segment(void) +{ + if (!IS_STATS_CHUNKS_ENABLED()) + { + return NULL; + } + + if (likely(cached_segment != NULL)) + { + return cached_segment; + } + + Size seg_size = ts_stats_chunk_segment_size((uint32) ts_guc_stats_max_chunks); + + char name[64]; + snprintf(name, sizeof(name), "ts_stats_chunk_db_%u", MyDatabaseId); + + bool found; + cached_segment = GetNamedDSMSegment(name, seg_size, ts_stats_chunk_init_segment, &found); + + if (cached_segment != NULL && cached_segment->magic != TS_STATS_SHMEM_MAGIC) + { + elog(WARNING, + "ts_stats_chunk: segment for database %u has unexpected magic, disabling", + MyDatabaseId); + cached_segment = NULL; + } + return cached_segment; +} + +#else /* PG17_LT */ + +/* The below section is about handling older PostgreSQL version, before PG 17. + * The memory segments for each database are registered in a global handle table. + * When the handle table is full, we sweep through the entries and remove the + * segments for the databases that have been dropped. If there are more than + * TS_STATS_MAX_DATABASES databases, or if the segment creation fails for any + * reason, we disable stats tracking for the current database. + */ + +static TsObservHandleEntry * +ts_stats_chunk_find_free_entry(TsObservHandleTable *tbl) +{ + for (uint32 i = 0; i < tbl->max_entries; i++) + { + if (!OidIsValid(tbl->entries[i].dboid)) + { + return &tbl->entries[i]; + } + } + return NULL; +} + +static TsObservHandleEntry * +ts_stats_chunk_find_entry(TsObservHandleTable *tbl, Oid dboid, int segment_type) +{ + for (uint32 i = 0; i < tbl->max_entries; i++) + { + if (tbl->entries[i].dboid == dboid && tbl->entries[i].segment_type == segment_type) + { + return &tbl->entries[i]; + } + } + return NULL; +} + +static void +ts_stats_chunk_sweep_stale(TsObservHandleTable *tbl) +{ + Assert(IsTransactionState()); + + typedef struct + { + uint32 idx; + Oid dboid; + } SnapEntry; + + SnapEntry snap[TS_STATS_MAX_DATABASES]; + SnapEntry gone[TS_STATS_MAX_DATABASES]; + dsm_handle unpin[TS_STATS_MAX_DATABASES]; + uint32 snap_n = 0, gone_n = 0, unpin_n = 0; + + /* Sweep through the handle table and take a snapshot of valid entries */ + LWLockAcquire(&tbl->lock, LW_EXCLUSIVE); + for (uint32 i = 0; i < TS_STATS_MAX_DATABASES; i++) + { + if (OidIsValid(tbl->entries[i].dboid)) + { + snap[snap_n++] = (SnapEntry){ i, tbl->entries[i].dboid }; + } + } + LWLockRelease(&tbl->lock); + + /* Check which entries are no longer valid */ + for (uint32 j = 0; j < snap_n; j++) + { + HeapTuple tup = SearchSysCache1(DATABASEOID, ObjectIdGetDatum(snap[j].dboid)); + if (!HeapTupleIsValid(tup)) + { + gone[gone_n++] = snap[j]; + } + else + { + ReleaseSysCache(tup); + } + } + + /* Remove stale entries from the handle table */ + LWLockAcquire(&tbl->lock, LW_EXCLUSIVE); + for (uint32 k = 0; k < gone_n; k++) + { + TsObservHandleEntry *e = &tbl->entries[gone[k].idx]; + if (e->dboid == gone[k].dboid) + { + unpin[unpin_n++] = e->handle; + e->dboid = InvalidOid; + e->handle = 0; + } + } + LWLockRelease(&tbl->lock); + + /* Unpin the stale segments */ + for (uint32 m = 0; m < unpin_n; m++) + { + dsm_unpin_segment(unpin[m]); + } +} + +static TsStatsChunkSegment * +ts_stats_chunk_create_segment(TsObservHandleTable *tbl) +{ + Size seg_size = ts_stats_chunk_segment_size((uint32) ts_guc_stats_max_chunks); + + dsm_segment *local_seg = dsm_create(seg_size, 0); + if (local_seg == NULL) + { + return NULL; + } + + dsm_pin_segment(local_seg); + dsm_pin_mapping(local_seg); + dsm_handle local_handle = dsm_segment_handle(local_seg); + + TsStatsChunkSegment *obs = dsm_segment_address(local_seg); + ts_stats_chunk_init_segment(obs); + + LWLockAcquire(&tbl->lock, LW_EXCLUSIVE); + + TsObservHandleEntry *entry = + ts_stats_chunk_find_entry(tbl, MyDatabaseId, TS_STATS_TYPE_CHUNK_SEGMENT); + if (entry != NULL) + { + dsm_handle winner_handle = entry->handle; + LWLockRelease(&tbl->lock); + + dsm_unpin_segment(local_handle); + dsm_detach(local_seg); + + dsm_segment *winner_seg = dsm_attach(winner_handle); + if (winner_seg == NULL) + { + return NULL; + } + dsm_pin_mapping(winner_seg); + return dsm_segment_address(winner_seg); + } + + /* Find a free slot, with one sweep retry. */ + entry = ts_stats_chunk_find_free_entry(tbl); + if (entry == NULL) + { + LWLockRelease(&tbl->lock); + ts_stats_chunk_sweep_stale(tbl); + LWLockAcquire(&tbl->lock, LW_EXCLUSIVE); + entry = ts_stats_chunk_find_free_entry(tbl); + } + + /* Table full. */ + if (entry == NULL) + { + LWLockRelease(&tbl->lock); + dsm_unpin_segment(local_handle); + dsm_detach(local_seg); + elog(LOG, + "ts_stats_chunk: handle table full after sweep, chunk statistics disabled " + "for database %u", + MyDatabaseId); + return NULL; + } + + /* Publish. */ + entry->dboid = MyDatabaseId; + entry->handle = local_handle; + entry->segment_type = TS_STATS_TYPE_CHUNK_SEGMENT; + LWLockRelease(&tbl->lock); + return obs; +} + +TsStatsChunkSegment * +ts_get_stats_chunk_segment(void) +{ + static bool rendezvous_checked = false; + + if (!IS_STATS_CHUNKS_ENABLED()) + { + return NULL; + } + + if (likely(cached_segment != NULL)) + { + return cached_segment; + } + + if (rendezvous_checked) + { + return NULL; + } + rendezvous_checked = true; + + TsObservHandleTable **rv = + (TsObservHandleTable **) find_rendezvous_variable("ts_stats_handles"); + TsObservHandleTable *tbl = *rv; + if (tbl == NULL) + { + return NULL; + } + + LWLockAcquire(&tbl->lock, LW_SHARED); + TsObservHandleEntry *entry = + ts_stats_chunk_find_entry(tbl, MyDatabaseId, TS_STATS_TYPE_CHUNK_SEGMENT); + + if (entry != NULL) + { + dsm_segment *seg = dsm_find_mapping(entry->handle); + if (seg == NULL) + { + seg = dsm_attach(entry->handle); + if (seg != NULL) + { + dsm_pin_mapping(seg); + } + } + LWLockRelease(&tbl->lock); + + if (seg != NULL) + { + cached_segment = dsm_segment_address(seg); + if (cached_segment->magic != TS_STATS_SHMEM_MAGIC) + { + elog(WARNING, + "ts_stats_chunk: segment for database %u has unexpected magic, disabling", + MyDatabaseId); + cached_segment = NULL; + } + return cached_segment; + } + } + else + { + LWLockRelease(&tbl->lock); + } + + cached_segment = ts_stats_chunk_create_segment(tbl); + return cached_segment; +} + +#endif /* PG17_GE */ + +/* + * Knuth's multiplicative hash. The constant 0x9E3779B9 = floor(2^32 / phi). + * Multiplying scrambles input structure across the 32-bit space. We don't + * need perfect hashing here, just a good enough distribution that puts + * consecutive relids at least window width apart to avoid clustering in + * the linear probing. + */ +static inline uint32 +hash_relid(const TsStatsChunkSegment *seg, Oid compressed_relid) +{ + uint32 bucket_size = seg->num_slots / TS_STATS_BUCKETS; + uint32 product = (uint32) compressed_relid * 0x9E3779B9U; + return (uint32) (((uint64_t) product * (uint64_t) bucket_size) >> 32); +} + +/* Return the sorted bucket order as an array of uint8 indices */ +static inline void +get_sorted_bucket_in_desc_order(TsStatsChunkSegment *seg, uint8 *out) +{ + Assert(out != NULL && seg != NULL); + uint64 seqnos[TS_STATS_BUCKETS]; + TsStatsChunkBucketMetadata *bucket_base = ts_stats_chunk_bucket_metadata(seg); + for (int i = 0; i < TS_STATS_BUCKETS; i++) + { + out[i] = (uint8) i; + seqnos[i] = pg_atomic_read_u64(&bucket_base[i].last_update_seqno); + } + + /* sort the bucket index by the last update sequence number in descending order */ + for (int j = 0; j < TS_STATS_BUCKETS - 1; j++) + { + for (int k = 0; k < TS_STATS_BUCKETS - j - 1; k++) + { + if (seqnos[out[k]] < seqnos[out[k + 1]]) + { + uint8 temp = out[k]; + out[k] = out[k + 1]; + out[k + 1] = temp; + } + } + } +} + +/* This function evicts/updates a chunk at the specified index and also sets the + * in-progress flag, plus updated the bucket's last update sequence number. + * This is a utility function to simplify 'ts_stats_chunk_segment_prepare_upsert' + */ +static inline int32 +ts_stats_chunk_update_at(TsStatsChunkSegment *seg, TsStatsRelids relids, int32 idx, + uint64 old_state) +{ + Assert(seg != NULL || OidIsValid(relids.compressed_relid) || + OidIsValid(relids.uncompressed_relid)); + if (seg == NULL || !OidIsValid(relids.compressed_relid) || + !OidIsValid(relids.uncompressed_relid)) + { + return TS_STATS_INVALID_IDX; + } + + uint32 bucket_size = seg->num_slots / TS_STATS_BUCKETS; + uint32 bucket_idx = idx / bucket_size; + uint64 new_seqno = pg_atomic_add_fetch_u64(&seg->update_seqno, 1); + uint64 new_state = (new_seqno << TS_STATS_CHUNK_METADATA_SEQNO_SHIFT) | + TS_STATS_CHUNK_METADATA_VALID_FLAG | + TS_STATS_CHUNK_METADATA_IN_PROGRESS_FLAG; + + TsStatsChunkMetadata *meta_base = ts_stats_chunk_metadata(seg); + TsStatsChunkBucketMetadata *bucket_base = ts_stats_chunk_bucket_metadata(seg); + + TsStatsChunkMetadata *meta = &meta_base[idx]; + + /* only evict when the state is the same as the old state, which means + * no other process has updated this slot since we read it last time */ + if (!pg_atomic_compare_exchange_u64(&meta->state, &old_state, new_state)) + { + return TS_STATS_INVALID_IDX; + } + + /* update the metadata */ + meta->relids = relids; + pg_atomic_write_u64(&bucket_base[bucket_idx].last_update_seqno, new_seqno); + + return (int32) idx; +} + +/* Public API */ +int32 +ts_stats_chunk_segment_prepare_upsert(TsStatsChunkSegment *seg, TsStatsRelids relids) +{ + Assert(seg != NULL); + if (seg == NULL || !OidIsValid(relids.compressed_relid) || + !OidIsValid(relids.uncompressed_relid)) + { + return TS_STATS_INVALID_IDX; + } + + /* get the base pointers */ + TsStatsChunkMetadata *meta_base = ts_stats_chunk_metadata(seg); + TsStatsChunk *slot_base = ts_stats_chunk_slots(seg); + + /* sort the buckets by the last update sequence number in descending order */ + uint8 bucket_order[TS_STATS_BUCKETS]; + get_sorted_bucket_in_desc_order(seg, bucket_order); + + /* get the home location within the buckets */ + uint32 home = hash_relid(seg, relids.compressed_relid); + uint32 bucket_size = seg->num_slots / TS_STATS_BUCKETS; + + /* keep the empty positions as we scan forward. these will + * be tried if the chunk was not found. */ + uint32 empty_positions[TS_STATS_PROBE_WIDTH * TS_STATS_BUCKETS]; + uint32 empty_count = 0; + + /* iterate over the buckets in most-recently-updated order */ + for (uint8 i = 0; i < TS_STATS_BUCKETS; i++) + { + uint8 bucket_idx = bucket_order[i]; + uint32 start = bucket_idx * bucket_size; + + /* linear probe within the bucket */ + for (uint8 j = 0; j < TS_STATS_PROBE_WIDTH; j++) + { + uint32 idx = start + ((home + j) % bucket_size); + TsStatsChunkMetadata *meta = &meta_base[idx]; + + uint64 state = pg_atomic_read_u64(&meta->state); + if (!TS_STATS_CHUNK_METADATA_IS_VALID(state)) + { + /* record the empty position for later use, but we still need + * to continue scanning the remaining slots */ + empty_positions[empty_count++] = idx; + continue; + } + if (meta->relids.compressed_relid == relids.compressed_relid) + { + /* update the chunk metadata state to in-progress and the bucket's + * last update sequence number. + */ + if (ts_stats_chunk_update_at(seg, relids, idx, state) != TS_STATS_INVALID_IDX) + { + return (int32) idx; + } + + /* if the update failed, then another process has updated this slot. + * if the slot still belongs to the same compressed_relid, we can try + * to update it again, so the update sequence numbers get updated. + */ + state = pg_atomic_read_u64(&meta->state); + if (TS_STATS_CHUNK_METADATA_IS_VALID(state) && + meta->relids.compressed_relid == relids.compressed_relid) + { + if (ts_stats_chunk_update_at(seg, relids, idx, state) != TS_STATS_INVALID_IDX) + { + return (int32) idx; + } + } + + /* if this failed at the second time, we better leave it alone and proceed + * with the next slot. + */ + } + } + } + + /* at this point we haven't found the chunk so we first check the empty slots. + * they may be taken conncurrently by another process, so they need to be checked + * again. the order of the empty slots are somwewhat ordered as we gathered them + * in the bucket recency order. best to traverse them in this recency order so + * the buckets keep the recency property as much as possible. + */ + for (uint32 i = 0; i < empty_count; i++) + { + uint32 idx = empty_positions[i]; + TsStatsChunkMetadata *meta = &meta_base[idx]; + + uint64 state = pg_atomic_read_u64(&meta->state); + if (TS_STATS_CHUNK_METADATA_IS_VALID(state)) + { + /* this slot is no longer empty, skip it */ + continue; + } + + /* try to claim this slot, and it will update its state to in-progress + * and the bucket's last update sequence number. + */ + if (ts_stats_chunk_update_at(seg, relids, idx, state) != TS_STATS_INVALID_IDX) + { + TsStatsChunk *slot = &slot_base[idx]; + ts_stats_chunk_init_slot(slot); + return (int32) idx; + } + } + + /* at this point we have not found the slot and neither empty ones to use + * we will need to evict (overwrite) one. the eviction only checks the + * oldest bucket's window and uses the slot that is the oldest and not + * in-progress. if every slot is in progress we try the next bucket in + * recency order. if none has an evictable slot, we give up and return + * invalid index. + * + * the worst case is that we checked the windows of all buckets and we + * tried to evict from all slots, so that is + * + * 2 x TS_STATS_BUCKETS x TS_STATS_PROBE_WIDTH checks, which is + * + * still acceptable, but very unlikely to happen. + */ + + uint8 oldest_bucket_idx = bucket_order[TS_STATS_BUCKETS - 1]; + uint32 start = oldest_bucket_idx * bucket_size; + int32 candidate_idx = TS_STATS_INVALID_IDX; + uint64 candidate_seqno = UINT64_MAX; + + for (uint8 j = 0; j < TS_STATS_PROBE_WIDTH; j++) + { + uint32 idx = start + ((home + j) % bucket_size); + TsStatsChunkMetadata *meta = &meta_base[idx]; + + uint64 state = pg_atomic_read_u64(&meta->state); + if (!TS_STATS_CHUNK_METADATA_IS_VALID(state)) + { + /* this slot is empty, although at this point it should not happen + * as the previous loops should have handled empty slots. */ + if (ts_stats_chunk_update_at(seg, relids, idx, state) != TS_STATS_INVALID_IDX) + { + TsStatsChunk *slot = &slot_base[idx]; + ts_stats_chunk_init_slot(slot); + return (int32) idx; + } + } + if (!TS_STATS_CHUNK_METADATA_IS_IN_PROGRESS(state)) + { + /* this slot is not in progress, mark as a candidate */ + uint64 seqno = TS_STATS_CHUNK_METADATA_GET_SEQNO(state); + if (seqno < candidate_seqno) + { + candidate_seqno = seqno; + candidate_idx = (int32) idx; + } + } + } + + /* if we found a candidate, try to evict it */ + if (candidate_idx != TS_STATS_INVALID_IDX) + { + TsStatsChunkMetadata *meta = &meta_base[candidate_idx]; + uint64 state = pg_atomic_read_u64(&meta->state); + + /* try to evict this slot, and it will update its state to in-progress + * and the bucket's last update sequence number. + */ + int32 updated_idx = ts_stats_chunk_update_at(seg, relids, candidate_idx, state); + + if (updated_idx != TS_STATS_INVALID_IDX) + { + TsStatsChunk *slot = &slot_base[updated_idx]; + ts_stats_chunk_init_slot(slot); + return updated_idx; + } + } + + return TS_STATS_INVALID_IDX; +} + +void +ts_stats_chunk_segment_finish_upsert(TsStatsChunkSegment *seg, int32 slot_idx, Oid compressed_relid) +{ + Assert(seg != NULL); + if (seg == NULL || slot_idx == TS_STATS_INVALID_IDX || !OidIsValid(compressed_relid)) + { + return; + } + + /* get the metadata pointer */ + TsStatsChunkMetadata *meta = ts_stats_chunk_metadata(seg) + slot_idx; + + /* don't do anything if the compressed_relid doesn't match */ + if (meta->relids.compressed_relid != compressed_relid) + { + return; + } + + /* get the current flag */ + uint64 state = pg_atomic_read_u64(&meta->state); + + /* only update if the slot is in the in-progress state and still valid */ + if (TS_STATS_CHUNK_METADATA_IS_IN_PROGRESS(state) && TS_STATS_CHUNK_METADATA_IS_VALID(state)) + { + /* clear the in-progress flag */ + uint64 new_state = (state & ~TS_STATS_CHUNK_METADATA_IN_PROGRESS_FLAG); + + /* we could do compare-exchange here, but we have no means to handle failure + * and we don't want to wait or retry, so better to simply overwrite it */ + pg_atomic_write_u64(&meta->state, new_state); + } +} + +int32 +ts_stats_chunk_segment_lookup(TsStatsChunkSegment *seg, Oid compressed_relid) +{ + Assert(seg != NULL); + if (seg == NULL || !OidIsValid(compressed_relid)) + { + return TS_STATS_INVALID_IDX; + } + + /* get the base pointers */ + TsStatsChunkMetadata *meta_base = ts_stats_chunk_metadata(seg); + + /* get the home location within the buckets */ + uint32 home = hash_relid(seg, compressed_relid); + uint32 bucket_size = seg->num_slots / TS_STATS_BUCKETS; + + /* sort the buckets by the last update sequence number in descending order */ + uint8 bucket_order[TS_STATS_BUCKETS]; + get_sorted_bucket_in_desc_order(seg, bucket_order); + + /* iterate over the buckets in most-recently-updated order */ + for (uint8 i = 0; i < TS_STATS_BUCKETS; i++) + { + uint8 bucket_idx = bucket_order[i]; + uint32 start = bucket_idx * bucket_size; + + /* linear probe within the bucket */ + for (uint32 j = 0; j < TS_STATS_PROBE_WIDTH; j++) + { + uint32 idx = start + ((home + j) % bucket_size); + TsStatsChunkMetadata *meta = &meta_base[idx]; + + uint64 state = pg_atomic_read_u64(&meta->state); + if (!TS_STATS_CHUNK_METADATA_IS_VALID(state)) + { + /* even if the current slot was evicted, we still need to check + * the all remaining slots in the bucket and all buckets until we + * exhausted the all, because the eviction may have removed old + * entried in the middle of the search window + */ + continue; + } + if (meta->relids.compressed_relid == compressed_relid) + { + return (int32) idx; + } + } + } + return TS_STATS_INVALID_IDX; +} diff --git a/src/ts_stats/ts_stats_segment.h b/src/ts_stats/ts_stats_segment.h new file mode 100644 index 00000000000..5ed3e106236 --- /dev/null +++ b/src/ts_stats/ts_stats_segment.h @@ -0,0 +1,55 @@ +/* + * This file and its contents are licensed under the Apache License 2.0. + * Please see the included NOTICE for copyright information and + * LICENSE-APACHE for a copy of the license. + */ +#pragma once + +#include "ts_stats_defs.h" + +/* + * Returns the chunk statistics segment for the current database, creating it + * lazily on first call. Returns NULL if statistics collection is disabled + * (max_chunks == 0), if DSM allocation fails, or if a published segment is + * found with an unexpected magic, i.e. not initialized. + * + * The returned pointer is cached per-backend. + */ +TsStatsChunkSegment *ts_get_stats_chunk_segment(void); + +/* + * Logical reset — clears all cached chunks WITHOUT touching the segment's + * structural fields. + */ +void ts_stats_chunk_segment_reset(TsStatsChunkSegment *seg); + +/* Try to find the location of the next available slot for upsert. + * - if the chunk already exists, return its slot index, and sets the + * in-progress flag for the slot + * - if the chunk doesn't exists, find try to find an empty slot for it and + * sets the in-progress flag for the slot + * - if there are no empty slots, find the oldest candidate in the oldest + * bucket and evict it by updating the slot's compressed_relid and setting + * the in-progress flag for the slot + * - the search is limited to the search window (TS_STATS_PROBE_WIDTH slots) in each + * bucket, and the buckets are traversed in recency order. if all slots are + * in-progress in the window, the function returns TS_STATS_INVALID_IDX + * + * Note that this call updates: + * - the in-progress flag of the slot, as mentioned above + * - the sequence number of the slot, which by itself should prevent eviction + * - the last update sequence for the bucket, which affects the recency order of the buckets + */ +int32 ts_stats_chunk_segment_prepare_upsert(TsStatsChunkSegment *seg, TsStatsRelids relids); + +/* Once the caller prepared the slot for upsert and updated the stats, it should call this + * function to clear the in-progress flag and thus make the slot eligible for eviction and + * communicate to the readers that the slot can be read consistently. + */ +void ts_stats_chunk_segment_finish_upsert(TsStatsChunkSegment *seg, int32 slot_idx, + Oid compressed_relid); + +/* Lookup — find an existing slot without allocating a new one. Return the global position of the + * slot within all slots, ignoring buckets. Returns TS_STATS_INVALID_IDX on miss. + */ +int32 ts_stats_chunk_segment_lookup(TsStatsChunkSegment *seg, Oid compressed_relid); diff --git a/src/ts_stats/ts_stats_srf.c b/src/ts_stats/ts_stats_srf.c new file mode 100644 index 00000000000..087a071a115 --- /dev/null +++ b/src/ts_stats/ts_stats_srf.c @@ -0,0 +1,437 @@ +/* + * This file and its contents are licensed under the Apache License 2.0. + * Please see the included NOTICE for copyright information and + * LICENSE-APACHE for a copy of the license. + */ + +#include "ts_stats_defs.h" +#include "ts_stats_segment.h" + +#include +#include +#include +#include +#include +#include + +/* + * SRF Pushdown args: + * $1 compressed_relid regclass — point lookup by hash key; emits 0 or 1 row. + * $2 uncompressed_relid regclass — linear scan, or cross-checks $1. + * $3 since timestamptz — recency walk; LRU short-circuit, O(matches). + * All NULL → full scan. If both relid args are set, both must agree. + */ + +TS_FUNCTION_INFO_V1(ts_stats_chunks); +TS_FUNCTION_INFO_V1(ts_stats_reset); + +/* + * 2 relids + 2 counts (batch, column) + 9 compression min/max/sum + * (batch_rows, batch_bytes, column_bytes) + 3 sqsum (one per MinMaxSumData) + * + 10 cmd_totals + 10 cmd_last_op + 4 n_* + 2 timestamps = 42. + */ +#define STATS_CHUNKS_NCOLS 42 +#define ARG_COMPRESSED_RELID 0 +#define ARG_UNCOMPRESSED_RELID 1 +#define ARG_SINCE 2 + +Datum +ts_stats_chunks(PG_FUNCTION_ARGS) +{ + ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + + InitMaterializedSRF(fcinfo, 0); + TupleDesc tupdesc = rsinfo->setDesc; + Tuplestorestate *tupstore = rsinfo->setResult; + + TsStatsChunkSegment *seg = ts_get_stats_chunk_segment(); + if (seg == NULL) + { + PG_RETURN_VOID(); + } + + bool comp_filter = !PG_ARGISNULL(ARG_COMPRESSED_RELID); + bool uncomp_filter = !PG_ARGISNULL(ARG_UNCOMPRESSED_RELID); + Oid comp_oid = comp_filter ? PG_GETARG_OID(ARG_COMPRESSED_RELID) : InvalidOid; + Oid uncomp_oid = uncomp_filter ? PG_GETARG_OID(ARG_UNCOMPRESSED_RELID) : InvalidOid; + + if (comp_filter && !OidIsValid(comp_oid)) + { + /* don't return any rows for invalid Oid */ + PG_RETURN_VOID(); + } + + if (uncomp_filter && !OidIsValid(uncomp_oid)) + { + /* don't return any rows for invalid Oid */ + PG_RETURN_VOID(); + } + + bool since_filter = !PG_ARGISNULL(ARG_SINCE); + uint64 since_offset = 0; + + /* base_timestamp is set once at init */ + TimestampTz since_ts = since_filter ? PG_GETARG_TIMESTAMPTZ(ARG_SINCE) : 0; + if (since_filter) + { + if (since_ts <= seg->base_timestamp) + { + /* if threshold is earlier than the base, it means all record matches, so we can disable + * the filter */ + since_filter = false; + } + else + { + since_offset = (uint64) (since_ts - seg->base_timestamp); + } + } + + /* First snapshot the values */ + TsStatsChunk *slot_snap = NULL; + TsStatsChunkMetadata *meta_snap = NULL; + bool *retry_map = NULL; + uint32 n_retry_entries = 0; + uint32 first_to_try = seg->num_slots - 1; + uint32 last_to_try = 0; + uint32 snap_count = 0; + TimestampTz base = 0; + + base = seg->base_timestamp; + TsStatsChunk *slot_base = ts_stats_chunk_slots(seg); + TsStatsChunkMetadata *meta_base = ts_stats_chunk_metadata(seg); + + if (comp_filter) + { + /* Point lookup based on compressed_relid */ + int32 slot_idx = ts_stats_chunk_segment_lookup(seg, comp_oid); + if (slot_idx == TS_STATS_INVALID_IDX) + { + /* no matching record found, return empty set */ + PG_RETURN_VOID(); + } + + TsStatsChunkMetadata *meta = &meta_base[slot_idx]; + TsStatsChunk *slot = &slot_base[slot_idx]; + + if (uncomp_filter) + { + /* if uncompressed_relid is also provided, check the uncompressed_relid matches as well + */ + if (meta->relids.uncompressed_relid != uncomp_oid) + { + /* no matching record found, return empty set */ + PG_RETURN_VOID(); + } + } + + if (since_filter) + { + uint64 last_update_us = slot->last_update_us; + if (last_update_us == 0 || last_update_us < since_offset) + { + /* record is older than the threshold, return empty set */ + PG_RETURN_VOID(); + } + } + + uint64 state = pg_atomic_read_u64(&meta->state); + if (TS_STATS_CHUNK_METADATA_IS_VALID(state) == 0) + { + /* slot is not valid anymore, return empty set */ + PG_RETURN_VOID(); + } + + if (TS_STATS_CHUNK_METADATA_IS_IN_PROGRESS(state)) + { + /* slot is being updated, mark it for retry */ + retry_map = palloc0(sizeof(bool) * seg->num_slots); + retry_map[slot_idx] = true; + n_retry_entries = 1; + first_to_try = last_to_try = slot_idx; + } + else + { + /* safe to read, take a snapshot */ + slot_snap = palloc(sizeof(TsStatsChunk)); + meta_snap = palloc(sizeof(TsStatsChunkMetadata)); + memcpy(slot_snap, slot, sizeof(TsStatsChunk)); + memcpy(meta_snap, meta, sizeof(TsStatsChunkMetadata)); + snap_count = 1; + } + } + else if (uncomp_filter) + { + /* Linear scan of the metadata matching uncompressed_relid, but only one */ + slot_snap = palloc(sizeof(TsStatsChunk)); + meta_snap = palloc(sizeof(TsStatsChunkMetadata)); + retry_map = palloc0(sizeof(bool) * seg->num_slots); + + for (uint32 i = 0; i < seg->num_slots; i++) + { + TsStatsChunk *slot = &slot_base[i]; + TsStatsChunkMetadata *meta = &meta_base[i]; + + uint64 state = pg_atomic_read_u64(&meta->state); + if (!TS_STATS_CHUNK_METADATA_IS_VALID(state)) + { + /* don't deal with evicted/empty entries */ + continue; + } + if (meta->relids.uncompressed_relid != uncomp_oid) + { + continue; + } + /* the update timestamp is read without checking the in-progress flag + * because its meaning doesn't depend on other fields and it is consistent + * as it is. the worst case is that we might miss an update here, but + * the in-progress flag is about the overall state of the chunk slot + */ + if (since_filter && slot->last_update_us < since_offset) + { + continue; + } + + if (TS_STATS_CHUNK_METADATA_IS_IN_PROGRESS(state)) + { + /* slot is being updated, mark it for retry */ + retry_map[i] = true; + ++n_retry_entries; + if (i < first_to_try) + { + first_to_try = i; + } + if (i > last_to_try) + { + last_to_try = i; + } + } + else + { + /* safe to read, take a snapshot */ + memcpy(slot_snap, slot, sizeof(TsStatsChunk)); + memcpy(meta_snap, meta, sizeof(TsStatsChunkMetadata)); + snap_count = 1; + } + /* either we did the snapshot or marked it for retry, in both + * cases we can stop the loop. */ + break; + } + } + else + { + /* full scan in slot order. check since timestamp if needed. */ + slot_snap = palloc(sizeof(TsStatsChunk) * seg->num_slots); + meta_snap = palloc(sizeof(TsStatsChunkMetadata) * seg->num_slots); + retry_map = palloc0(sizeof(bool) * seg->num_slots); + + for (uint32 i = 0; i < seg->num_slots; i++) + { + TsStatsChunk *slot = &slot_base[i]; + TsStatsChunkMetadata *meta = &meta_base[i]; + + uint64 state = pg_atomic_read_u64(&meta->state); + if (!TS_STATS_CHUNK_METADATA_IS_VALID(state)) + { + /* don't deal with evicted/empty entries */ + continue; + } + /* the update timestamp is read without checking the in-progress flag + * because its meaning doesn't depend on other fields and it is consistent + * as it is. the worst case is that we might miss an update here, but + * the in-progress flag is about the overall state of the chunk slot + */ + if (since_filter && slot->last_update_us < since_offset) + { + continue; + } + + if (TS_STATS_CHUNK_METADATA_IS_IN_PROGRESS(state)) + { + /* slot is being updated, mark it for retry */ + retry_map[i] = true; + ++n_retry_entries; + if (i < first_to_try) + { + first_to_try = i; + } + if (i > last_to_try) + { + last_to_try = i; + } + } + else + { + /* safe to read, take a snapshot */ + memcpy(&slot_snap[snap_count], slot, sizeof(TsStatsChunk)); + memcpy(&meta_snap[snap_count], meta, sizeof(TsStatsChunkMetadata)); + snap_count++; + } + } + } + + /* give three attempts to retry the in-progress entries, use + * first_to_try and last_to_try to limit the range + */ + for (uint8 r = 0; r < 3 && n_retry_entries > 0; r++) + { + for (uint32 i = first_to_try; i <= last_to_try; i++) + { + if (!retry_map[i]) + { + /* this entry is not marked for retry, skip it */ + continue; + } + + TsStatsChunk *slot = &slot_base[i]; + TsStatsChunkMetadata *meta = &meta_base[i]; + + uint64 state = pg_atomic_read_u64(&meta->state); + if (TS_STATS_CHUNK_METADATA_IS_IN_PROGRESS(state)) + { + /* still in progress, keep it in the retry map for the next round */ + continue; + } + + /* no longer in progress, take a snapshot and remove it from the retry map */ + memcpy(&slot_snap[snap_count], slot, sizeof(TsStatsChunk)); + memcpy(&meta_snap[snap_count], meta, sizeof(TsStatsChunkMetadata)); + snap_count++; + retry_map[i] = false; + n_retry_entries--; + } + } + + /* Build the result tuples */ + Datum values[STATS_CHUNKS_NCOLS]; + bool nulls[STATS_CHUNKS_NCOLS] = { false }; + + for (uint32 i = 0; i < snap_count; i++) + { + TsStatsChunk *s = &slot_snap[i]; + TsStatsChunkMetadata *m = &meta_snap[i]; + int c = 0; + + values[c++] = ObjectIdGetDatum(m->relids.compressed_relid); + values[c++] = ObjectIdGetDatum(m->relids.uncompressed_relid); + + /* Compression aggregates */ + values[c++] = Int64GetDatum((int64) pg_atomic_read_u32(&s->compressed_batch_count)); + values[c++] = Int64GetDatum((int64) pg_atomic_read_u32(&s->compressed_block_count)); + + if (pg_atomic_read_u32(&s->compressed_batch_count) == 0) + { + for (int j = 0; j < 8; j++) + { + nulls[c + j] = true; + } + c += 8; + } + else + { + values[c++] = + Int64GetDatum((int64) pg_atomic_read_u32(&s->compressed_batch_rows.min_val)); + values[c++] = + Int64GetDatum((int64) pg_atomic_read_u32(&s->compressed_batch_rows.max_val)); + values[c++] = + Int64GetDatum((int64) pg_atomic_read_u64(&s->compressed_batch_rows.sum_val)); + values[c++] = Float8GetDatum(mms_sqsum(&s->compressed_batch_rows)); + values[c++] = + Int64GetDatum((int64) pg_atomic_read_u32(&s->compressed_batch_bytes.min_val)); + values[c++] = + Int64GetDatum((int64) pg_atomic_read_u32(&s->compressed_batch_bytes.max_val)); + values[c++] = + Int64GetDatum((int64) pg_atomic_read_u64(&s->compressed_batch_bytes.sum_val)); + values[c++] = Float8GetDatum(mms_sqsum(&s->compressed_batch_bytes)); + } + + if (pg_atomic_read_u32(&s->compressed_block_count) == 0) + { + for (int j = 0; j < 4; j++) + { + nulls[c + j] = true; + } + c += 4; + } + else + { + values[c++] = + Int64GetDatum((int64) pg_atomic_read_u32(&s->compressed_block_bytes.min_val)); + values[c++] = + Int64GetDatum((int64) pg_atomic_read_u32(&s->compressed_block_bytes.max_val)); + values[c++] = + Int64GetDatum((int64) pg_atomic_read_u64(&s->compressed_block_bytes.sum_val)); + values[c++] = Float8GetDatum(mms_sqsum(&s->compressed_block_bytes)); + } + + /* Cmd totals */ + values[c++] = Int64GetDatum(pg_atomic_read_u64(&s->cmd_totals.batches_deleted)); + values[c++] = Int64GetDatum(pg_atomic_read_u64(&s->cmd_totals.batches_decompressed)); + values[c++] = Int64GetDatum(pg_atomic_read_u64(&s->cmd_totals.tuples_decompressed)); + values[c++] = Int64GetDatum(pg_atomic_read_u64(&s->cmd_totals.batches_scanned)); + values[c++] = Int64GetDatum(pg_atomic_read_u64(&s->cmd_totals.batches_checked_by_bloom)); + values[c++] = Int64GetDatum(pg_atomic_read_u64(&s->cmd_totals.batches_pruned_by_bloom)); + values[c++] = Int64GetDatum(pg_atomic_read_u64(&s->cmd_totals.batches_without_bloom)); + values[c++] = + Int64GetDatum(pg_atomic_read_u64(&s->cmd_totals.batches_bloom_false_positives)); + values[c++] = Int64GetDatum(pg_atomic_read_u64(&s->cmd_totals.batches_filtered_compressed)); + values[c++] = + Int64GetDatum(pg_atomic_read_u64(&s->cmd_totals.batches_filtered_decompressed)); + + /* Last cmd stats */ + values[c++] = Int64GetDatum(pg_atomic_read_u64(&s->cmd_last_op.batches_deleted)); + values[c++] = Int64GetDatum(pg_atomic_read_u64(&s->cmd_last_op.batches_decompressed)); + values[c++] = Int64GetDatum(pg_atomic_read_u64(&s->cmd_last_op.tuples_decompressed)); + values[c++] = Int64GetDatum(pg_atomic_read_u64(&s->cmd_last_op.batches_scanned)); + values[c++] = Int64GetDatum(pg_atomic_read_u64(&s->cmd_last_op.batches_checked_by_bloom)); + values[c++] = Int64GetDatum(pg_atomic_read_u64(&s->cmd_last_op.batches_pruned_by_bloom)); + values[c++] = Int64GetDatum(pg_atomic_read_u64(&s->cmd_last_op.batches_without_bloom)); + values[c++] = + Int64GetDatum(pg_atomic_read_u64(&s->cmd_last_op.batches_bloom_false_positives)); + values[c++] = + Int64GetDatum(pg_atomic_read_u64(&s->cmd_last_op.batches_filtered_compressed)); + values[c++] = + Int64GetDatum(pg_atomic_read_u64(&s->cmd_last_op.batches_filtered_decompressed)); + + /* Operation counts */ + values[c++] = Int64GetDatum((int64) pg_atomic_read_u32(&s->n_selects)); + values[c++] = Int64GetDatum((int64) pg_atomic_read_u32(&s->n_inserts)); + values[c++] = Int64GetDatum((int64) pg_atomic_read_u32(&s->n_updates)); + values[c++] = Int64GetDatum((int64) pg_atomic_read_u32(&s->n_deletes)); + + /* Timestamps */ + values[c++] = TimestampTzGetDatum(base + (TimestampTz) s->first_update_us); + values[c++] = TimestampTzGetDatum(base + (TimestampTz) s->last_update_us); + + Assert(c == STATS_CHUNKS_NCOLS); + + tuplestore_putvalues(tupstore, tupdesc, values, nulls); + + /* Reset nulls for the next iteration. */ + for (int j = 0; j < STATS_CHUNKS_NCOLS; j++) + { + nulls[j] = false; + } + } + + /* free snapshots */ + if (slot_snap != NULL) + { + pfree(slot_snap); + } + if (meta_snap != NULL) + { + pfree(meta_snap); + } + PG_RETURN_VOID(); +} + +Datum +ts_stats_reset(PG_FUNCTION_ARGS) +{ + TsStatsChunkSegment *seg = ts_get_stats_chunk_segment(); + if (seg != NULL) + { + ts_stats_chunk_segment_reset(seg); + } + PG_RETURN_VOID(); +} diff --git a/test/expected/c_unit_tests.out b/test/expected/c_unit_tests.out index 939302f0d05..dd62af76c57 100644 --- a/test/expected/c_unit_tests.out +++ b/test/expected/c_unit_tests.out @@ -16,6 +16,8 @@ CREATE OR REPLACE FUNCTION test.jsonb_utils() RETURNS VOID AS :MODULE_PATHNAME, 'ts_test_jsonb_utils' LANGUAGE C; CREATE OR REPLACE FUNCTION test.compression_settings() RETURNS VOID AS :MODULE_PATHNAME, 'ts_test_compression_settings' LANGUAGE C; +CREATE OR REPLACE FUNCTION test.chunk_stats() RETURNS VOID +AS :MODULE_PATHNAME, 'ts_test_chunk_stats' LANGUAGE C; SET ROLE :ROLE_DEFAULT_PERM_USER; SELECT test.time_to_internal_conversion(); time_to_internal_conversion @@ -52,3 +54,9 @@ SELECT test.compression_settings(); ---------------------- +SELECT test.chunk_stats(); +NOTICE: Eviction test completed: 9216 evictions, 1024 slots, average lifespan 1023.53, lifespan stddev 167.91, min lifespan 138, max lifespan 1975 + chunk_stats +------------- + + diff --git a/test/expected/observ_funcs.out b/test/expected/observ_funcs.out new file mode 100644 index 00000000000..b0e993583bb --- /dev/null +++ b/test/expected/observ_funcs.out @@ -0,0 +1,55 @@ +-- This file and its contents are licensed under the Apache License 2.0. +-- Please see the included NOTICE for copyright information and +-- LICENSE-APACHE for a copy of the license. +-- Smoke tests. +-- Empty cache returns no rows. +SELECT count(*) FROM _timescaledb_functions.chunk_statistics(); + count +------- + 0 + +-- Pushdown: invalid OID returns no rows (cannot match any cached chunk). +SELECT count(*) FROM _timescaledb_functions.chunk_statistics(compressed_relid => 0::regclass); + count +------- + 0 + +SELECT count(*) FROM _timescaledb_functions.chunk_statistics(uncompressed_relid => 0::regclass); + count +------- + 0 + +-- Pushdown: non-existent OID returns no rows. +SELECT count(*) FROM _timescaledb_functions.chunk_statistics(compressed_relid => 1::regclass); + count +------- + 0 + +SELECT count(*) FROM _timescaledb_functions.chunk_statistics(uncompressed_relid => 1::regclass); + count +------- + 0 + +-- Pushdown: far-future since returns no rows even if the cache is non-empty. +SELECT count(*) FROM _timescaledb_functions.chunk_statistics(since => '2999-01-01'::timestamptz); + count +------- + 0 + +-- Reset is callable; cache stays empty. +SELECT _timescaledb_functions.chunk_statistics_reset(); + chunk_statistics_reset +------------------------ + + +SELECT count(*) FROM _timescaledb_functions.chunk_statistics(); + count +------- + 0 + +-- The view exists and has the expected shape (column count is enough for a smoke). +SELECT count(*) FROM timescaledb_information.stat_chunk_activity; + count +------- + 0 + diff --git a/test/expected/pg_dump.out b/test/expected/pg_dump.out index 0382dacd2a0..ba68eff13e0 100644 --- a/test/expected/pg_dump.out +++ b/test/expected/pg_dump.out @@ -454,6 +454,7 @@ WHERE refclassid = 'pg_catalog.pg_extension'::pg_catalog.regclass AND timescaledb_information.job_history timescaledb_information.job_stats timescaledb_information.jobs + timescaledb_information.stat_chunk_activity -- Make sure we can't run our restoring functions as a normal perm user as that would disable functionality for the whole db \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER diff --git a/test/sql/CMakeLists.txt b/test/sql/CMakeLists.txt index 6c0562d1cd8..020c1247438 100644 --- a/test/sql/CMakeLists.txt +++ b/test/sql/CMakeLists.txt @@ -39,6 +39,7 @@ set(TEST_FILES insert_single.sql lateral.sql merge.sql + observ_funcs.sql partition.sql partition_coercion.sql partitioning.sql diff --git a/test/sql/c_unit_tests.sql b/test/sql/c_unit_tests.sql index e2a9d33c069..d2fe4a8fa39 100644 --- a/test/sql/c_unit_tests.sql +++ b/test/sql/c_unit_tests.sql @@ -24,6 +24,9 @@ AS :MODULE_PATHNAME, 'ts_test_jsonb_utils' LANGUAGE C; CREATE OR REPLACE FUNCTION test.compression_settings() RETURNS VOID AS :MODULE_PATHNAME, 'ts_test_compression_settings' LANGUAGE C; +CREATE OR REPLACE FUNCTION test.chunk_stats() RETURNS VOID +AS :MODULE_PATHNAME, 'ts_test_chunk_stats' LANGUAGE C; + SET ROLE :ROLE_DEFAULT_PERM_USER; SELECT test.time_to_internal_conversion(); @@ -33,4 +36,5 @@ SELECT test.time_utils(); SELECT test.bmslist_utils(); SELECT test.jsonb_utils(); SELECT test.compression_settings(); +SELECT test.chunk_stats(); diff --git a/test/sql/observ_funcs.sql b/test/sql/observ_funcs.sql new file mode 100644 index 00000000000..834b53ad34d --- /dev/null +++ b/test/sql/observ_funcs.sql @@ -0,0 +1,26 @@ +-- This file and its contents are licensed under the Apache License 2.0. +-- Please see the included NOTICE for copyright information and +-- LICENSE-APACHE for a copy of the license. + +-- Smoke tests. + +-- Empty cache returns no rows. +SELECT count(*) FROM _timescaledb_functions.chunk_statistics(); + +-- Pushdown: invalid OID returns no rows (cannot match any cached chunk). +SELECT count(*) FROM _timescaledb_functions.chunk_statistics(compressed_relid => 0::regclass); +SELECT count(*) FROM _timescaledb_functions.chunk_statistics(uncompressed_relid => 0::regclass); + +-- Pushdown: non-existent OID returns no rows. +SELECT count(*) FROM _timescaledb_functions.chunk_statistics(compressed_relid => 1::regclass); +SELECT count(*) FROM _timescaledb_functions.chunk_statistics(uncompressed_relid => 1::regclass); + +-- Pushdown: far-future since returns no rows even if the cache is non-empty. +SELECT count(*) FROM _timescaledb_functions.chunk_statistics(since => '2999-01-01'::timestamptz); + +-- Reset is callable; cache stays empty. +SELECT _timescaledb_functions.chunk_statistics_reset(); +SELECT count(*) FROM _timescaledb_functions.chunk_statistics(); + +-- The view exists and has the expected shape (column count is enough for a smoke). +SELECT count(*) FROM timescaledb_information.stat_chunk_activity; diff --git a/test/src/CMakeLists.txt b/test/src/CMakeLists.txt index c1ef2691592..12c428462df 100644 --- a/test/src/CMakeLists.txt +++ b/test/src/CMakeLists.txt @@ -5,6 +5,7 @@ set(SOURCES test_compression_settings.c test_bmslist_utils.c test_jsonb_utils.c + test_stats.c test_scanner.c test_time_to_internal.c test_time_utils.c diff --git a/test/src/test_stats.c b/test/src/test_stats.c new file mode 100644 index 00000000000..3494ed151fd --- /dev/null +++ b/test/src/test_stats.c @@ -0,0 +1,139 @@ +/* + * This file and its contents are licensed under the Apache License 2.0. + * Please see the included NOTICE for copyright information and + * LICENSE-APACHE for a copy of the license. + */ + +/* + * Unit tests for the chunk-stats for the parts that are hard to + * reach from SQL. + * + * The DSM/shmem layer is bypassed: we palloc a tiny segment in regular + * backend memory, set its structural fields. + */ + +#include +#include +#include + +#include "test_utils.h" + +#include "ts_stats/ts_stats_defs.h" +#include "ts_stats/ts_stats_segment.h" + +static TsStatsChunkSegment * +make_test_segment(uint32 num_slots) +{ + TsStatsChunkSegment *seg = palloc0(ts_stats_chunk_segment_size(num_slots)); + seg->magic = TS_STATS_SHMEM_MAGIC; + seg->dboid = MyDatabaseId; + seg->base_timestamp = GetCurrentTimestamp(); + seg->num_slots = num_slots; + return seg; +} + +static void +test_simple_upsert_lookup() +{ + TsStatsChunkSegment *seg = make_test_segment(16); + TsStatsRelids relids = { + .compressed_relid = 12345, + .uncompressed_relid = 54321, + }; + int32 slot_idx = ts_stats_chunk_segment_prepare_upsert(seg, relids); + Assert(slot_idx != TS_STATS_INVALID_IDX && "Failed to prepare upsert"); + ts_stats_chunk_segment_finish_upsert(seg, slot_idx, relids.compressed_relid); + Assert(ts_stats_chunk_segment_lookup(seg, relids.compressed_relid) == slot_idx && + "Lookup index does not match upsert index"); + pfree(seg); +} + +static int32 +insert_one_chunk(TsStatsChunkSegment *seg, TsStatsRelids relids) +{ + int32 slot_idx = ts_stats_chunk_segment_prepare_upsert(seg, relids); + Assert(slot_idx != TS_STATS_INVALID_IDX && "Failed to prepare upsert"); + ts_stats_chunk_segment_finish_upsert(seg, slot_idx, relids.compressed_relid); + return slot_idx; +} + +static void +test_eviction() +{ +#define NUM_SLOTS 1024 +#define MIN_LIFESPAN (NUM_SLOTS / TS_STATS_PROBE_WIDTH / TS_STATS_BUCKETS) + TsStatsChunkSegment *seg = make_test_segment(NUM_SLOTS); + TsStatsChunkMetadata *meta_base = ts_stats_chunk_metadata(seg); + Oid base_compressed_relid = 10000; + Oid uncompressed_relid = 54321; + int32 slot_indices[NUM_SLOTS] = { 0 }; + uint64 slot_seqno[NUM_SLOTS] = { 0 }; + uint64 sum_lifespan = 0; + double sumsq_lifespan = 0; + int32 num_evictions = 0; + uint64 min_lifespan = UINT64_MAX; + uint64 max_lifespan = 0; + + for (int j = 0; j < 10; j++) + { + for (int i = 0; i < NUM_SLOTS; i++) + { + /* + * the i,j makes sure we don't always hit the same slots which would favour the + * eviction policy. + */ + TsStatsRelids relids = { + .compressed_relid = base_compressed_relid + i + (j * 11 * NUM_SLOTS), + .uncompressed_relid = uncompressed_relid, + }; + int32 slot_idx = insert_one_chunk(seg, relids); + Assert(slot_idx != TS_STATS_INVALID_IDX && "Failed to prepare upsert"); + Assert(slot_idx < NUM_SLOTS && + "Slot index exceeds number of slots, eviction should have happened"); + if (slot_indices[slot_idx] != 0) + { + uint64 old_seqno = slot_seqno[slot_idx]; + uint64 new_seqno = pg_atomic_read_u64(&meta_base[slot_idx].state) >> + TS_STATS_CHUNK_METADATA_SEQNO_SHIFT; + uint64 lifespan = new_seqno - old_seqno; + sum_lifespan += lifespan; + sumsq_lifespan += (double) lifespan * lifespan; + num_evictions++; + if (lifespan < min_lifespan) + { + min_lifespan = lifespan; + } + if (lifespan > max_lifespan) + { + max_lifespan = lifespan; + } + Assert(new_seqno > (old_seqno + MIN_LIFESPAN) && + "Sequence number did not increase as expected after eviction"); + } + slot_indices[slot_idx] = relids.compressed_relid; + slot_seqno[slot_idx] = pg_atomic_read_u64(&meta_base[slot_idx].state) >> + TS_STATS_CHUNK_METADATA_SEQNO_SHIFT; + Assert(ts_stats_chunk_segment_lookup(seg, relids.compressed_relid) == slot_idx && + "Lookup index does not match upsert index"); + } + } + + elog(NOTICE, + "Eviction test completed: %d evictions, %d slots, average lifespan %.2f, lifespan stddev " + "%.2f, min lifespan %d, max lifespan %d", + num_evictions, + NUM_SLOTS, + (double) sum_lifespan / num_evictions, + sqrt((sumsq_lifespan / num_evictions) - + ((sum_lifespan / num_evictions) * (sum_lifespan / num_evictions))), + (int) min_lifespan, + (int) max_lifespan); + pfree(seg); +} + +TS_TEST_FN(ts_test_chunk_stats) +{ + test_simple_upsert_lookup(); + test_eviction(); + PG_RETURN_VOID(); +} diff --git a/tsl/src/chunk_split.c b/tsl/src/chunk_split.c index f7261f33362..3267064cb77 100644 --- a/tsl/src/chunk_split.c +++ b/tsl/src/chunk_split.c @@ -416,8 +416,10 @@ route_next_compressed_tuple(TupleTableSlot *slot, SplitContext *scontext, int *r tuple = ExecFetchSlotHeapTuple(slot, false, NULL); - RowDecompressor decompressor = - build_decompressor(slot->tts_tupleDescriptor, csp->noncompressed_tupdesc); + RowDecompressor decompressor = build_decompressor(slot->tts_tupleDescriptor, + csp->noncompressed_tupdesc, + csettings->fd.compress_relid, + csettings->fd.relid); heap_deform_tuple(tuple, decompressor.in_desc, diff --git a/tsl/src/compression/compression.c b/tsl/src/compression/compression.c index 157a9babe8d..2de4dc54e82 100644 --- a/tsl/src/compression/compression.c +++ b/tsl/src/compression/compression.c @@ -48,6 +48,7 @@ #include "ts_catalog/array_utils.h" #include "ts_catalog/catalog.h" #include "ts_catalog/compression_settings.h" +#include "ts_stats/ts_stats_record.h" #include /* @@ -1352,6 +1353,9 @@ row_compressor_init(RowCompressor *row_compressor, const CompressionSettings *se check_for_limited_size_compressors(row_compressor->per_column, row_compressor->n_input_columns); + row_compressor->cached_relids.compressed_relid = settings->fd.compress_relid; + row_compressor->cached_relids.uncompressed_relid = settings->fd.relid; + ts_stats_compression_acc_init(&row_compressor->observ_acc); MemoryContextSwitchTo(old_context); } @@ -1630,6 +1634,9 @@ row_compressor_build_tuple(RowCompressor *row_compressor) { row_compressor->compressed_values[compressed_col] = PointerGetDatum(compressed_data); + + ts_stats_compression_acc_column(&row_compressor->observ_acc, + VARSIZE_ANY_EXHDR(compressed_data)); } } else if (column->segment_info != NULL) @@ -1788,6 +1795,9 @@ row_compressor_flush(RowCompressor *row_compressor, BulkWriter *writer, bool cha row_compressor->rows_compressed_into_current_value); } + ts_stats_compression_acc_batch(&row_compressor->observ_acc, + row_compressor->rows_compressed_into_current_value); + MemoryContextSwitchTo(old_cxt); row_compressor_clear_batch(row_compressor, changed_groups); } @@ -1801,6 +1811,9 @@ row_compressor_reset(RowCompressor *row_compressor) void row_compressor_close(RowCompressor *row_compressor) { + /* Flush the observability data */ + ts_stats_chunk_record_compression(row_compressor->cached_relids, &row_compressor->observ_acc); + pfree(row_compressor->compressed_is_null); pfree(row_compressor->compressed_values); pfree(row_compressor->per_column); @@ -2006,7 +2019,7 @@ bulk_writer_close(BulkWriter *writer) **********************/ RowDecompressor -build_decompressor(const TupleDesc in_desc, const TupleDesc out_desc) +build_decompressor(const TupleDesc in_desc, const TupleDesc out_desc, Oid in_oid, Oid out_oid) { AttrNumber count_meta_attnum = InvalidAttrNumber; AttrMap *attrmap = build_decompress_attrmap(out_desc, in_desc, &count_meta_attnum); @@ -2049,9 +2062,21 @@ build_decompressor(const TupleDesc in_desc, const TupleDesc out_desc) detoaster_init(&decompressor.detoaster, CurrentMemoryContext); + row_decompressor_init_stats(&decompressor, in_oid, out_oid, CMD_SELECT); + return decompressor; } +void +row_decompressor_init_stats(RowDecompressor *decompressor, Oid compressed_relid, + Oid uncompressed_relid, CmdType cmd_type) +{ + memset(&decompressor->observ_counters, 0, sizeof(decompressor->observ_counters)); + decompressor->cached_relids.compressed_relid = compressed_relid; + decompressor->cached_relids.uncompressed_relid = uncompressed_relid; + decompressor->cmd_type = cmd_type; +} + void row_decompressor_reset(RowDecompressor *decompressor) { @@ -2061,9 +2086,22 @@ row_decompressor_reset(RowDecompressor *decompressor) decompressor->tuples_decompressed = 0; } +void +row_decompressor_flush_stats(RowDecompressor *decompressor) +{ + /* Flush the observability data */ + ts_stats_chunk_record_cmd(decompressor->cached_relids, + decompressor->cmd_type, + &decompressor->observ_counters); + /* Reset the cached relids */ + decompressor->cached_relids.compressed_relid = InvalidOid; + decompressor->cached_relids.uncompressed_relid = InvalidOid; +} + void row_decompressor_close(RowDecompressor *decompressor) { + row_decompressor_flush_stats(decompressor); MemoryContextDelete(decompressor->per_compressed_row_ctx); detoaster_close(&decompressor->detoaster); free_attrmap(decompressor->attrmap); @@ -2098,8 +2136,10 @@ decompress_chunk(Oid in_table, Oid out_table) PushActiveSnapshot(GetLatestSnapshot()); BulkWriter writer = bulk_writer_build(out_rel, 0); - RowDecompressor decompressor = - build_decompressor(RelationGetDescr(in_rel), RelationGetDescr(out_rel)); + RowDecompressor decompressor = build_decompressor(RelationGetDescr(in_rel), + RelationGetDescr(out_rel), + RelationGetRelid(in_rel), + RelationGetRelid(out_rel)); TupleTableSlot *slot = table_slot_create(in_rel, NULL); TableScanDesc scan = table_beginscan(in_rel, GetActiveSnapshot(), 0, (ScanKey) NULL); int64 report_reltuples = calculate_reltuples_to_report(in_rel->rd_rel->reltuples); diff --git a/tsl/src/compression/compression.h b/tsl/src/compression/compression.h index 11b5c0e7258..4121d2dff9c 100644 --- a/tsl/src/compression/compression.h +++ b/tsl/src/compression/compression.h @@ -6,6 +6,7 @@ #pragma once #include +#include "ts_stats/ts_stats_defs.h" #include #include #include @@ -20,6 +21,7 @@ typedef struct BulkInsertStateData *BulkInsertState; #include "hypertable.h" #include "nodes/columnar_scan/detoaster.h" #include "ts_catalog/compression_settings.h" +#include "ts_stats/ts_stats_record.h" /* * Compressed data starts with a specialized varlen type starting with the usual @@ -179,6 +181,10 @@ typedef struct RowDecompressor AttrMap *attrmap; Detoaster detoaster; + + TsStatsRelids cached_relids; + CmdType cmd_type; + SharedCounters observ_counters; } RowDecompressor; /* @@ -306,6 +312,10 @@ typedef struct RowCompressor bool needs_analyze_segmentby; List *metadata_builders; /* List of BatchMetadataBuilder */ + + TsStatsRelids cached_relids; + CompressionStatsAccumulator observ_acc; + } RowCompressor; /* @@ -429,10 +439,14 @@ extern void segment_info_update(SegmentInfo *segment_info, Datum val, bool is_nu extern BulkWriter bulk_writer_build(Relation out_rel, int insert_options); extern BulkWriter *bulk_writer_alloc(Relation out_rel, int insert_options); extern void bulk_writer_close(BulkWriter *writer); -extern RowDecompressor build_decompressor(const TupleDesc in_desc, const TupleDesc out_desc); +extern RowDecompressor build_decompressor(const TupleDesc in_desc, const TupleDesc out_desc, + Oid in_oid, Oid out_oid); extern void row_decompressor_reset(RowDecompressor *decompressor); extern void row_decompressor_close(RowDecompressor *decompressor); +extern void row_decompressor_init_stats(RowDecompressor *decompressor, Oid compressed_relid, + Oid uncompressed_relid, CmdType cmd_type); +extern void row_decompressor_flush_stats(RowDecompressor *decompressor); extern int decompress_batch(RowDecompressor *decompressor); extern bool decompress_batch_next_row(RowDecompressor *decompressor, AttrNumber *attnos, int num_attnos); diff --git a/tsl/src/compression/compression_dml.c b/tsl/src/compression/compression_dml.c index 298f97b3005..4dbb655fd34 100644 --- a/tsl/src/compression/compression_dml.c +++ b/tsl/src/compression/compression_dml.c @@ -22,6 +22,7 @@ #include #include "foreach_ptr.h" +#include "ts_stats/ts_stats_record.h" #include #include #include @@ -60,7 +61,7 @@ static struct decompress_batches_stats decompress_batches_scan(Relation in_rel, Relation out_rel, Relation index_rel, Snapshot snapshot, bool *skip_current_tuple, bool delete_only, List *is_nulls, InvalidationContext *invalidation_ctx, CachedDecompressionState *cdst, - TupleTableSlot *insert_slot); + TupleTableSlot *insert_slot, CmdType cmd_type); static BatchQualSummary batch_matches(RowDecompressor *decompressor, ScanKeyData *scankeys, int num_scankeys, tuple_filtering_constraints *constraints, @@ -92,6 +93,8 @@ static bool can_vectorize_constraint_checks(tuple_filtering_constraints *constra static void update_scankeys(ScanKeyWithAttnos *scankeys, TupleTableSlot *slot, int null_flags); static void init_upsert_bloom_state(ChunkInsertState *cis); static Bitmapset *get_arbiter_index_attnums(ChunkInsertState *cis); +static inline void copy_stats_to_observ_counters(RowDecompressor *d, + const struct decompress_batches_stats *s); static AttrNumber TupleDescGetAttrNumber(TupleDesc desc, const char *name) @@ -521,7 +524,9 @@ decompress_batches_for_insert(ChunkInsertState *cis, TupleTableSlot *slot) NIL, NULL /* no CAgg invalidation for inserts */, cdst, - slot); + slot, + CMD_INSERT); + if (index_rel) { index_close(index_rel, AccessShareLock); @@ -685,7 +690,8 @@ decompress_batches_for_update_delete(ModifyHypertableState *ht_state, Chunk *chu is_null, ht_state->has_continuous_aggregate ? &invalidation_ctx : NULL, &temp_cdst, - NULL); + NULL, + ht_state->mt->operation); /* close the selected index */ if (matching_index_rel) @@ -821,7 +827,7 @@ static struct decompress_batches_stats decompress_batches_scan(Relation in_rel, Relation out_rel, Relation index_rel, Snapshot snapshot, bool *skip_current_tuple, bool delete_only, List *is_nulls, InvalidationContext *invalidation_ctx, CachedDecompressionState *cdst, - TupleTableSlot *insert_slot) + TupleTableSlot *insert_slot, CmdType cmd_type) { HeapTuple compressed_tuple; BulkWriter writer; @@ -845,6 +851,16 @@ decompress_batches_scan(Relation in_rel, Relation out_rel, Relation index_rel, S struct decompress_batches_stats stats = { 0 }; + /* + * initialize stats related fields of the compressor, so it can be + * used even if the decompression is skipped by the bloom filter or + * other filters + */ + row_decompressor_init_stats(&decompressor, + RelationGetRelid(in_rel), + RelationGetRelid(out_rel), + cmd_type); + /* TODO: Optimization by reusing the index scan while working on a single chunk */ if (index_rel) @@ -961,7 +977,14 @@ decompress_batches_scan(Relation in_rel, Relation out_rel, Relation index_rel, S if (!decompressor_initialized) { - decompressor = build_decompressor(RelationGetDescr(in_rel), RelationGetDescr(out_rel)); + decompressor = build_decompressor(RelationGetDescr(in_rel), + RelationGetDescr(out_rel), + RelationGetRelid(in_rel), + RelationGetRelid(out_rel)); + + /* set the cmd type in the metrics record */ + decompressor.cmd_type = cmd_type; + decompressor_initialized = true; writer = bulk_writer_build(out_rel, 0); meta_count_attno = TupleDescGetAttrNumber(decompressor.in_desc, @@ -1040,6 +1063,7 @@ decompress_batches_scan(Relation in_rel, Relation out_rel, Relation index_rel, S if (skip_current_tuple && *skip_current_tuple) { + copy_stats_to_observ_counters(&decompressor, &stats); row_decompressor_close(&decompressor); bulk_writer_close(&writer); decompress_batch_endscan(scan); @@ -1075,6 +1099,7 @@ decompress_batches_scan(Relation in_rel, Relation out_rel, Relation index_rel, S if (result != TM_Ok) { write_logical_replication_msg_decompression_end(); + copy_stats_to_observ_counters(&decompressor, &stats); row_decompressor_close(&decompressor); bulk_writer_close(&writer); decompress_batch_endscan(scan); @@ -1116,11 +1141,22 @@ decompress_batches_scan(Relation in_rel, Relation out_rel, Relation index_rel, S } ExecDropSingleTupleTableSlot(slot); decompress_batch_endscan(scan); + copy_stats_to_observ_counters(&decompressor, &stats); + if (decompressor_initialized) { row_decompressor_close(&decompressor); bulk_writer_close(&writer); } + else + { + /* + * we accumulate stats in the decompressor even if + * skip decompression because of other filters like + * the bloom, null or index filter. + */ + row_decompressor_flush_stats(&decompressor); + } if (ts_guc_debug_compression_path_info) { @@ -2458,3 +2494,18 @@ can_vectorize_constraint_checks(tuple_filtering_constraints *constraints, return true; } + +static inline void +copy_stats_to_observ_counters(RowDecompressor *d, const struct decompress_batches_stats *s) +{ + d->observ_counters.batches_deleted = s->batches_deleted; + d->observ_counters.batches_decompressed = s->batches_decompressed; + d->observ_counters.tuples_decompressed = s->tuples_decompressed; + d->observ_counters.batches_scanned = s->batches_scanned; + d->observ_counters.batches_checked_by_bloom = s->batches_checked_by_bloom; + d->observ_counters.batches_pruned_by_bloom = s->batches_pruned_by_bloom; + d->observ_counters.batches_without_bloom = s->batches_without_bloom; + d->observ_counters.batches_bloom_false_positives = s->batches_bloom_false_positives; + d->observ_counters.batches_filtered_compressed = s->batches_filtered_compressed; + d->observ_counters.batches_filtered_decompressed = s->batches_filtered_decompressed; +} diff --git a/tsl/src/compression/recompress.c b/tsl/src/compression/recompress.c index 8d72369c6a6..373fa8eb3c5 100644 --- a/tsl/src/compression/recompress.c +++ b/tsl/src/compression/recompress.c @@ -334,7 +334,9 @@ recompress_chunk_segmentwise_impl(Chunk *uncompressed_chunk, /******************** row decompressor **************/ RowDecompressor decompressor = build_decompressor(RelationGetDescr(compressed_chunk_rel), - RelationGetDescr(uncompressed_chunk_rel)); + RelationGetDescr(uncompressed_chunk_rel), + RelationGetRelid(compressed_chunk_rel), + RelationGetRelid(uncompressed_chunk_rel)); /********** row compressor *******************/ RowCompressor row_compressor; @@ -769,7 +771,9 @@ perform_recompression(RecompressContext *recompress_ctx, Relation compressed_chu PushActiveSnapshot(GetTransactionSnapshot()); decompressor = build_decompressor(RelationGetDescr(compressed_chunk_rel), - RelationGetDescr(uncompressed_chunk_rel)); + RelationGetDescr(uncompressed_chunk_rel), + RelationGetRelid(compressed_chunk_rel), + RelationGetRelid(uncompressed_chunk_rel)); tuplesortstate = tuplesort_begin_heap(RelationGetDescr(uncompressed_chunk_rel), recompress_ctx->n_keys, diff --git a/tsl/src/nodes/columnar_scan/compressed_batch.c b/tsl/src/nodes/columnar_scan/compressed_batch.c index 678799738e1..c9bfa59b58f 100644 --- a/tsl/src/nodes/columnar_scan/compressed_batch.c +++ b/tsl/src/nodes/columnar_scan/compressed_batch.c @@ -195,6 +195,7 @@ decompress_column(DecompressContext *dcontext, DecompressBatchState *batch_state bool isnull; Datum value = slot_getattr(compressed_slot, column_description->compressed_scan_attno, &isnull); + dcontext->compressed_relid = compressed_slot->tts_tableOid; if (isnull) { @@ -228,6 +229,11 @@ decompress_column(DecompressContext *dcontext, DecompressBatchState *batch_state return; } + dcontext->batches_decompressed++; + /* to backfill the column's compressed bytes. */ + ts_stats_compression_acc_column(&dcontext->observ_acc, + VARSIZE_ANY_EXHDR(DatumGetPointer(value))); + /* Decompress the entire batch if it is supported. */ ArrowArray *arrow = NULL; if (dcontext->enable_bulk_decompression && column_description->bulk_decompression_supported) @@ -1031,6 +1037,9 @@ compressed_batch_set_compressed_tuple(DecompressContext *dcontext, } } + dcontext->tuples_decompressed += batch_state->total_batch_rows; + ts_stats_compression_acc_batch(&dcontext->observ_acc, batch_state->total_batch_rows); + CompressedBatchVectorQualState cbvqstate = { .vqstate = { .vectorized_quals_constified = dcontext->vectorized_quals_constified, diff --git a/tsl/src/nodes/columnar_scan/decompress_context.h b/tsl/src/nodes/columnar_scan/decompress_context.h index 37883e0cca1..2f558124ef3 100644 --- a/tsl/src/nodes/columnar_scan/decompress_context.h +++ b/tsl/src/nodes/columnar_scan/decompress_context.h @@ -14,6 +14,7 @@ #include "batch_array.h" #include "detoaster.h" +#include "ts_stats/ts_stats_record.h" typedef enum CompressionColumnType { @@ -98,6 +99,12 @@ typedef struct DecompressContext int32 chunk_status; + /* Stats */ + Oid compressed_relid; + uint64 batches_decompressed; + uint64 tuples_decompressed; + CompressionStatsAccumulator observ_acc; + } DecompressContext; #endif /* TIMESCALEDB_DECOMPRESS_CONTEXT_H */ diff --git a/tsl/src/nodes/columnar_scan/exec.c b/tsl/src/nodes/columnar_scan/exec.c index 7318fb4d742..aeec3235511 100644 --- a/tsl/src/nodes/columnar_scan/exec.c +++ b/tsl/src/nodes/columnar_scan/exec.c @@ -5,6 +5,7 @@ */ #include +#include "ts_stats/ts_stats_defs.h" #include #include #include @@ -31,6 +32,7 @@ #include "nodes/columnar_scan/exec.h" #include "nodes/columnar_scan/planner.h" #include "ts_catalog/array_utils.h" +#include "ts_stats/ts_stats_record.h" #if PG18_GE #include @@ -198,6 +200,8 @@ columnar_scan_begin(CustomScanState *node, EState *estate, int eflags) Plan *compressed_scan = linitial(cscan->custom_plans); Assert(list_length(cscan->custom_plans) == 1); + ts_stats_compression_acc_init(&dcontext->observ_acc); + PlanState *ps = &node->ss.ps; if (ps->ps_ProjInfo) { @@ -504,11 +508,24 @@ columnar_scan_end(CustomScanState *node) { ColumnarScanState *chunk_state = (ColumnarScanState *) node; BatchQueue *bq = chunk_state->batch_queue; + DecompressContext *dcontext = &chunk_state->decompress_context; bq->funcs->free(bq); ExecEndNode(linitial(node->custom_ps)); detoaster_close(&chunk_state->decompress_context.detoaster); + + /* Finish the observability aggregates. */ + SharedCounters sc = { 0 }; + sc.tuples_decompressed = dcontext->tuples_decompressed; + sc.batches_decompressed = dcontext->batches_decompressed; + TsStatsRelids relids = { + .compressed_relid = dcontext->compressed_relid, + .uncompressed_relid = chunk_state->chunk_relid, + }; + ts_stats_chunk_record_cmd(relids, CMD_SELECT, &sc); + + ts_stats_chunk_record_decompression(relids, &dcontext->observ_acc); } /* diff --git a/tsl/test/expected/compress_observ-15.out b/tsl/test/expected/compress_observ-15.out new file mode 100644 index 00000000000..d3ea65ba857 --- /dev/null +++ b/tsl/test/expected/compress_observ-15.out @@ -0,0 +1,1395 @@ +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license. +-- Test the integration of the compression and observability features +CREATE TABLE t(ts timestamptz, a int, b int, c int, d int, seg int); +SELECT create_hypertable('t', by_range('ts', interval '1 day')); + create_hypertable +------------------- + (1,t) + +CREATE VIEW observ AS +SELECT + -- skip the OIDs for test stability: + -- compressed_relid, + -- uncompressed_relid, + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_sum, compressed_batch_rows_sqsum, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_sum, compressed_batch_bytes_sqsum, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_sum, compressed_block_bytes_sqsum, + -- op totals + total_batches_deleted, total_batches_decompressed, + total_tuples_decompressed, total_batches_scanned, + total_batches_checked_by_bloom, total_batches_pruned_by_bloom, + total_batches_without_bloom, total_batches_bloom_false_positives, + total_batches_filtered_compressed, total_batches_filtered_decompressed, + -- last op + last_op_batches_deleted, last_op_batches_decompressed, + last_op_tuples_decompressed, last_op_batches_scanned, + last_op_batches_checked_by_bloom, last_op_batches_pruned_by_bloom, + last_op_batches_without_bloom, last_op_batches_bloom_false_positives, + last_op_batches_filtered_compressed, last_op_batches_filtered_decompressed, + -- op counters + n_selects, n_inserts, n_updates, n_deletes + -- skip the below two for test stability: + -- first_update, + -- last_update +FROM +( + SELECT * FROM _timescaledb_functions.chunk_statistics() + ORDER BY last_update ASC, compressed_relid ASC, uncompressed_relid ASC +) sub; +CREATE VIEW observ_main_view AS +SELECT + chunk, compressed_chunk, + chunk_id, hypertable_id, hypertable, + -- compression stats + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_avg, compressed_batch_rows_stddev, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_avg, compressed_batch_bytes_stddev, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_avg, compressed_block_bytes_stddev, + -- op totals + total_batches_deleted, total_batches_decompressed, + total_tuples_decompressed, total_batches_scanned, + total_batches_checked_by_bloom, total_batches_pruned_by_bloom, + total_batches_without_bloom, total_batches_bloom_false_positives, + total_batches_filtered_compressed, total_batches_filtered_decompressed, + -- last op stats + last_op_batches_deleted, last_op_batches_decompressed, + last_op_tuples_decompressed, last_op_batches_scanned, + last_op_batches_checked_by_bloom, last_op_batches_pruned_by_bloom, + last_op_batches_without_bloom, last_op_batches_bloom_false_positives, + last_op_batches_filtered_compressed, last_op_batches_filtered_decompressed, + -- op counters + n_selects, n_inserts, n_updates, n_deletes + -- these will make the test flaky, so skip them: + -- first_update, + -- last_update +FROM +( + SELECT * + FROM timescaledb_information.stat_chunk_activity + ORDER BY last_update ASC, hypertable_id ASC, chunk_id ASC +) sub; +-- Initial config: bloom(a,b) +ALTER TABLE t SET ( + timescaledb.compress, + timescaledb.compress_segmentby = 'seg', + timescaledb.compress_orderby = 'ts', + timescaledb.compress_index = 'bloom(a,b), bloom(d)' +); +INSERT INTO t +SELECT ts, (i % 10), (i % 5), (i % 3), i, 1 +FROM generate_series('2024-01-01'::timestamptz, '2024-01-03', interval '1 hour') ts, + generate_series(1, 100) i; +\pset expanded on +SELECT compress_chunk(c) FROM show_chunks('t') c; +-[ RECORD 1 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_1_chunk +-[ RECORD 2 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_2_chunk +-[ RECORD 3 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_3_chunk + +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 0 +total_tuples_decompressed | 0 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 0 +total_tuples_decompressed | 0 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 0 +total_tuples_decompressed | 0 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 + +SELECT COUNT(*) FROM t; +-[ RECORD 1 ] +count | 4900 + +SELECT count(*),min(ts),max(ts) from t where a % 19 = 9 or b % 19 = 9 or c % 19 = 2 or seg % 19 = 2; +-[ RECORD 1 ]----------------------- +count | 1960 +min | Mon Jan 01 00:00:00 2024 PST +max | Wed Jan 03 00:00:00 2024 PST + +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 8 +total_tuples_decompressed | 1600 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 8 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 12 +total_tuples_decompressed | 2400 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 12 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 4 +total_tuples_decompressed | 900 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 4 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 + +-- Test DELETE stats +BEGIN; +DELETE FROM t WHERE a % 19 = 2; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 10 +total_tuples_decompressed | 3200 +total_batches_scanned | 2 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 2 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 15 +total_tuples_decompressed | 4800 +total_batches_scanned | 3 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 3 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 5 +total_tuples_decompressed | 1800 +total_batches_scanned | 1 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 1 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 + +-- Test UPDATE stats +BEGIN; +UPDATE t SET a = 42 WHERE a % 19 = 2; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 12 +total_tuples_decompressed | 4800 +total_batches_scanned | 4 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 2 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 18 +total_tuples_decompressed | 7200 +total_batches_scanned | 6 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 3 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 6 +total_tuples_decompressed | 2700 +total_batches_scanned | 2 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 1 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 + +-- Show the information schema view as well: +SELECT * FROM observ_main_view; +-[ RECORD 1 ]-------------------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_1_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_4_chunk +chunk_id | 1 +hypertable_id | 1 +hypertable | t +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 200 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_avg | 2676 +compressed_batch_bytes_stddev | 584 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_avg | 535 +compressed_block_bytes_stddev | 273.740314897167 +total_batches_deleted | 0 +total_batches_decompressed | 12 +total_tuples_decompressed | 4800 +total_batches_scanned | 4 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 2 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 +-[ RECORD 2 ]-------------------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_2_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_5_chunk +chunk_id | 2 +hypertable_id | 1 +hypertable | t +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 282.842712474619 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_avg | 2676 +compressed_batch_bytes_stddev | 883.073420881111 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_avg | 535 +compressed_block_bytes_stddev | 309.946059823318 +total_batches_deleted | 0 +total_batches_decompressed | 18 +total_tuples_decompressed | 7200 +total_batches_scanned | 6 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 3 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 +-[ RECORD 3 ]-------------------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_3_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_6_chunk +chunk_id | 3 +hypertable_id | 1 +hypertable | t +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_avg | 900 +compressed_batch_rows_stddev | 0 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_avg | 2964 +compressed_batch_bytes_stddev | 0 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_avg | 592 +compressed_block_bytes_stddev | 259.505992223687 +total_batches_deleted | 0 +total_batches_decompressed | 6 +total_tuples_decompressed | 2700 +total_batches_scanned | 2 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 1 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 + +-- Reset the stats and check how we re-populate them as we select from the chunk again: +SELECT _timescaledb_functions.chunk_statistics_reset(); +-[ RECORD 1 ]----------+- +chunk_statistics_reset | + +SELECT count(*) FROM t where a % 19 = 9; +-[ RECORD 1 ] +count | 490 + +-- Only care about compression stats here: +SELECT + chunk, compressed_chunk, + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_avg, compressed_batch_rows_stddev, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_avg, compressed_batch_bytes_stddev, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_avg, compressed_block_bytes_stddev +FROM observ_main_view; +-[ RECORD 1 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_1_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_4_chunk +compressed_batch_count | 2 +compressed_block_count | 2 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 200 +compressed_batch_bytes_min | 0 +compressed_batch_bytes_max | 756 +compressed_batch_bytes_avg | 378 +compressed_batch_bytes_stddev | 378 +compressed_block_bytes_min | 460 +compressed_block_bytes_max | 756 +compressed_block_bytes_avg | 608 +compressed_block_bytes_stddev | 148 +-[ RECORD 2 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_2_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_5_chunk +compressed_batch_count | 3 +compressed_block_count | 3 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 282.842712474619 +compressed_batch_bytes_min | 0 +compressed_batch_bytes_max | 756 +compressed_batch_bytes_avg | 501 +compressed_batch_bytes_stddev | 354.51124414075 +compressed_block_bytes_min | 324 +compressed_block_bytes_max | 756 +compressed_block_bytes_avg | 609 +compressed_block_bytes_stddev | 201.787567065521 +-[ RECORD 3 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_3_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_6_chunk +compressed_batch_count | 0 +compressed_block_count | 1 +compressed_batch_rows_min | +compressed_batch_rows_max | +compressed_batch_rows_avg | +compressed_batch_rows_stddev | +compressed_batch_bytes_min | +compressed_batch_bytes_max | +compressed_batch_bytes_avg | +compressed_batch_bytes_stddev | +compressed_block_bytes_min | 676 +compressed_block_bytes_max | 676 +compressed_block_bytes_avg | 676 +compressed_block_bytes_stddev | 0 + +-- Select more and check how things change: +SELECT count(*),min(ts),max(ts) from t where a % 19 = 9 or b % 19 = 9 or c % 19 = 2 or seg % 19 = 2; +-[ RECORD 1 ]----------------------- +count | 1960 +min | Mon Jan 01 00:00:00 2024 PST +max | Wed Jan 03 00:00:00 2024 PST + +SELECT + chunk, compressed_chunk, + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_avg, compressed_batch_rows_stddev, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_avg, compressed_batch_bytes_stddev, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_avg, compressed_block_bytes_stddev +FROM observ_main_view; +-[ RECORD 1 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_1_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_4_chunk +compressed_batch_count | 2 +compressed_block_count | 8 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 200 +compressed_batch_bytes_min | 0 +compressed_batch_bytes_max | 756 +compressed_batch_bytes_avg | 378 +compressed_batch_bytes_stddev | 378 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 756 +compressed_block_bytes_avg | 430 +compressed_block_bytes_stddev | 168.321121669266 +-[ RECORD 2 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_2_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_5_chunk +compressed_batch_count | 3 +compressed_block_count | 12 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 282.842712474619 +compressed_batch_bytes_min | 0 +compressed_batch_bytes_max | 756 +compressed_batch_bytes_avg | 501 +compressed_batch_bytes_stddev | 354.51124414075 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 756 +compressed_block_bytes_avg | 431 +compressed_block_bytes_stddev | 201.075994478594 +-[ RECORD 3 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_3_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_6_chunk +compressed_batch_count | 0 +compressed_block_count | 4 +compressed_batch_rows_min | +compressed_batch_rows_max | +compressed_batch_rows_avg | +compressed_batch_rows_stddev | +compressed_batch_bytes_min | +compressed_batch_bytes_max | +compressed_batch_bytes_avg | +compressed_batch_bytes_stddev | +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 676 +compressed_block_bytes_avg | 482 +compressed_block_bytes_stddev | 150.983442800858 + +-- The stats should be evicted after we decompress and thus throwing away +-- the compressed chunks. +SELECT decompress_chunk(c) FROM show_chunks('t') c; +-[ RECORD 1 ]----+--------------------------------------- +decompress_chunk | _timescaledb_internal._hyper_1_1_chunk +-[ RECORD 2 ]----+--------------------------------------- +decompress_chunk | _timescaledb_internal._hyper_1_2_chunk +-[ RECORD 3 ]----+--------------------------------------- +decompress_chunk | _timescaledb_internal._hyper_1_3_chunk + +-- This should be empty after the decompression +SELECT + chunk, compressed_chunk, + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_avg, compressed_batch_rows_stddev, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_avg, compressed_batch_bytes_stddev, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_avg, compressed_block_bytes_stddev +FROM observ_main_view; + +-- Re-compress before doing the bloom tests +SELECT compress_chunk(c) FROM show_chunks('t') c; +-[ RECORD 1 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_1_chunk +-[ RECORD 2 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_2_chunk +-[ RECORD 3 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_3_chunk + +-- Test DELETE stats with bloom filter matches +BEGIN; +DELETE FROM t WHERE a = 7 AND b = 2; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 2 +total_tuples_decompressed | 1600 +total_batches_scanned | 2 +total_batches_checked_by_bloom | 2 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 2 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 2 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 3 +total_tuples_decompressed | 2400 +total_batches_scanned | 3 +total_batches_checked_by_bloom | 3 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 3 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 3 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 1 +total_tuples_decompressed | 900 +total_batches_scanned | 1 +total_batches_checked_by_bloom | 1 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 1 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 1 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 + +-- Test DELETE stats with bloom filter misses +BEGIN; +DELETE FROM t WHERE d = 200; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 2 +total_tuples_decompressed | 1600 +total_batches_scanned | 4 +total_batches_checked_by_bloom | 4 +total_batches_pruned_by_bloom | 2 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 2 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 2 +last_op_batches_pruned_by_bloom | 2 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 2 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 2 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 3 +total_tuples_decompressed | 2400 +total_batches_scanned | 6 +total_batches_checked_by_bloom | 6 +total_batches_pruned_by_bloom | 3 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 3 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 3 +last_op_batches_pruned_by_bloom | 3 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 3 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 2 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 1 +total_tuples_decompressed | 900 +total_batches_scanned | 2 +total_batches_checked_by_bloom | 2 +total_batches_pruned_by_bloom | 1 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 1 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 1 +last_op_batches_pruned_by_bloom | 1 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 1 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 2 + +-- Test UPDATE stats with bloom filter matches +BEGIN; +UPDATE t SET a = 42 WHERE a = 7 AND b = 2; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 4 +total_tuples_decompressed | 3200 +total_batches_scanned | 6 +total_batches_checked_by_bloom | 6 +total_batches_pruned_by_bloom | 2 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 2 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 2 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 2 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 1 +n_deletes | 2 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 6 +total_tuples_decompressed | 4800 +total_batches_scanned | 9 +total_batches_checked_by_bloom | 9 +total_batches_pruned_by_bloom | 3 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 3 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 3 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 3 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 1 +n_deletes | 2 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 2 +total_tuples_decompressed | 1800 +total_batches_scanned | 3 +total_batches_checked_by_bloom | 3 +total_batches_pruned_by_bloom | 1 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 1 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 1 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 1 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 1 +n_deletes | 2 + +-- Test UPDATE stats with bloom filter misses +BEGIN; +UPDATE t SET a = 42 WHERE d = 200; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 4 +total_tuples_decompressed | 3200 +total_batches_scanned | 8 +total_batches_checked_by_bloom | 8 +total_batches_pruned_by_bloom | 4 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 4 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 2 +last_op_batches_pruned_by_bloom | 2 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 2 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 2 +n_deletes | 2 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 6 +total_tuples_decompressed | 4800 +total_batches_scanned | 12 +total_batches_checked_by_bloom | 12 +total_batches_pruned_by_bloom | 6 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 6 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 3 +last_op_batches_pruned_by_bloom | 3 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 3 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 2 +n_deletes | 2 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 2 +total_tuples_decompressed | 1800 +total_batches_scanned | 4 +total_batches_checked_by_bloom | 4 +total_batches_pruned_by_bloom | 2 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 2 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 1 +last_op_batches_pruned_by_bloom | 1 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 1 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 2 +n_deletes | 2 + +DROP VIEW observ; +DROP VIEW observ_main_view; +DROP TABLE t; diff --git a/tsl/test/expected/compress_observ-16.out b/tsl/test/expected/compress_observ-16.out new file mode 100644 index 00000000000..d3ea65ba857 --- /dev/null +++ b/tsl/test/expected/compress_observ-16.out @@ -0,0 +1,1395 @@ +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license. +-- Test the integration of the compression and observability features +CREATE TABLE t(ts timestamptz, a int, b int, c int, d int, seg int); +SELECT create_hypertable('t', by_range('ts', interval '1 day')); + create_hypertable +------------------- + (1,t) + +CREATE VIEW observ AS +SELECT + -- skip the OIDs for test stability: + -- compressed_relid, + -- uncompressed_relid, + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_sum, compressed_batch_rows_sqsum, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_sum, compressed_batch_bytes_sqsum, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_sum, compressed_block_bytes_sqsum, + -- op totals + total_batches_deleted, total_batches_decompressed, + total_tuples_decompressed, total_batches_scanned, + total_batches_checked_by_bloom, total_batches_pruned_by_bloom, + total_batches_without_bloom, total_batches_bloom_false_positives, + total_batches_filtered_compressed, total_batches_filtered_decompressed, + -- last op + last_op_batches_deleted, last_op_batches_decompressed, + last_op_tuples_decompressed, last_op_batches_scanned, + last_op_batches_checked_by_bloom, last_op_batches_pruned_by_bloom, + last_op_batches_without_bloom, last_op_batches_bloom_false_positives, + last_op_batches_filtered_compressed, last_op_batches_filtered_decompressed, + -- op counters + n_selects, n_inserts, n_updates, n_deletes + -- skip the below two for test stability: + -- first_update, + -- last_update +FROM +( + SELECT * FROM _timescaledb_functions.chunk_statistics() + ORDER BY last_update ASC, compressed_relid ASC, uncompressed_relid ASC +) sub; +CREATE VIEW observ_main_view AS +SELECT + chunk, compressed_chunk, + chunk_id, hypertable_id, hypertable, + -- compression stats + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_avg, compressed_batch_rows_stddev, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_avg, compressed_batch_bytes_stddev, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_avg, compressed_block_bytes_stddev, + -- op totals + total_batches_deleted, total_batches_decompressed, + total_tuples_decompressed, total_batches_scanned, + total_batches_checked_by_bloom, total_batches_pruned_by_bloom, + total_batches_without_bloom, total_batches_bloom_false_positives, + total_batches_filtered_compressed, total_batches_filtered_decompressed, + -- last op stats + last_op_batches_deleted, last_op_batches_decompressed, + last_op_tuples_decompressed, last_op_batches_scanned, + last_op_batches_checked_by_bloom, last_op_batches_pruned_by_bloom, + last_op_batches_without_bloom, last_op_batches_bloom_false_positives, + last_op_batches_filtered_compressed, last_op_batches_filtered_decompressed, + -- op counters + n_selects, n_inserts, n_updates, n_deletes + -- these will make the test flaky, so skip them: + -- first_update, + -- last_update +FROM +( + SELECT * + FROM timescaledb_information.stat_chunk_activity + ORDER BY last_update ASC, hypertable_id ASC, chunk_id ASC +) sub; +-- Initial config: bloom(a,b) +ALTER TABLE t SET ( + timescaledb.compress, + timescaledb.compress_segmentby = 'seg', + timescaledb.compress_orderby = 'ts', + timescaledb.compress_index = 'bloom(a,b), bloom(d)' +); +INSERT INTO t +SELECT ts, (i % 10), (i % 5), (i % 3), i, 1 +FROM generate_series('2024-01-01'::timestamptz, '2024-01-03', interval '1 hour') ts, + generate_series(1, 100) i; +\pset expanded on +SELECT compress_chunk(c) FROM show_chunks('t') c; +-[ RECORD 1 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_1_chunk +-[ RECORD 2 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_2_chunk +-[ RECORD 3 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_3_chunk + +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 0 +total_tuples_decompressed | 0 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 0 +total_tuples_decompressed | 0 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 0 +total_tuples_decompressed | 0 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 + +SELECT COUNT(*) FROM t; +-[ RECORD 1 ] +count | 4900 + +SELECT count(*),min(ts),max(ts) from t where a % 19 = 9 or b % 19 = 9 or c % 19 = 2 or seg % 19 = 2; +-[ RECORD 1 ]----------------------- +count | 1960 +min | Mon Jan 01 00:00:00 2024 PST +max | Wed Jan 03 00:00:00 2024 PST + +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 8 +total_tuples_decompressed | 1600 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 8 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 12 +total_tuples_decompressed | 2400 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 12 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 4 +total_tuples_decompressed | 900 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 4 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 + +-- Test DELETE stats +BEGIN; +DELETE FROM t WHERE a % 19 = 2; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 10 +total_tuples_decompressed | 3200 +total_batches_scanned | 2 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 2 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 15 +total_tuples_decompressed | 4800 +total_batches_scanned | 3 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 3 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 5 +total_tuples_decompressed | 1800 +total_batches_scanned | 1 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 1 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 + +-- Test UPDATE stats +BEGIN; +UPDATE t SET a = 42 WHERE a % 19 = 2; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 12 +total_tuples_decompressed | 4800 +total_batches_scanned | 4 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 2 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 18 +total_tuples_decompressed | 7200 +total_batches_scanned | 6 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 3 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 6 +total_tuples_decompressed | 2700 +total_batches_scanned | 2 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 1 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 + +-- Show the information schema view as well: +SELECT * FROM observ_main_view; +-[ RECORD 1 ]-------------------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_1_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_4_chunk +chunk_id | 1 +hypertable_id | 1 +hypertable | t +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 200 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_avg | 2676 +compressed_batch_bytes_stddev | 584 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_avg | 535 +compressed_block_bytes_stddev | 273.740314897167 +total_batches_deleted | 0 +total_batches_decompressed | 12 +total_tuples_decompressed | 4800 +total_batches_scanned | 4 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 2 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 +-[ RECORD 2 ]-------------------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_2_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_5_chunk +chunk_id | 2 +hypertable_id | 1 +hypertable | t +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 282.842712474619 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_avg | 2676 +compressed_batch_bytes_stddev | 883.073420881111 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_avg | 535 +compressed_block_bytes_stddev | 309.946059823318 +total_batches_deleted | 0 +total_batches_decompressed | 18 +total_tuples_decompressed | 7200 +total_batches_scanned | 6 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 3 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 +-[ RECORD 3 ]-------------------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_3_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_6_chunk +chunk_id | 3 +hypertable_id | 1 +hypertable | t +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_avg | 900 +compressed_batch_rows_stddev | 0 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_avg | 2964 +compressed_batch_bytes_stddev | 0 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_avg | 592 +compressed_block_bytes_stddev | 259.505992223687 +total_batches_deleted | 0 +total_batches_decompressed | 6 +total_tuples_decompressed | 2700 +total_batches_scanned | 2 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 1 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 + +-- Reset the stats and check how we re-populate them as we select from the chunk again: +SELECT _timescaledb_functions.chunk_statistics_reset(); +-[ RECORD 1 ]----------+- +chunk_statistics_reset | + +SELECT count(*) FROM t where a % 19 = 9; +-[ RECORD 1 ] +count | 490 + +-- Only care about compression stats here: +SELECT + chunk, compressed_chunk, + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_avg, compressed_batch_rows_stddev, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_avg, compressed_batch_bytes_stddev, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_avg, compressed_block_bytes_stddev +FROM observ_main_view; +-[ RECORD 1 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_1_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_4_chunk +compressed_batch_count | 2 +compressed_block_count | 2 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 200 +compressed_batch_bytes_min | 0 +compressed_batch_bytes_max | 756 +compressed_batch_bytes_avg | 378 +compressed_batch_bytes_stddev | 378 +compressed_block_bytes_min | 460 +compressed_block_bytes_max | 756 +compressed_block_bytes_avg | 608 +compressed_block_bytes_stddev | 148 +-[ RECORD 2 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_2_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_5_chunk +compressed_batch_count | 3 +compressed_block_count | 3 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 282.842712474619 +compressed_batch_bytes_min | 0 +compressed_batch_bytes_max | 756 +compressed_batch_bytes_avg | 501 +compressed_batch_bytes_stddev | 354.51124414075 +compressed_block_bytes_min | 324 +compressed_block_bytes_max | 756 +compressed_block_bytes_avg | 609 +compressed_block_bytes_stddev | 201.787567065521 +-[ RECORD 3 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_3_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_6_chunk +compressed_batch_count | 0 +compressed_block_count | 1 +compressed_batch_rows_min | +compressed_batch_rows_max | +compressed_batch_rows_avg | +compressed_batch_rows_stddev | +compressed_batch_bytes_min | +compressed_batch_bytes_max | +compressed_batch_bytes_avg | +compressed_batch_bytes_stddev | +compressed_block_bytes_min | 676 +compressed_block_bytes_max | 676 +compressed_block_bytes_avg | 676 +compressed_block_bytes_stddev | 0 + +-- Select more and check how things change: +SELECT count(*),min(ts),max(ts) from t where a % 19 = 9 or b % 19 = 9 or c % 19 = 2 or seg % 19 = 2; +-[ RECORD 1 ]----------------------- +count | 1960 +min | Mon Jan 01 00:00:00 2024 PST +max | Wed Jan 03 00:00:00 2024 PST + +SELECT + chunk, compressed_chunk, + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_avg, compressed_batch_rows_stddev, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_avg, compressed_batch_bytes_stddev, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_avg, compressed_block_bytes_stddev +FROM observ_main_view; +-[ RECORD 1 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_1_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_4_chunk +compressed_batch_count | 2 +compressed_block_count | 8 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 200 +compressed_batch_bytes_min | 0 +compressed_batch_bytes_max | 756 +compressed_batch_bytes_avg | 378 +compressed_batch_bytes_stddev | 378 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 756 +compressed_block_bytes_avg | 430 +compressed_block_bytes_stddev | 168.321121669266 +-[ RECORD 2 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_2_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_5_chunk +compressed_batch_count | 3 +compressed_block_count | 12 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 282.842712474619 +compressed_batch_bytes_min | 0 +compressed_batch_bytes_max | 756 +compressed_batch_bytes_avg | 501 +compressed_batch_bytes_stddev | 354.51124414075 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 756 +compressed_block_bytes_avg | 431 +compressed_block_bytes_stddev | 201.075994478594 +-[ RECORD 3 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_3_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_6_chunk +compressed_batch_count | 0 +compressed_block_count | 4 +compressed_batch_rows_min | +compressed_batch_rows_max | +compressed_batch_rows_avg | +compressed_batch_rows_stddev | +compressed_batch_bytes_min | +compressed_batch_bytes_max | +compressed_batch_bytes_avg | +compressed_batch_bytes_stddev | +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 676 +compressed_block_bytes_avg | 482 +compressed_block_bytes_stddev | 150.983442800858 + +-- The stats should be evicted after we decompress and thus throwing away +-- the compressed chunks. +SELECT decompress_chunk(c) FROM show_chunks('t') c; +-[ RECORD 1 ]----+--------------------------------------- +decompress_chunk | _timescaledb_internal._hyper_1_1_chunk +-[ RECORD 2 ]----+--------------------------------------- +decompress_chunk | _timescaledb_internal._hyper_1_2_chunk +-[ RECORD 3 ]----+--------------------------------------- +decompress_chunk | _timescaledb_internal._hyper_1_3_chunk + +-- This should be empty after the decompression +SELECT + chunk, compressed_chunk, + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_avg, compressed_batch_rows_stddev, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_avg, compressed_batch_bytes_stddev, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_avg, compressed_block_bytes_stddev +FROM observ_main_view; + +-- Re-compress before doing the bloom tests +SELECT compress_chunk(c) FROM show_chunks('t') c; +-[ RECORD 1 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_1_chunk +-[ RECORD 2 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_2_chunk +-[ RECORD 3 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_3_chunk + +-- Test DELETE stats with bloom filter matches +BEGIN; +DELETE FROM t WHERE a = 7 AND b = 2; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 2 +total_tuples_decompressed | 1600 +total_batches_scanned | 2 +total_batches_checked_by_bloom | 2 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 2 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 2 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 3 +total_tuples_decompressed | 2400 +total_batches_scanned | 3 +total_batches_checked_by_bloom | 3 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 3 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 3 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 1 +total_tuples_decompressed | 900 +total_batches_scanned | 1 +total_batches_checked_by_bloom | 1 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 1 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 1 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 + +-- Test DELETE stats with bloom filter misses +BEGIN; +DELETE FROM t WHERE d = 200; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 2 +total_tuples_decompressed | 1600 +total_batches_scanned | 4 +total_batches_checked_by_bloom | 4 +total_batches_pruned_by_bloom | 2 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 2 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 2 +last_op_batches_pruned_by_bloom | 2 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 2 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 2 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 3 +total_tuples_decompressed | 2400 +total_batches_scanned | 6 +total_batches_checked_by_bloom | 6 +total_batches_pruned_by_bloom | 3 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 3 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 3 +last_op_batches_pruned_by_bloom | 3 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 3 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 2 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 1 +total_tuples_decompressed | 900 +total_batches_scanned | 2 +total_batches_checked_by_bloom | 2 +total_batches_pruned_by_bloom | 1 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 1 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 1 +last_op_batches_pruned_by_bloom | 1 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 1 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 2 + +-- Test UPDATE stats with bloom filter matches +BEGIN; +UPDATE t SET a = 42 WHERE a = 7 AND b = 2; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 4 +total_tuples_decompressed | 3200 +total_batches_scanned | 6 +total_batches_checked_by_bloom | 6 +total_batches_pruned_by_bloom | 2 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 2 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 2 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 2 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 1 +n_deletes | 2 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 6 +total_tuples_decompressed | 4800 +total_batches_scanned | 9 +total_batches_checked_by_bloom | 9 +total_batches_pruned_by_bloom | 3 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 3 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 3 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 3 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 1 +n_deletes | 2 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 2 +total_tuples_decompressed | 1800 +total_batches_scanned | 3 +total_batches_checked_by_bloom | 3 +total_batches_pruned_by_bloom | 1 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 1 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 1 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 1 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 1 +n_deletes | 2 + +-- Test UPDATE stats with bloom filter misses +BEGIN; +UPDATE t SET a = 42 WHERE d = 200; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 4 +total_tuples_decompressed | 3200 +total_batches_scanned | 8 +total_batches_checked_by_bloom | 8 +total_batches_pruned_by_bloom | 4 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 4 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 2 +last_op_batches_pruned_by_bloom | 2 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 2 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 2 +n_deletes | 2 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 6 +total_tuples_decompressed | 4800 +total_batches_scanned | 12 +total_batches_checked_by_bloom | 12 +total_batches_pruned_by_bloom | 6 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 6 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 3 +last_op_batches_pruned_by_bloom | 3 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 3 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 2 +n_deletes | 2 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 2 +total_tuples_decompressed | 1800 +total_batches_scanned | 4 +total_batches_checked_by_bloom | 4 +total_batches_pruned_by_bloom | 2 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 2 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 1 +last_op_batches_pruned_by_bloom | 1 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 1 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 2 +n_deletes | 2 + +DROP VIEW observ; +DROP VIEW observ_main_view; +DROP TABLE t; diff --git a/tsl/test/expected/compress_observ-17.out b/tsl/test/expected/compress_observ-17.out new file mode 100644 index 00000000000..d3ea65ba857 --- /dev/null +++ b/tsl/test/expected/compress_observ-17.out @@ -0,0 +1,1395 @@ +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license. +-- Test the integration of the compression and observability features +CREATE TABLE t(ts timestamptz, a int, b int, c int, d int, seg int); +SELECT create_hypertable('t', by_range('ts', interval '1 day')); + create_hypertable +------------------- + (1,t) + +CREATE VIEW observ AS +SELECT + -- skip the OIDs for test stability: + -- compressed_relid, + -- uncompressed_relid, + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_sum, compressed_batch_rows_sqsum, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_sum, compressed_batch_bytes_sqsum, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_sum, compressed_block_bytes_sqsum, + -- op totals + total_batches_deleted, total_batches_decompressed, + total_tuples_decompressed, total_batches_scanned, + total_batches_checked_by_bloom, total_batches_pruned_by_bloom, + total_batches_without_bloom, total_batches_bloom_false_positives, + total_batches_filtered_compressed, total_batches_filtered_decompressed, + -- last op + last_op_batches_deleted, last_op_batches_decompressed, + last_op_tuples_decompressed, last_op_batches_scanned, + last_op_batches_checked_by_bloom, last_op_batches_pruned_by_bloom, + last_op_batches_without_bloom, last_op_batches_bloom_false_positives, + last_op_batches_filtered_compressed, last_op_batches_filtered_decompressed, + -- op counters + n_selects, n_inserts, n_updates, n_deletes + -- skip the below two for test stability: + -- first_update, + -- last_update +FROM +( + SELECT * FROM _timescaledb_functions.chunk_statistics() + ORDER BY last_update ASC, compressed_relid ASC, uncompressed_relid ASC +) sub; +CREATE VIEW observ_main_view AS +SELECT + chunk, compressed_chunk, + chunk_id, hypertable_id, hypertable, + -- compression stats + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_avg, compressed_batch_rows_stddev, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_avg, compressed_batch_bytes_stddev, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_avg, compressed_block_bytes_stddev, + -- op totals + total_batches_deleted, total_batches_decompressed, + total_tuples_decompressed, total_batches_scanned, + total_batches_checked_by_bloom, total_batches_pruned_by_bloom, + total_batches_without_bloom, total_batches_bloom_false_positives, + total_batches_filtered_compressed, total_batches_filtered_decompressed, + -- last op stats + last_op_batches_deleted, last_op_batches_decompressed, + last_op_tuples_decompressed, last_op_batches_scanned, + last_op_batches_checked_by_bloom, last_op_batches_pruned_by_bloom, + last_op_batches_without_bloom, last_op_batches_bloom_false_positives, + last_op_batches_filtered_compressed, last_op_batches_filtered_decompressed, + -- op counters + n_selects, n_inserts, n_updates, n_deletes + -- these will make the test flaky, so skip them: + -- first_update, + -- last_update +FROM +( + SELECT * + FROM timescaledb_information.stat_chunk_activity + ORDER BY last_update ASC, hypertable_id ASC, chunk_id ASC +) sub; +-- Initial config: bloom(a,b) +ALTER TABLE t SET ( + timescaledb.compress, + timescaledb.compress_segmentby = 'seg', + timescaledb.compress_orderby = 'ts', + timescaledb.compress_index = 'bloom(a,b), bloom(d)' +); +INSERT INTO t +SELECT ts, (i % 10), (i % 5), (i % 3), i, 1 +FROM generate_series('2024-01-01'::timestamptz, '2024-01-03', interval '1 hour') ts, + generate_series(1, 100) i; +\pset expanded on +SELECT compress_chunk(c) FROM show_chunks('t') c; +-[ RECORD 1 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_1_chunk +-[ RECORD 2 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_2_chunk +-[ RECORD 3 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_3_chunk + +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 0 +total_tuples_decompressed | 0 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 0 +total_tuples_decompressed | 0 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 0 +total_tuples_decompressed | 0 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 + +SELECT COUNT(*) FROM t; +-[ RECORD 1 ] +count | 4900 + +SELECT count(*),min(ts),max(ts) from t where a % 19 = 9 or b % 19 = 9 or c % 19 = 2 or seg % 19 = 2; +-[ RECORD 1 ]----------------------- +count | 1960 +min | Mon Jan 01 00:00:00 2024 PST +max | Wed Jan 03 00:00:00 2024 PST + +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 8 +total_tuples_decompressed | 1600 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 8 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 12 +total_tuples_decompressed | 2400 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 12 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 4 +total_tuples_decompressed | 900 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 4 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 + +-- Test DELETE stats +BEGIN; +DELETE FROM t WHERE a % 19 = 2; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 10 +total_tuples_decompressed | 3200 +total_batches_scanned | 2 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 2 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 15 +total_tuples_decompressed | 4800 +total_batches_scanned | 3 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 3 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 5 +total_tuples_decompressed | 1800 +total_batches_scanned | 1 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 1 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 + +-- Test UPDATE stats +BEGIN; +UPDATE t SET a = 42 WHERE a % 19 = 2; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 12 +total_tuples_decompressed | 4800 +total_batches_scanned | 4 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 2 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 18 +total_tuples_decompressed | 7200 +total_batches_scanned | 6 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 3 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 6 +total_tuples_decompressed | 2700 +total_batches_scanned | 2 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 1 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 + +-- Show the information schema view as well: +SELECT * FROM observ_main_view; +-[ RECORD 1 ]-------------------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_1_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_4_chunk +chunk_id | 1 +hypertable_id | 1 +hypertable | t +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 200 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_avg | 2676 +compressed_batch_bytes_stddev | 584 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_avg | 535 +compressed_block_bytes_stddev | 273.740314897167 +total_batches_deleted | 0 +total_batches_decompressed | 12 +total_tuples_decompressed | 4800 +total_batches_scanned | 4 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 2 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 +-[ RECORD 2 ]-------------------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_2_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_5_chunk +chunk_id | 2 +hypertable_id | 1 +hypertable | t +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 282.842712474619 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_avg | 2676 +compressed_batch_bytes_stddev | 883.073420881111 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_avg | 535 +compressed_block_bytes_stddev | 309.946059823318 +total_batches_deleted | 0 +total_batches_decompressed | 18 +total_tuples_decompressed | 7200 +total_batches_scanned | 6 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 3 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 +-[ RECORD 3 ]-------------------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_3_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_6_chunk +chunk_id | 3 +hypertable_id | 1 +hypertable | t +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_avg | 900 +compressed_batch_rows_stddev | 0 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_avg | 2964 +compressed_batch_bytes_stddev | 0 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_avg | 592 +compressed_block_bytes_stddev | 259.505992223687 +total_batches_deleted | 0 +total_batches_decompressed | 6 +total_tuples_decompressed | 2700 +total_batches_scanned | 2 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 1 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 + +-- Reset the stats and check how we re-populate them as we select from the chunk again: +SELECT _timescaledb_functions.chunk_statistics_reset(); +-[ RECORD 1 ]----------+- +chunk_statistics_reset | + +SELECT count(*) FROM t where a % 19 = 9; +-[ RECORD 1 ] +count | 490 + +-- Only care about compression stats here: +SELECT + chunk, compressed_chunk, + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_avg, compressed_batch_rows_stddev, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_avg, compressed_batch_bytes_stddev, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_avg, compressed_block_bytes_stddev +FROM observ_main_view; +-[ RECORD 1 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_1_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_4_chunk +compressed_batch_count | 2 +compressed_block_count | 2 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 200 +compressed_batch_bytes_min | 0 +compressed_batch_bytes_max | 756 +compressed_batch_bytes_avg | 378 +compressed_batch_bytes_stddev | 378 +compressed_block_bytes_min | 460 +compressed_block_bytes_max | 756 +compressed_block_bytes_avg | 608 +compressed_block_bytes_stddev | 148 +-[ RECORD 2 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_2_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_5_chunk +compressed_batch_count | 3 +compressed_block_count | 3 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 282.842712474619 +compressed_batch_bytes_min | 0 +compressed_batch_bytes_max | 756 +compressed_batch_bytes_avg | 501 +compressed_batch_bytes_stddev | 354.51124414075 +compressed_block_bytes_min | 324 +compressed_block_bytes_max | 756 +compressed_block_bytes_avg | 609 +compressed_block_bytes_stddev | 201.787567065521 +-[ RECORD 3 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_3_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_6_chunk +compressed_batch_count | 0 +compressed_block_count | 1 +compressed_batch_rows_min | +compressed_batch_rows_max | +compressed_batch_rows_avg | +compressed_batch_rows_stddev | +compressed_batch_bytes_min | +compressed_batch_bytes_max | +compressed_batch_bytes_avg | +compressed_batch_bytes_stddev | +compressed_block_bytes_min | 676 +compressed_block_bytes_max | 676 +compressed_block_bytes_avg | 676 +compressed_block_bytes_stddev | 0 + +-- Select more and check how things change: +SELECT count(*),min(ts),max(ts) from t where a % 19 = 9 or b % 19 = 9 or c % 19 = 2 or seg % 19 = 2; +-[ RECORD 1 ]----------------------- +count | 1960 +min | Mon Jan 01 00:00:00 2024 PST +max | Wed Jan 03 00:00:00 2024 PST + +SELECT + chunk, compressed_chunk, + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_avg, compressed_batch_rows_stddev, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_avg, compressed_batch_bytes_stddev, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_avg, compressed_block_bytes_stddev +FROM observ_main_view; +-[ RECORD 1 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_1_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_4_chunk +compressed_batch_count | 2 +compressed_block_count | 8 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 200 +compressed_batch_bytes_min | 0 +compressed_batch_bytes_max | 756 +compressed_batch_bytes_avg | 378 +compressed_batch_bytes_stddev | 378 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 756 +compressed_block_bytes_avg | 430 +compressed_block_bytes_stddev | 168.321121669266 +-[ RECORD 2 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_2_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_5_chunk +compressed_batch_count | 3 +compressed_block_count | 12 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 282.842712474619 +compressed_batch_bytes_min | 0 +compressed_batch_bytes_max | 756 +compressed_batch_bytes_avg | 501 +compressed_batch_bytes_stddev | 354.51124414075 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 756 +compressed_block_bytes_avg | 431 +compressed_block_bytes_stddev | 201.075994478594 +-[ RECORD 3 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_3_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_6_chunk +compressed_batch_count | 0 +compressed_block_count | 4 +compressed_batch_rows_min | +compressed_batch_rows_max | +compressed_batch_rows_avg | +compressed_batch_rows_stddev | +compressed_batch_bytes_min | +compressed_batch_bytes_max | +compressed_batch_bytes_avg | +compressed_batch_bytes_stddev | +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 676 +compressed_block_bytes_avg | 482 +compressed_block_bytes_stddev | 150.983442800858 + +-- The stats should be evicted after we decompress and thus throwing away +-- the compressed chunks. +SELECT decompress_chunk(c) FROM show_chunks('t') c; +-[ RECORD 1 ]----+--------------------------------------- +decompress_chunk | _timescaledb_internal._hyper_1_1_chunk +-[ RECORD 2 ]----+--------------------------------------- +decompress_chunk | _timescaledb_internal._hyper_1_2_chunk +-[ RECORD 3 ]----+--------------------------------------- +decompress_chunk | _timescaledb_internal._hyper_1_3_chunk + +-- This should be empty after the decompression +SELECT + chunk, compressed_chunk, + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_avg, compressed_batch_rows_stddev, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_avg, compressed_batch_bytes_stddev, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_avg, compressed_block_bytes_stddev +FROM observ_main_view; + +-- Re-compress before doing the bloom tests +SELECT compress_chunk(c) FROM show_chunks('t') c; +-[ RECORD 1 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_1_chunk +-[ RECORD 2 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_2_chunk +-[ RECORD 3 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_3_chunk + +-- Test DELETE stats with bloom filter matches +BEGIN; +DELETE FROM t WHERE a = 7 AND b = 2; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 2 +total_tuples_decompressed | 1600 +total_batches_scanned | 2 +total_batches_checked_by_bloom | 2 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 2 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 2 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 3 +total_tuples_decompressed | 2400 +total_batches_scanned | 3 +total_batches_checked_by_bloom | 3 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 3 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 3 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 1 +total_tuples_decompressed | 900 +total_batches_scanned | 1 +total_batches_checked_by_bloom | 1 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 1 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 1 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 + +-- Test DELETE stats with bloom filter misses +BEGIN; +DELETE FROM t WHERE d = 200; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 2 +total_tuples_decompressed | 1600 +total_batches_scanned | 4 +total_batches_checked_by_bloom | 4 +total_batches_pruned_by_bloom | 2 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 2 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 2 +last_op_batches_pruned_by_bloom | 2 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 2 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 2 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 3 +total_tuples_decompressed | 2400 +total_batches_scanned | 6 +total_batches_checked_by_bloom | 6 +total_batches_pruned_by_bloom | 3 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 3 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 3 +last_op_batches_pruned_by_bloom | 3 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 3 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 2 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 1 +total_tuples_decompressed | 900 +total_batches_scanned | 2 +total_batches_checked_by_bloom | 2 +total_batches_pruned_by_bloom | 1 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 1 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 1 +last_op_batches_pruned_by_bloom | 1 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 1 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 2 + +-- Test UPDATE stats with bloom filter matches +BEGIN; +UPDATE t SET a = 42 WHERE a = 7 AND b = 2; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 4 +total_tuples_decompressed | 3200 +total_batches_scanned | 6 +total_batches_checked_by_bloom | 6 +total_batches_pruned_by_bloom | 2 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 2 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 2 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 2 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 1 +n_deletes | 2 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 6 +total_tuples_decompressed | 4800 +total_batches_scanned | 9 +total_batches_checked_by_bloom | 9 +total_batches_pruned_by_bloom | 3 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 3 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 3 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 3 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 1 +n_deletes | 2 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 2 +total_tuples_decompressed | 1800 +total_batches_scanned | 3 +total_batches_checked_by_bloom | 3 +total_batches_pruned_by_bloom | 1 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 1 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 1 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 1 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 1 +n_deletes | 2 + +-- Test UPDATE stats with bloom filter misses +BEGIN; +UPDATE t SET a = 42 WHERE d = 200; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 2092 +compressed_batch_bytes_max | 3260 +compressed_batch_bytes_sum | 5352 +compressed_batch_bytes_sqsum | 15004064 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 1156 +compressed_block_bytes_sum | 5352 +compressed_block_bytes_sqsum | 3613728 +total_batches_deleted | 0 +total_batches_decompressed | 4 +total_tuples_decompressed | 3200 +total_batches_scanned | 8 +total_batches_checked_by_bloom | 8 +total_batches_pruned_by_bloom | 4 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 4 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 2 +last_op_batches_pruned_by_bloom | 2 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 2 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 2 +n_deletes | 2 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1428 +compressed_batch_bytes_max | 3340 +compressed_batch_bytes_sum | 8028 +compressed_batch_bytes_sqsum | 23822384 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 1220 +compressed_block_bytes_sum | 8028 +compressed_block_bytes_sqsum | 5737584 +total_batches_deleted | 0 +total_batches_decompressed | 6 +total_tuples_decompressed | 4800 +total_batches_scanned | 12 +total_batches_checked_by_bloom | 12 +total_batches_pruned_by_bloom | 6 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 6 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 3 +last_op_batches_pruned_by_bloom | 3 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 3 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 2 +n_deletes | 2 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2964 +compressed_batch_bytes_max | 2964 +compressed_batch_bytes_sum | 2964 +compressed_batch_bytes_sqsum | 8785296 +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 1036 +compressed_block_bytes_sum | 2964 +compressed_block_bytes_sqsum | 2093776 +total_batches_deleted | 0 +total_batches_decompressed | 2 +total_tuples_decompressed | 1800 +total_batches_scanned | 4 +total_batches_checked_by_bloom | 4 +total_batches_pruned_by_bloom | 2 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 2 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 1 +last_op_batches_pruned_by_bloom | 1 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 1 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 2 +n_deletes | 2 + +DROP VIEW observ; +DROP VIEW observ_main_view; +DROP TABLE t; diff --git a/tsl/test/expected/compress_observ-18.out b/tsl/test/expected/compress_observ-18.out new file mode 100644 index 00000000000..0adc8733b16 --- /dev/null +++ b/tsl/test/expected/compress_observ-18.out @@ -0,0 +1,1395 @@ +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license. +-- Test the integration of the compression and observability features +CREATE TABLE t(ts timestamptz, a int, b int, c int, d int, seg int); +SELECT create_hypertable('t', by_range('ts', interval '1 day')); + create_hypertable +------------------- + (1,t) + +CREATE VIEW observ AS +SELECT + -- skip the OIDs for test stability: + -- compressed_relid, + -- uncompressed_relid, + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_sum, compressed_batch_rows_sqsum, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_sum, compressed_batch_bytes_sqsum, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_sum, compressed_block_bytes_sqsum, + -- op totals + total_batches_deleted, total_batches_decompressed, + total_tuples_decompressed, total_batches_scanned, + total_batches_checked_by_bloom, total_batches_pruned_by_bloom, + total_batches_without_bloom, total_batches_bloom_false_positives, + total_batches_filtered_compressed, total_batches_filtered_decompressed, + -- last op + last_op_batches_deleted, last_op_batches_decompressed, + last_op_tuples_decompressed, last_op_batches_scanned, + last_op_batches_checked_by_bloom, last_op_batches_pruned_by_bloom, + last_op_batches_without_bloom, last_op_batches_bloom_false_positives, + last_op_batches_filtered_compressed, last_op_batches_filtered_decompressed, + -- op counters + n_selects, n_inserts, n_updates, n_deletes + -- skip the below two for test stability: + -- first_update, + -- last_update +FROM +( + SELECT * FROM _timescaledb_functions.chunk_statistics() + ORDER BY last_update ASC, compressed_relid ASC, uncompressed_relid ASC +) sub; +CREATE VIEW observ_main_view AS +SELECT + chunk, compressed_chunk, + chunk_id, hypertable_id, hypertable, + -- compression stats + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_avg, compressed_batch_rows_stddev, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_avg, compressed_batch_bytes_stddev, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_avg, compressed_block_bytes_stddev, + -- op totals + total_batches_deleted, total_batches_decompressed, + total_tuples_decompressed, total_batches_scanned, + total_batches_checked_by_bloom, total_batches_pruned_by_bloom, + total_batches_without_bloom, total_batches_bloom_false_positives, + total_batches_filtered_compressed, total_batches_filtered_decompressed, + -- last op stats + last_op_batches_deleted, last_op_batches_decompressed, + last_op_tuples_decompressed, last_op_batches_scanned, + last_op_batches_checked_by_bloom, last_op_batches_pruned_by_bloom, + last_op_batches_without_bloom, last_op_batches_bloom_false_positives, + last_op_batches_filtered_compressed, last_op_batches_filtered_decompressed, + -- op counters + n_selects, n_inserts, n_updates, n_deletes + -- these will make the test flaky, so skip them: + -- first_update, + -- last_update +FROM +( + SELECT * + FROM timescaledb_information.stat_chunk_activity + ORDER BY last_update ASC, hypertable_id ASC, chunk_id ASC +) sub; +-- Initial config: bloom(a,b) +ALTER TABLE t SET ( + timescaledb.compress, + timescaledb.compress_segmentby = 'seg', + timescaledb.compress_orderby = 'ts', + timescaledb.compress_index = 'bloom(a,b), bloom(d)' +); +INSERT INTO t +SELECT ts, (i % 10), (i % 5), (i % 3), i, 1 +FROM generate_series('2024-01-01'::timestamptz, '2024-01-03', interval '1 hour') ts, + generate_series(1, 100) i; +\pset expanded on +SELECT compress_chunk(c) FROM show_chunks('t') c; +-[ RECORD 1 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_1_chunk +-[ RECORD 2 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_2_chunk +-[ RECORD 3 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_3_chunk + +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 1412 +compressed_batch_bytes_max | 2236 +compressed_batch_bytes_sum | 3648 +compressed_batch_bytes_sqsum | 6993440 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 748 +compressed_block_bytes_sum | 3648 +compressed_block_bytes_sqsum | 1657248 +total_batches_deleted | 0 +total_batches_decompressed | 0 +total_tuples_decompressed | 0 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1004 +compressed_batch_bytes_max | 2236 +compressed_batch_bytes_sum | 5476 +compressed_batch_bytes_sqsum | 11007408 +compressed_block_bytes_min | 100 +compressed_block_bytes_max | 748 +compressed_block_bytes_sum | 5476 +compressed_block_bytes_sqsum | 2613488 +total_batches_deleted | 0 +total_batches_decompressed | 0 +total_tuples_decompressed | 0 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2028 +compressed_batch_bytes_max | 2028 +compressed_batch_bytes_sum | 2028 +compressed_batch_bytes_sqsum | 4112784 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 668 +compressed_block_bytes_sum | 2028 +compressed_block_bytes_sqsum | 972240 +total_batches_deleted | 0 +total_batches_decompressed | 0 +total_tuples_decompressed | 0 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 + +SELECT COUNT(*) FROM t; +-[ RECORD 1 ] +count | 4900 + +SELECT count(*),min(ts),max(ts) from t where a % 19 = 9 or b % 19 = 9 or c % 19 = 2 or seg % 19 = 2; +-[ RECORD 1 ]----------------------- +count | 1960 +min | Mon Jan 01 00:00:00 2024 PST +max | Wed Jan 03 00:00:00 2024 PST + +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 1412 +compressed_batch_bytes_max | 2236 +compressed_batch_bytes_sum | 3648 +compressed_batch_bytes_sqsum | 6993440 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 748 +compressed_block_bytes_sum | 3648 +compressed_block_bytes_sqsum | 1657248 +total_batches_deleted | 0 +total_batches_decompressed | 8 +total_tuples_decompressed | 1600 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 8 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1004 +compressed_batch_bytes_max | 2236 +compressed_batch_bytes_sum | 5476 +compressed_batch_bytes_sqsum | 11007408 +compressed_block_bytes_min | 100 +compressed_block_bytes_max | 748 +compressed_block_bytes_sum | 5476 +compressed_block_bytes_sqsum | 2613488 +total_batches_deleted | 0 +total_batches_decompressed | 12 +total_tuples_decompressed | 2400 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 12 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2028 +compressed_batch_bytes_max | 2028 +compressed_batch_bytes_sum | 2028 +compressed_batch_bytes_sqsum | 4112784 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 668 +compressed_block_bytes_sum | 2028 +compressed_block_bytes_sqsum | 972240 +total_batches_deleted | 0 +total_batches_decompressed | 4 +total_tuples_decompressed | 900 +total_batches_scanned | 0 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 4 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 0 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 0 + +-- Test DELETE stats +BEGIN; +DELETE FROM t WHERE a % 19 = 2; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 1412 +compressed_batch_bytes_max | 2236 +compressed_batch_bytes_sum | 3648 +compressed_batch_bytes_sqsum | 6993440 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 748 +compressed_block_bytes_sum | 3648 +compressed_block_bytes_sqsum | 1657248 +total_batches_deleted | 0 +total_batches_decompressed | 10 +total_tuples_decompressed | 3200 +total_batches_scanned | 2 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 2 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1004 +compressed_batch_bytes_max | 2236 +compressed_batch_bytes_sum | 5476 +compressed_batch_bytes_sqsum | 11007408 +compressed_block_bytes_min | 100 +compressed_block_bytes_max | 748 +compressed_block_bytes_sum | 5476 +compressed_block_bytes_sqsum | 2613488 +total_batches_deleted | 0 +total_batches_decompressed | 15 +total_tuples_decompressed | 4800 +total_batches_scanned | 3 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 3 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2028 +compressed_batch_bytes_max | 2028 +compressed_batch_bytes_sum | 2028 +compressed_batch_bytes_sqsum | 4112784 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 668 +compressed_block_bytes_sum | 2028 +compressed_block_bytes_sqsum | 972240 +total_batches_deleted | 0 +total_batches_decompressed | 5 +total_tuples_decompressed | 1800 +total_batches_scanned | 1 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 1 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 + +-- Test UPDATE stats +BEGIN; +UPDATE t SET a = 42 WHERE a % 19 = 2; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 1412 +compressed_batch_bytes_max | 2236 +compressed_batch_bytes_sum | 3648 +compressed_batch_bytes_sqsum | 6993440 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 748 +compressed_block_bytes_sum | 3648 +compressed_block_bytes_sqsum | 1657248 +total_batches_deleted | 0 +total_batches_decompressed | 12 +total_tuples_decompressed | 4800 +total_batches_scanned | 4 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 2 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1004 +compressed_batch_bytes_max | 2236 +compressed_batch_bytes_sum | 5476 +compressed_batch_bytes_sqsum | 11007408 +compressed_block_bytes_min | 100 +compressed_block_bytes_max | 748 +compressed_block_bytes_sum | 5476 +compressed_block_bytes_sqsum | 2613488 +total_batches_deleted | 0 +total_batches_decompressed | 18 +total_tuples_decompressed | 7200 +total_batches_scanned | 6 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 3 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2028 +compressed_batch_bytes_max | 2028 +compressed_batch_bytes_sum | 2028 +compressed_batch_bytes_sqsum | 4112784 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 668 +compressed_block_bytes_sum | 2028 +compressed_block_bytes_sqsum | 972240 +total_batches_deleted | 0 +total_batches_decompressed | 6 +total_tuples_decompressed | 2700 +total_batches_scanned | 2 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 1 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 + +-- Show the information schema view as well: +SELECT * FROM observ_main_view; +-[ RECORD 1 ]-------------------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_1_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_4_chunk +chunk_id | 1 +hypertable_id | 1 +hypertable | t +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 200 +compressed_batch_bytes_min | 1412 +compressed_batch_bytes_max | 2236 +compressed_batch_bytes_avg | 1824 +compressed_batch_bytes_stddev | 412 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 748 +compressed_block_bytes_avg | 364 +compressed_block_bytes_stddev | 180.681377014899 +total_batches_deleted | 0 +total_batches_decompressed | 12 +total_tuples_decompressed | 4800 +total_batches_scanned | 4 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 2 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 +-[ RECORD 2 ]-------------------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_2_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_5_chunk +chunk_id | 2 +hypertable_id | 1 +hypertable | t +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 282.842712474619 +compressed_batch_bytes_min | 1004 +compressed_batch_bytes_max | 2236 +compressed_batch_bytes_avg | 1825 +compressed_batch_bytes_stddev | 580.770369614551 +compressed_block_bytes_min | 100 +compressed_block_bytes_max | 748 +compressed_block_bytes_avg | 365 +compressed_block_bytes_stddev | 202.38295931778 +total_batches_deleted | 0 +total_batches_decompressed | 18 +total_tuples_decompressed | 7200 +total_batches_scanned | 6 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 3 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 +-[ RECORD 3 ]-------------------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_3_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_6_chunk +chunk_id | 3 +hypertable_id | 1 +hypertable | t +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_avg | 900 +compressed_batch_rows_stddev | 0 +compressed_batch_bytes_min | 2028 +compressed_batch_bytes_max | 2028 +compressed_batch_bytes_avg | 2028 +compressed_batch_bytes_stddev | 0 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 668 +compressed_block_bytes_avg | 405 +compressed_block_bytes_stddev | 173.022079515881 +total_batches_deleted | 0 +total_batches_decompressed | 6 +total_tuples_decompressed | 2700 +total_batches_scanned | 2 +total_batches_checked_by_bloom | 0 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 1 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 0 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 1 +n_inserts | 0 +n_updates | 1 +n_deletes | 1 + +-- Reset the stats and check how we re-populate them as we select from the chunk again: +SELECT _timescaledb_functions.chunk_statistics_reset(); +-[ RECORD 1 ]----------+- +chunk_statistics_reset | + +SELECT count(*) FROM t where a % 19 = 9; +-[ RECORD 1 ] +count | 490 + +-- Only care about compression stats here: +SELECT + chunk, compressed_chunk, + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_avg, compressed_batch_rows_stddev, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_avg, compressed_batch_bytes_stddev, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_avg, compressed_block_bytes_stddev +FROM observ_main_view; +-[ RECORD 1 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_1_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_4_chunk +compressed_batch_count | 2 +compressed_block_count | 2 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 200 +compressed_batch_bytes_min | 0 +compressed_batch_bytes_max | 748 +compressed_batch_bytes_avg | 374 +compressed_batch_bytes_stddev | 374 +compressed_block_bytes_min | 460 +compressed_block_bytes_max | 748 +compressed_block_bytes_avg | 604 +compressed_block_bytes_stddev | 144 +-[ RECORD 2 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_2_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_5_chunk +compressed_batch_count | 3 +compressed_block_count | 3 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 282.842712474619 +compressed_batch_bytes_min | 0 +compressed_batch_bytes_max | 748 +compressed_batch_bytes_avg | 498 +compressed_batch_bytes_stddev | 352.610581551692 +compressed_block_bytes_min | 324 +compressed_block_bytes_max | 748 +compressed_block_bytes_avg | 606 +compressed_block_bytes_stddev | 199.875516815398 +-[ RECORD 3 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_3_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_6_chunk +compressed_batch_count | 0 +compressed_block_count | 1 +compressed_batch_rows_min | +compressed_batch_rows_max | +compressed_batch_rows_avg | +compressed_batch_rows_stddev | +compressed_batch_bytes_min | +compressed_batch_bytes_max | +compressed_batch_bytes_avg | +compressed_batch_bytes_stddev | +compressed_block_bytes_min | 668 +compressed_block_bytes_max | 668 +compressed_block_bytes_avg | 668 +compressed_block_bytes_stddev | 0 + +-- Select more and check how things change: +SELECT count(*),min(ts),max(ts) from t where a % 19 = 9 or b % 19 = 9 or c % 19 = 2 or seg % 19 = 2; +-[ RECORD 1 ]----------------------- +count | 1960 +min | Mon Jan 01 00:00:00 2024 PST +max | Wed Jan 03 00:00:00 2024 PST + +SELECT + chunk, compressed_chunk, + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_avg, compressed_batch_rows_stddev, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_avg, compressed_batch_bytes_stddev, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_avg, compressed_block_bytes_stddev +FROM observ_main_view; +-[ RECORD 1 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_1_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_4_chunk +compressed_batch_count | 2 +compressed_block_count | 8 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 200 +compressed_batch_bytes_min | 0 +compressed_batch_bytes_max | 748 +compressed_batch_bytes_avg | 374 +compressed_batch_bytes_stddev | 374 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 748 +compressed_block_bytes_avg | 414 +compressed_block_bytes_stddev | 168.463645929916 +-[ RECORD 2 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_2_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_5_chunk +compressed_batch_count | 3 +compressed_block_count | 12 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_avg | 800 +compressed_batch_rows_stddev | 282.842712474619 +compressed_batch_bytes_min | 0 +compressed_batch_bytes_max | 748 +compressed_batch_bytes_avg | 498 +compressed_batch_bytes_stddev | 352.610581551692 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 748 +compressed_block_bytes_avg | 414 +compressed_block_bytes_stddev | 196.533288104925 +-[ RECORD 3 ]-----------------+----------------------------------------------- +chunk | _timescaledb_internal._hyper_1_3_chunk +compressed_chunk | _timescaledb_internal.compress_hyper_2_6_chunk +compressed_batch_count | 0 +compressed_block_count | 4 +compressed_batch_rows_min | +compressed_batch_rows_max | +compressed_batch_rows_avg | +compressed_batch_rows_stddev | +compressed_batch_bytes_min | +compressed_batch_bytes_max | +compressed_batch_bytes_avg | +compressed_batch_bytes_stddev | +compressed_block_bytes_min | 260 +compressed_block_bytes_max | 668 +compressed_block_bytes_avg | 460 +compressed_block_bytes_stddev | 150.412765415705 + +-- The stats should be evicted after we decompress and thus throwing away +-- the compressed chunks. +SELECT decompress_chunk(c) FROM show_chunks('t') c; +-[ RECORD 1 ]----+--------------------------------------- +decompress_chunk | _timescaledb_internal._hyper_1_1_chunk +-[ RECORD 2 ]----+--------------------------------------- +decompress_chunk | _timescaledb_internal._hyper_1_2_chunk +-[ RECORD 3 ]----+--------------------------------------- +decompress_chunk | _timescaledb_internal._hyper_1_3_chunk + +-- This should be empty after the decompression +SELECT + chunk, compressed_chunk, + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_avg, compressed_batch_rows_stddev, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_avg, compressed_batch_bytes_stddev, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_avg, compressed_block_bytes_stddev +FROM observ_main_view; + +-- Re-compress before doing the bloom tests +SELECT compress_chunk(c) FROM show_chunks('t') c; +-[ RECORD 1 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_1_chunk +-[ RECORD 2 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_2_chunk +-[ RECORD 3 ]--+--------------------------------------- +compress_chunk | _timescaledb_internal._hyper_1_3_chunk + +-- Test DELETE stats with bloom filter matches +BEGIN; +DELETE FROM t WHERE a = 7 AND b = 2; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 1412 +compressed_batch_bytes_max | 2236 +compressed_batch_bytes_sum | 3648 +compressed_batch_bytes_sqsum | 6993440 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 748 +compressed_block_bytes_sum | 3648 +compressed_block_bytes_sqsum | 1657248 +total_batches_deleted | 0 +total_batches_decompressed | 2 +total_tuples_decompressed | 1600 +total_batches_scanned | 2 +total_batches_checked_by_bloom | 2 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 2 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 2 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1004 +compressed_batch_bytes_max | 2236 +compressed_batch_bytes_sum | 5476 +compressed_batch_bytes_sqsum | 11007408 +compressed_block_bytes_min | 100 +compressed_block_bytes_max | 748 +compressed_block_bytes_sum | 5476 +compressed_block_bytes_sqsum | 2613488 +total_batches_deleted | 0 +total_batches_decompressed | 3 +total_tuples_decompressed | 2400 +total_batches_scanned | 3 +total_batches_checked_by_bloom | 3 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 3 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 3 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2028 +compressed_batch_bytes_max | 2028 +compressed_batch_bytes_sum | 2028 +compressed_batch_bytes_sqsum | 4112784 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 668 +compressed_block_bytes_sum | 2028 +compressed_block_bytes_sqsum | 972240 +total_batches_deleted | 0 +total_batches_decompressed | 1 +total_tuples_decompressed | 900 +total_batches_scanned | 1 +total_batches_checked_by_bloom | 1 +total_batches_pruned_by_bloom | 0 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 0 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 1 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 1 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 1 + +-- Test DELETE stats with bloom filter misses +BEGIN; +DELETE FROM t WHERE d = 200; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 1412 +compressed_batch_bytes_max | 2236 +compressed_batch_bytes_sum | 3648 +compressed_batch_bytes_sqsum | 6993440 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 748 +compressed_block_bytes_sum | 3648 +compressed_block_bytes_sqsum | 1657248 +total_batches_deleted | 0 +total_batches_decompressed | 2 +total_tuples_decompressed | 1600 +total_batches_scanned | 4 +total_batches_checked_by_bloom | 4 +total_batches_pruned_by_bloom | 2 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 2 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 2 +last_op_batches_pruned_by_bloom | 2 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 2 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 2 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1004 +compressed_batch_bytes_max | 2236 +compressed_batch_bytes_sum | 5476 +compressed_batch_bytes_sqsum | 11007408 +compressed_block_bytes_min | 100 +compressed_block_bytes_max | 748 +compressed_block_bytes_sum | 5476 +compressed_block_bytes_sqsum | 2613488 +total_batches_deleted | 0 +total_batches_decompressed | 3 +total_tuples_decompressed | 2400 +total_batches_scanned | 6 +total_batches_checked_by_bloom | 6 +total_batches_pruned_by_bloom | 3 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 3 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 3 +last_op_batches_pruned_by_bloom | 3 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 3 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 2 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2028 +compressed_batch_bytes_max | 2028 +compressed_batch_bytes_sum | 2028 +compressed_batch_bytes_sqsum | 4112784 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 668 +compressed_block_bytes_sum | 2028 +compressed_block_bytes_sqsum | 972240 +total_batches_deleted | 0 +total_batches_decompressed | 1 +total_tuples_decompressed | 900 +total_batches_scanned | 2 +total_batches_checked_by_bloom | 2 +total_batches_pruned_by_bloom | 1 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 1 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 1 +last_op_batches_pruned_by_bloom | 1 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 1 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 0 +n_deletes | 2 + +-- Test UPDATE stats with bloom filter matches +BEGIN; +UPDATE t SET a = 42 WHERE a = 7 AND b = 2; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 1412 +compressed_batch_bytes_max | 2236 +compressed_batch_bytes_sum | 3648 +compressed_batch_bytes_sqsum | 6993440 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 748 +compressed_block_bytes_sum | 3648 +compressed_block_bytes_sqsum | 1657248 +total_batches_deleted | 0 +total_batches_decompressed | 4 +total_tuples_decompressed | 3200 +total_batches_scanned | 6 +total_batches_checked_by_bloom | 6 +total_batches_pruned_by_bloom | 2 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 2 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 2 +last_op_tuples_decompressed | 1600 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 2 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 1 +n_deletes | 2 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1004 +compressed_batch_bytes_max | 2236 +compressed_batch_bytes_sum | 5476 +compressed_batch_bytes_sqsum | 11007408 +compressed_block_bytes_min | 100 +compressed_block_bytes_max | 748 +compressed_block_bytes_sum | 5476 +compressed_block_bytes_sqsum | 2613488 +total_batches_deleted | 0 +total_batches_decompressed | 6 +total_tuples_decompressed | 4800 +total_batches_scanned | 9 +total_batches_checked_by_bloom | 9 +total_batches_pruned_by_bloom | 3 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 3 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 3 +last_op_tuples_decompressed | 2400 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 3 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 1 +n_deletes | 2 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2028 +compressed_batch_bytes_max | 2028 +compressed_batch_bytes_sum | 2028 +compressed_batch_bytes_sqsum | 4112784 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 668 +compressed_block_bytes_sum | 2028 +compressed_block_bytes_sqsum | 972240 +total_batches_deleted | 0 +total_batches_decompressed | 2 +total_tuples_decompressed | 1800 +total_batches_scanned | 3 +total_batches_checked_by_bloom | 3 +total_batches_pruned_by_bloom | 1 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 1 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 1 +last_op_tuples_decompressed | 900 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 1 +last_op_batches_pruned_by_bloom | 0 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 0 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 1 +n_deletes | 2 + +-- Test UPDATE stats with bloom filter misses +BEGIN; +UPDATE t SET a = 42 WHERE d = 200; +ROLLBACK; +SELECT * FROM observ; -- see view definition above +-[ RECORD 1 ]-------------------------+--------- +compressed_batch_count | 2 +compressed_block_count | 10 +compressed_batch_rows_min | 600 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 1600 +compressed_batch_rows_sqsum | 1360000 +compressed_batch_bytes_min | 1412 +compressed_batch_bytes_max | 2236 +compressed_batch_bytes_sum | 3648 +compressed_batch_bytes_sqsum | 6993440 +compressed_block_bytes_min | 132 +compressed_block_bytes_max | 748 +compressed_block_bytes_sum | 3648 +compressed_block_bytes_sqsum | 1657248 +total_batches_deleted | 0 +total_batches_decompressed | 4 +total_tuples_decompressed | 3200 +total_batches_scanned | 8 +total_batches_checked_by_bloom | 8 +total_batches_pruned_by_bloom | 4 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 4 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 2 +last_op_batches_checked_by_bloom | 2 +last_op_batches_pruned_by_bloom | 2 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 2 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 2 +n_deletes | 2 +-[ RECORD 2 ]-------------------------+--------- +compressed_batch_count | 3 +compressed_block_count | 15 +compressed_batch_rows_min | 400 +compressed_batch_rows_max | 1000 +compressed_batch_rows_sum | 2400 +compressed_batch_rows_sqsum | 2160000 +compressed_batch_bytes_min | 1004 +compressed_batch_bytes_max | 2236 +compressed_batch_bytes_sum | 5476 +compressed_batch_bytes_sqsum | 11007408 +compressed_block_bytes_min | 100 +compressed_block_bytes_max | 748 +compressed_block_bytes_sum | 5476 +compressed_block_bytes_sqsum | 2613488 +total_batches_deleted | 0 +total_batches_decompressed | 6 +total_tuples_decompressed | 4800 +total_batches_scanned | 12 +total_batches_checked_by_bloom | 12 +total_batches_pruned_by_bloom | 6 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 6 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 3 +last_op_batches_checked_by_bloom | 3 +last_op_batches_pruned_by_bloom | 3 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 3 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 2 +n_deletes | 2 +-[ RECORD 3 ]-------------------------+--------- +compressed_batch_count | 1 +compressed_block_count | 5 +compressed_batch_rows_min | 900 +compressed_batch_rows_max | 900 +compressed_batch_rows_sum | 900 +compressed_batch_rows_sqsum | 810000 +compressed_batch_bytes_min | 2028 +compressed_batch_bytes_max | 2028 +compressed_batch_bytes_sum | 2028 +compressed_batch_bytes_sqsum | 4112784 +compressed_block_bytes_min | 188 +compressed_block_bytes_max | 668 +compressed_block_bytes_sum | 2028 +compressed_block_bytes_sqsum | 972240 +total_batches_deleted | 0 +total_batches_decompressed | 2 +total_tuples_decompressed | 1800 +total_batches_scanned | 4 +total_batches_checked_by_bloom | 4 +total_batches_pruned_by_bloom | 2 +total_batches_without_bloom | 0 +total_batches_bloom_false_positives | 0 +total_batches_filtered_compressed | 2 +total_batches_filtered_decompressed | 0 +last_op_batches_deleted | 0 +last_op_batches_decompressed | 0 +last_op_tuples_decompressed | 0 +last_op_batches_scanned | 1 +last_op_batches_checked_by_bloom | 1 +last_op_batches_pruned_by_bloom | 1 +last_op_batches_without_bloom | 0 +last_op_batches_bloom_false_positives | 0 +last_op_batches_filtered_compressed | 1 +last_op_batches_filtered_decompressed | 0 +n_selects | 0 +n_inserts | 0 +n_updates | 2 +n_deletes | 2 + +DROP VIEW observ; +DROP VIEW observ_main_view; +DROP TABLE t; diff --git a/tsl/test/shared/expected/extension.out b/tsl/test/shared/expected/extension.out index 70f04d68b7d..e61952c12ba 100644 --- a/tsl/test/shared/expected/extension.out +++ b/tsl/test/shared/expected/extension.out @@ -40,6 +40,8 @@ ORDER BY pronamespace::regnamespace::text COLLATE "C", p.oid::regprocedure::text _timescaledb_functions.chunk_constraint_add_table_constraint(_timescaledb_catalog.chunk_constraint) _timescaledb_functions.chunk_id_from_relid(oid) _timescaledb_functions.chunk_rewrite_cleanup() + _timescaledb_functions.chunk_statistics(regclass,regclass,timestamp with time zone) + _timescaledb_functions.chunk_statistics_reset() _timescaledb_functions.chunk_status(regclass) _timescaledb_functions.chunk_status_text(integer) _timescaledb_functions.chunk_status_text(regclass) diff --git a/tsl/test/sql/.gitignore b/tsl/test/sql/.gitignore index 9ff2cc72f47..7af56556318 100644 --- a/tsl/test/sql/.gitignore +++ b/tsl/test/sql/.gitignore @@ -11,3 +11,4 @@ /plan_skip_scan_dagg-*.sql /columnar_index_scan-*.sql /vector_agg_planning-*.sql +/compress_observ-*.sql diff --git a/tsl/test/sql/CMakeLists.txt b/tsl/test/sql/CMakeLists.txt index 061ded82e04..c247d1289a0 100644 --- a/tsl/test/sql/CMakeLists.txt +++ b/tsl/test/sql/CMakeLists.txt @@ -98,6 +98,7 @@ set(TEST_TEMPLATES cagg_permissions.sql.in columnar_index_scan.sql.in compress_bloom_sparse.sql.in + compress_observ.sql.in compression_permissions.sql.in foreign_keys_test.sql.in modify_exclusion.sql.in diff --git a/tsl/test/sql/compress_observ.sql.in b/tsl/test/sql/compress_observ.sql.in new file mode 100644 index 00000000000..aad19b58956 --- /dev/null +++ b/tsl/test/sql/compress_observ.sql.in @@ -0,0 +1,202 @@ +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license. + +-- Test the integration of the compression and observability features + +CREATE TABLE t(ts timestamptz, a int, b int, c int, d int, seg int); +SELECT create_hypertable('t', by_range('ts', interval '1 day')); + +CREATE VIEW observ AS +SELECT + -- skip the OIDs for test stability: + -- compressed_relid, + -- uncompressed_relid, + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_sum, compressed_batch_rows_sqsum, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_sum, compressed_batch_bytes_sqsum, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_sum, compressed_block_bytes_sqsum, + -- op totals + total_batches_deleted, total_batches_decompressed, + total_tuples_decompressed, total_batches_scanned, + total_batches_checked_by_bloom, total_batches_pruned_by_bloom, + total_batches_without_bloom, total_batches_bloom_false_positives, + total_batches_filtered_compressed, total_batches_filtered_decompressed, + -- last op + last_op_batches_deleted, last_op_batches_decompressed, + last_op_tuples_decompressed, last_op_batches_scanned, + last_op_batches_checked_by_bloom, last_op_batches_pruned_by_bloom, + last_op_batches_without_bloom, last_op_batches_bloom_false_positives, + last_op_batches_filtered_compressed, last_op_batches_filtered_decompressed, + -- op counters + n_selects, n_inserts, n_updates, n_deletes + -- skip the below two for test stability: + -- first_update, + -- last_update +FROM +( + SELECT * FROM _timescaledb_functions.chunk_statistics() + ORDER BY last_update ASC, compressed_relid ASC, uncompressed_relid ASC +) sub; + +CREATE VIEW observ_main_view AS +SELECT + chunk, compressed_chunk, + chunk_id, hypertable_id, hypertable, + -- compression stats + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_avg, compressed_batch_rows_stddev, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_avg, compressed_batch_bytes_stddev, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_avg, compressed_block_bytes_stddev, + -- op totals + total_batches_deleted, total_batches_decompressed, + total_tuples_decompressed, total_batches_scanned, + total_batches_checked_by_bloom, total_batches_pruned_by_bloom, + total_batches_without_bloom, total_batches_bloom_false_positives, + total_batches_filtered_compressed, total_batches_filtered_decompressed, + -- last op stats + last_op_batches_deleted, last_op_batches_decompressed, + last_op_tuples_decompressed, last_op_batches_scanned, + last_op_batches_checked_by_bloom, last_op_batches_pruned_by_bloom, + last_op_batches_without_bloom, last_op_batches_bloom_false_positives, + last_op_batches_filtered_compressed, last_op_batches_filtered_decompressed, + -- op counters + n_selects, n_inserts, n_updates, n_deletes + -- these will make the test flaky, so skip them: + -- first_update, + -- last_update +FROM +( + SELECT * + FROM timescaledb_information.stat_chunk_activity + ORDER BY last_update ASC, hypertable_id ASC, chunk_id ASC +) sub; + +-- Initial config: bloom(a,b) +ALTER TABLE t SET ( + timescaledb.compress, + timescaledb.compress_segmentby = 'seg', + timescaledb.compress_orderby = 'ts', + timescaledb.compress_index = 'bloom(a,b), bloom(d)' +); + +INSERT INTO t +SELECT ts, (i % 10), (i % 5), (i % 3), i, 1 +FROM generate_series('2024-01-01'::timestamptz, '2024-01-03', interval '1 hour') ts, + generate_series(1, 100) i; + +\pset expanded on + +SELECT compress_chunk(c) FROM show_chunks('t') c; + +SELECT * FROM observ; -- see view definition above + +SELECT COUNT(*) FROM t; +SELECT count(*),min(ts),max(ts) from t where a % 19 = 9 or b % 19 = 9 or c % 19 = 2 or seg % 19 = 2; + +SELECT * FROM observ; -- see view definition above + +-- Test DELETE stats +BEGIN; +DELETE FROM t WHERE a % 19 = 2; +ROLLBACK; + +SELECT * FROM observ; -- see view definition above + +-- Test UPDATE stats +BEGIN; +UPDATE t SET a = 42 WHERE a % 19 = 2; +ROLLBACK; + +SELECT * FROM observ; -- see view definition above + +-- Show the information schema view as well: +SELECT * FROM observ_main_view; + +-- Reset the stats and check how we re-populate them as we select from the chunk again: +SELECT _timescaledb_functions.chunk_statistics_reset(); +SELECT count(*) FROM t where a % 19 = 9; + +-- Only care about compression stats here: +SELECT + chunk, compressed_chunk, + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_avg, compressed_batch_rows_stddev, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_avg, compressed_batch_bytes_stddev, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_avg, compressed_block_bytes_stddev +FROM observ_main_view; + +-- Select more and check how things change: +SELECT count(*),min(ts),max(ts) from t where a % 19 = 9 or b % 19 = 9 or c % 19 = 2 or seg % 19 = 2; + +SELECT + chunk, compressed_chunk, + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_avg, compressed_batch_rows_stddev, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_avg, compressed_batch_bytes_stddev, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_avg, compressed_block_bytes_stddev +FROM observ_main_view; + +-- The stats should be evicted after we decompress and thus throwing away +-- the compressed chunks. +SELECT decompress_chunk(c) FROM show_chunks('t') c; + +-- This should be empty after the decompression +SELECT + chunk, compressed_chunk, + compressed_batch_count, compressed_block_count, + compressed_batch_rows_min, compressed_batch_rows_max, + compressed_batch_rows_avg, compressed_batch_rows_stddev, + compressed_batch_bytes_min, compressed_batch_bytes_max, + compressed_batch_bytes_avg, compressed_batch_bytes_stddev, + compressed_block_bytes_min, compressed_block_bytes_max, + compressed_block_bytes_avg, compressed_block_bytes_stddev +FROM observ_main_view; + +-- Re-compress before doing the bloom tests +SELECT compress_chunk(c) FROM show_chunks('t') c; + +-- Test DELETE stats with bloom filter matches +BEGIN; +DELETE FROM t WHERE a = 7 AND b = 2; +ROLLBACK; + +SELECT * FROM observ; -- see view definition above + +-- Test DELETE stats with bloom filter misses +BEGIN; +DELETE FROM t WHERE d = 200; +ROLLBACK; + +SELECT * FROM observ; -- see view definition above + +-- Test UPDATE stats with bloom filter matches +BEGIN; +UPDATE t SET a = 42 WHERE a = 7 AND b = 2; +ROLLBACK; + +SELECT * FROM observ; -- see view definition above + +-- Test UPDATE stats with bloom filter misses +BEGIN; +UPDATE t SET a = 42 WHERE d = 200; +ROLLBACK; + +SELECT * FROM observ; -- see view definition above + + +DROP VIEW observ; +DROP VIEW observ_main_view; +DROP TABLE t;